kit

native.c (177755B)

---

 /* src/arch/rv64/native.c — RISC-V (RV64GC, LP64D) NativeTarget implementation.
  *
  * Mirrors the aa64 reference (src/arch/aa64/native.c): a physical-emission
  * NativeTarget driven at -O0 by the shared NativeDirectTarget and at -O1+ by
  * the optimizer emit path. ABI decisions go through the abi/ interface; this
  * file owns only ISA emission and the RV64 frame layout.
  *
  * Frame model (single, top-record): s0 (x8) is the frame pointer anchored at
  * the saved s0/ra pair; slots live below s0 at positive byte offsets `off`
  * (address = s0 - off); outgoing args sit at the bottom of the frame (sp+0..).
  *   frame_size  = align16(16 + cum_off + max_outgoing + va_save_sz)
  *   fp_pair_off = frame_size - 16 - va_save_sz   (saved pair, sp-relative)
  *   CFA = s0 + (frame_size - fp_pair_off)
  * RISC-V has no condition flags: comparisons materialize a 0/1 via SLT/SLTU or
  * FLT/FLE; branches compare two registers directly. x0 is a hardware zero. */
 
 #include <string.h>
 
 #include "abi/abi.h"
 #include "arch/riscv/asm.h"
 #include "arch/riscv/isa.h"
 #include "arch/riscv/regs.h"
 #include "arch/riscv/rv64.h"
 #include "arch/riscv/variant.h"
 #include "asm/asm.h"
 #include "asm/asm_lex.h"
 #include "cg/native_argmove.h"
 #include "cg/native_asm.h"
 #include "cg/native_direct_target.h"
 #include "cg/native_frame.h"
 #include "cg/type.h"
 #include "core/arena.h"
 #include "core/bytes.h"
 #include "core/pool.h"
 #include "core/slice.h"
 #include "obj/obj.h"
 
 enum {
   RV_TMP0 = 5u,  /* t0: backend-private temp (reserved, never allocable) */
   RV_TMP1 = 6u,  /* t1: backend-private temp */
   RV_TMP2 = 7u,  /* t2: backend/asm temp (reserved in phys table) */
   RV_TMP3 = 28u, /* t3: backend/asm temp (reserved in phys table) */
   RV_FTMP0 = 0u, /* ft0: backend/asm FP temp */
   RV_FTMP1 = 1u, /* ft1: backend/asm FP temp */
   RV_FA0 = 10u,  /* fa0..fa7 = f10..f17 (FP arg/return registers) */
   RV_FA7 = 17u,
   /* Single-pass (-O0) tcc-style prologue. The frame-independent entry
    *   `addi sp,sp,-(frame_save_size+va_save) ; sd s0,0(sp) ; sd ra,ptr(sp) ;
    *    addi s0,sp,0`
    * (RV_NDT_FIXED_ENTRY_WORDS) is emitted live in rv_func_begin: it pre-decrements
    * sp by just the saved-pair + variadic-GP-save area, stores the s0/ra pair at
    * the top, and anchors s0 there. The pre-decrement is always small
    * (<= 16 + 64 = 80 on rv64, 8 + 32 = 40 on rv32), so it always fits imm12 and
    * needs no lui+add far form. Only the second `sub sp` that grows the rest of
    * the frame below the pair is deferred (patched once frame_size is final); the
    * single-pass path uses no callee-saved registers (enforced in the NDT scratch
    * acquire) and RISC-V has no Windows-style stack probe, so nothing else is
    * deferred and the reserved region is tiny: worst-case sub = lui(1)+addi(1)+
    * add(1) = 3 words. The sret + variadic GP spills are s0-relative and emitted
    * live after the region (rv_emit_entry_save_stores). */
   RV_NDT_FIXED_ENTRY_WORDS = 4u,
   RV_NDT_SUB_WORDS = 4u, /* region buffer: worst-case far `sub sp` is 3 (+1 pad) */
   /* Known-frame (-O1) prologues are emitted directly, not into the fixed -O0
    * NOP region, and additionally save callee-saved registers (up to 11 int + 12
    * fp, each up to 4 words for a far s0-relative offset) on top of the header,
    * sret, and variadic spills. Size the build buffer for the worst case. */
   RV_KNOWN_PROLOGUE_WORDS = 192u,
 };
 
 /* s1..s11 (11) + fs0..fs11 (12); separate int/fp collect arrays use this cap.
  */
 #define RV_MAX_CALLEE_SAVES 16u
 #define RV_MAX_REG_ARG_MOVES 16u
 
 extern void debug_emit_row(Debug*, ObjSecId text_section, u32 offset, SrcLoc);
 extern void debug_func_pc_range(Debug*, ObjSecId text_section, u32 begin_ofs,
                                 u32 end_ofs);
 
 /* ============================ low-level emit ============================ */
 
 void rv64_emit32(MCEmitter* mc, u32 word) {
   /* Read the pre-write offset (only the -g line table needs it) before
    * mc_emit32 advances the cursor. */
   if (mc->debug) {
     u32 ofs = obj_pos(mc->obj, mc->section_id);
     mc_emit32(mc, word);
     debug_emit_row(mc->debug, mc->section_id, ofs, mc->loc);
   } else {
     mc_emit32(mc, word);
   }
 }
 
 void rv64_emit16(MCEmitter* mc, u32 halfword) {
   u8 b[2];
   u32 ofs = obj_pos(mc->obj, mc->section_id);
   b[0] = (u8)(halfword & 0xff);
   b[1] = (u8)((halfword >> 8) & 0xff);
   mc_emit_bytes(mc, b, sizeof b);
   if (mc->debug) debug_emit_row(mc->debug, mc->section_id, ofs, mc->loc);
 }
 
 static void rv_patch32(ObjBuilder* obj, ObjSecId sec, u32 off, u32 word) {
   u8 b[4];
   wr_u32_le(b, word);
   obj_patch(obj, sec, off, b, sizeof b);
 }
 
 static int fits_i12(i64 v) { return v >= -2048 && v <= 2047; }
 static int fits_i32(i64 v) {
   return v >= (i64)(i32)0x80000000 && v <= (i64)(i32)0x7fffffff;
 }
 
 static i64 floor_div_4096(i64 v) {
   if (v >= 0) return v / 4096;
   return -((-v + 4095) / 4096);
 }
 
 static void rv_emit_li32(const RiscvVariant* v, MCEmitter* mc, u32 rd,
                          i32 imm) {
   if (imm >= -2048 && imm <= 2047) {
     rv64_emit32(mc, rv_addi(rd, RV_ZERO, imm));
     return;
   }
   {
     i64 hi64 = floor_div_4096((i64)imm + 0x800);
     i32 hi = (i32)hi64;
     i32 lo = (i32)((i64)imm - hi64 * 4096);
     rv64_emit32(mc, rv_lui(rd, (u32)hi & 0xfffffu));
     /* ADDIW is RV64-only; on RV32 the value fits 32 bits so plain ADDI is
      * exact (and identical to ADDIW's low result on RV64). */
     if (lo)
       rv64_emit32(mc,
                   v->has_w_forms ? rv_addiw(rd, rd, lo) : rv_addi(rd, rd, lo));
   }
 }
 
 static i32 sext12(u32 v) {
   v &= 0xfffu;
   return (v & 0x800u) ? (i32)v - 4096 : (i32)v;
 }
 
 /* Builds a full XLEN-wide value. The recursion / slli-12 chain assembles bits
  * above 32 and is only ever reached on rv64 (a single rv32 register cannot hold
  * a value wider than 32 bits — the cg layer legalizes those into pairs). */
 static void rv_emit_li64(const RiscvVariant* v, MCEmitter* mc, u32 rd,
                          u64 imm) {
   if (fits_i32((i64)imm)) {
     rv_emit_li32(v, mc, rd, (i32)(i64)imm);
     return;
   }
   {
     i32 lo = sext12((u32)imm);
     u64 hi = (imm - (u64)(i64)lo) >> 12;
     rv_emit_li64(v, mc, rd, hi);
     rv64_emit32(mc, rv_slli(rd, rd, 12));
     if (lo) rv64_emit32(mc, rv_addi(rd, rd, lo));
   }
 }
 
 /* sf!=0 selects a full native-width materialization; sf==0 a 32-bit value. On
  * rv32 the native width is 32, so the wide branch collapses to the 32-bit
  * path. */
 static void rv_emit_load_imm(const RiscvVariant* v, MCEmitter* mc, u32 sf,
                              u32 rd, i64 imm) {
   if (!sf || v->xlen == 32u) {
     rv_emit_li32(v, mc, rd, (i32)imm);
     return;
   }
   if (fits_i32(imm))
     rv_emit_li32(v, mc, rd, (i32)imm);
   else
     rv_emit_li64(v, mc, rd, (u64)imm);
 }
 
 /* rd = base + off, materializing the offset when it exceeds imm12. Uses RV_TMP1
  * as scratch for the wide path, so callers must keep RV_TMP1 free. */
 static void rv_emit_addr_adjust(const RiscvVariant* v, MCEmitter* mc, u32 rd,
                                 u32 base, i32 off) {
   if (off == 0) {
     if (rd != base) rv64_emit32(mc, rv_addi(rd, base, 0));
     return;
   }
   if (fits_i12(off)) {
     rv64_emit32(mc, rv_addi(rd, base, off));
     return;
   }
   rv_emit_load_imm(v, mc, 1, RV_TMP1, (i64)off);
   rv64_emit32(mc, rv_add(rd, base, RV_TMP1));
 }
 
 static u32 enc_int_store(const RiscvVariant* v, u32 nbytes, u32 src, u32 base,
                          i32 off) {
   switch (nbytes) {
     case 1:
       return rv_sb(src, base, off);
     case 2:
       return rv_sh(src, base, off);
     case 4:
       return rv_sw(src, base, off);
     default:
       /* The widest GPR store is SD on rv64, SW on rv32. */
       return v->ptr_bytes == 8u ? rv_sd(src, base, off) : rv_sw(src, base, off);
   }
 }
 static u32 enc_int_load(const RiscvVariant* v, u32 nbytes, int sign_ext, u32 rd,
                         u32 base, i32 off) {
   switch (nbytes) {
     case 1:
       return sign_ext ? rv_lb(rd, base, off) : rv_lbu(rd, base, off);
     case 2:
       return sign_ext ? rv_lh(rd, base, off) : rv_lhu(rd, base, off);
     case 4:
       /* LWU (zero-extending 32-bit load) is RV64-only; on rv32 a 4-byte load
        * is just LW (no wider container to zero-extend into). */
       return sign_ext || v->xlen == 32u ? rv_lw(rd, base, off)
                                         : rv_lwu(rd, base, off);
     default:
       /* The widest GPR load is LD on rv64, LW on rv32. */
       return v->ptr_bytes == 8u ? rv_ld(rd, base, off) : rv_lw(rd, base, off);
   }
 }
 
 /* Pointer-width GPR load/store (GOT entries, frame-value bases, saved ra/s0,
  * sret/indirect/va_list pointers): LD/SD on rv64, LW/SW on rv32. */
 static u32 rv_ld_ptr(const RiscvVariant* v, u32 rd, u32 base, i32 off) {
   return v->ptr_bytes == 8u ? rv_ld(rd, base, off) : rv_lw(rd, base, off);
 }
 static u32 rv_sd_ptr(const RiscvVariant* v, u32 src, u32 base, i32 off) {
   return v->ptr_bytes == 8u ? rv_sd(src, base, off) : rv_sw(src, base, off);
 }
 
 /* ============================ target state ============================ */
 
 /* Frame slots and callee-save records live in the shared NativeFrame
  * bookkeeping (cg/native_frame.h); these aliases keep the rv64-local spellings.
  */
 typedef NativeFrameSlotEntry RvNativeSlot;
 typedef NativeFrameCalleeSave RvCalleeSave;
 
 typedef enum RvPatchKind { RV_PATCH_ALLOCA } RvPatchKind;
 
 typedef struct RvPatch {
   u8 kind; /* RvPatchKind */
   u32 pos;
   u32 dst_reg;
 } RvPatch;
 
 typedef struct RvNativeTarget {
   NativeTarget base;
   /* Immutable per-XLEN descriptor (rv32 / rv64), set once in the constructor
    * from c->target.arch. Every XLEN-dependent emit site reads it; with the
    * rv64 variant each site reproduces the historical literal exactly. */
   const RiscvVariant* variant;
   SrcLoc loc;
   const CGFuncDesc* func;
 
   /* Shared frame bookkeeping: slot table, cum_off, max_outgoing, callee-save
    * set, and the known_frame / has_alloca / frame_final flags. */
   NativeFrame frame;
   u32 frame_size_final;
   u32 fp_pair_off;
   u32 minimal_prologue_words; /* known-frame path: exact prologue length, else 0
                                */
 
   /* Known-frame (-O1) leaf no-frame tier (aa64's slim_prologue equivalent),
    * settled in rv_func_begin_known_frame; always 0 on the single-pass path. A
    * leaf with no callee-saves, no body slots, no outgoing args, no
    * sret/variadic and register-only params never reads s0 nor clobbers ra, so
    * it emits NO prologue and a bare `ret` — the whole frame setup/teardown is
    * elided. RISC-V has no pre/post-indexed store, so aa64's fp_at_bottom fold
    * would save zero instructions on a kept frame and is intentionally not
    * ported (see doc/plan/ARCH.md §2); this leaf tier is the rv64 win. */
   u8 slim_prologue;
 
   u32 incoming_stack_size; /* fixed-param stack bytes (tail-call check) */
   u32 next_param_int;
   u32 next_param_fp;
   u32 next_param_stack;
   u8 has_sret;
   u8 is_variadic;
   NativeFrameSlot sret_ptr_slot;
 
   RvPatch* patches;
   u32 npatches;
   u32 patches_cap;
   u32 nalloca;
 
   u32 func_start;
   u32 prologue_pos;
   u32 prologue_region_words; /* single-pass: reserved deferred-`sub` region */
   MCLabel epilogue_label;
 } RvNativeTarget;
 
 static RvNativeTarget* rv_of(NativeTarget* t) { return (RvNativeTarget*)t; }
 
 static _Noreturn void rv_panic(RvNativeTarget* a, const char* msg) {
   compiler_panic(a->base.c, a->loc, "rv64 native target: %s", msg);
 }
 
 static RvNativeSlot* rv_slot_get(RvNativeTarget* a, NativeFrameSlot fs) {
   return native_frame_slot_at(&a->frame, fs);
 }
 
 /* s0-relative byte offset of a frame slot's base (address = s0 + ret). */
 static i32 rv_s0_off_slot(const RvNativeSlot* s) { return -(i32)s->off; }
 
 static u32 rv_va_save_sz(const RvNativeTarget* a) {
   /* ABI-derived: the variadic register-save area is gp_reg_count*gp_slot_size
    * (a0..a7 = 64 bytes for LP64D, 32 for ILP32). Only present in variadics. */
   return a->is_variadic ? native_frame_va_save_bytes(a->base.c->abi) : 0u;
 }
 
 /* s0-relative byte offset of incoming stack arg at byte_off. Stack args sit
  * just above the saved pair; the variadic GP save area (when present) is
  * contiguous with them at [s0 + frame_save_size). */
 static i32 rv_s0_off_in_arg(const RvNativeTarget* a, u32 byte_off) {
   u32 base = a->variant->frame_save_size;
   if (a->is_variadic) base += rv_va_save_sz(a);
   return (i32)(base + byte_off);
 }
 
 /* Callee-saved registers are homed just below the locals at rv_save_off() —
  * they are NOT frame slots, so the frame size must reserve their bytes
  * explicitly. Integer saves are ptr_bytes wide (sd on rv64, sw on rv32); FP
  * saves are always 8 bytes (fsd, even on rv32d). On rv64 both are 8 so the sum
  * is identical to the historical ncallee_saves*8. Zero at -O0. */
 static u32 rv_callee_save_bytes(const RvNativeTarget* a) {
   u32 ptr = a->variant->ptr_bytes;
   u32 i, bytes = 0;
   for (i = 0; i < a->frame.ncallee_saves; ++i)
     bytes += a->frame.callee_saves[i].cls == NATIVE_REG_FP ? 8u : ptr;
   return bytes;
 }
 
 static u32 rv_frame_size(const RvNativeTarget* a) {
   u32 raw = a->variant->frame_save_size + a->frame.cum_off +
             rv_callee_save_bytes(a) + a->frame.max_outgoing + rv_va_save_sz(a);
   return align_up_u32(raw, 16u);
 }
 
 static u32 rv_fp_pair_off(const RvNativeTarget* a, u32 frame_size) {
   return frame_size - a->variant->frame_save_size - rv_va_save_sz(a);
 }
 
 /* ============================ type helpers ============================ */
 
 /* Scalar size/align/mem/class/loc constructors are shared in native_target.h
  * (native_type_size, native_type_align, native_mem_for_type,
  * native_class_for_type_fp_le8, native_loc_reg, native_loc_stack,
  * native_loc_is_fp). loc_reg's mask is arch-specific and stays here. */
 
 /* True when a scalar value is WIDER than XLEN's natural single-register width,
  * i.e. it needs the "wide" (rv64 64-bit) ops rather than the base ops. On rv64
  * a pointer is 8 bytes and counts as wide alongside i64/double; on rv32 a
  * pointer is 4 bytes and fits a single 32-bit register, so it is NOT wide and
  * the base (non-W) ops apply. (Kept named rv_is_64 to minimize churn; for the
  * rv64 variant the result is byte-identical to the old predicate.) */
 static int rv_is_64(NativeTarget* t, KitCgTypeId type) {
   const RiscvVariant* v = rv_of(t)->variant;
   return native_type_size(t, type) >= 8u ||
          (v->xlen == 64u && cg_type_is_ptr(t->c, type));
 }
 
 /* Scalar byte width of a register operand. Reads the NDT-stamped szinfo
  * descriptor (one byte) when present, else falls back to the live type query —
  * byte-identical to native_type_size for in-range scalars. */
 static u32 loc_size32(NativeTarget* t, NativeLoc loc) {
   if (loc.szinfo & NATIVE_SZINFO_VALID) return native_szinfo_size(loc.szinfo);
   return native_type_size(t, loc.type);
 }
 
 static u32 rv_internal_tmp(RvNativeTarget* a, u32 avoid_mask) {
   static const u8 temps[] = {RV_TMP1, RV_TMP0, RV_TMP2, RV_TMP3};
   for (u32 i = 0; i < sizeof temps / sizeof temps[0]; ++i)
     if (!(avoid_mask & (1u << temps[i]))) return temps[i];
   rv_panic(a, "no backend temporary for frame component");
 }
 
 /* Load a typed scalar value from a frame slot. Address components retain their
  * own type because spill-slot coloring may reuse a wider descriptor; loading
  * the descriptor width would consume neighboring bytes. Far s0 offsets are
  * materialized through a distinct backend-owned register. */
 static void rv_load_frame_component(RvNativeTarget* a, u32 dst,
                                     NativeFrameSlot slot, KitCgTypeId type,
                                     u32 avoid_mask) {
   RvNativeSlot* s = rv_slot_get(a, slot);
   i32 off = rv_s0_off_slot(s);
   u32 size;
   if (!type) rv_panic(a, "frame-value component has no exact type");
   size = native_type_size(&a->base, type);
   if (fits_i12(off)) {
     rv64_emit32(a->base.mc,
                 enc_int_load(a->variant, size, 0, dst, RV_S0, off));
     return;
   }
   {
     u32 tmp = rv_internal_tmp(a, avoid_mask | (1u << dst) | (1u << RV_S0));
     rv_emit_load_imm(a->variant, a->base.mc, 1, tmp, (i64)off);
     rv64_emit32(a->base.mc, rv_add(tmp, RV_S0, tmp));
     rv64_emit32(a->base.mc,
                 enc_int_load(a->variant, size, 0, dst, tmp, 0));
   }
 }
 
 /* The original rv_is_64 predicate exactly, but sourcing the width from the
  * cached descriptor: `size >= 8 || (xlen==64 && is_ptr)`. The is_ptr clause
  * keeps the rv32 4-byte-pointer distinction (a stamped 4-byte pointer caches
  * size 4, so >= 8 is false and the xlen guard decides). Byte-identical. */
 static int loc_is_64(NativeTarget* t, NativeLoc loc) {
   const RiscvVariant* v = rv_of(t)->variant;
   return loc_size32(t, loc) >= 8u ||
          (v->xlen == 64u && cg_type_is_ptr(t->c, loc.type));
 }
 
 static u32 loc_reg(NativeLoc loc) { return loc.v.reg & 0x1fu; }
 
 /* ============================ register tables ============================ */
 
 #define RV_PHYS_INT_ARG(r, idx)                        \
   {.reg = (r),                                         \
    .cls = NATIVE_REG_INT,                              \
    .abi_index = (idx),                                 \
    .flags = NATIVE_REG_CALLER_SAVED | NATIVE_REG_ARG | \
             ((idx) < 2u ? NATIVE_REG_RET : 0),         \
    .spill_cost = 1u,                                   \
    .copy_cost = 1u}
 #define RV_PHYS_INT_CALLER(r)                               \
   {.reg = (r),                                              \
    .cls = NATIVE_REG_INT,                                   \
    .abi_index = 0xffu,                                      \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLER_SAVED, \
    .spill_cost = 1u,                                        \
    .copy_cost = 1u}
 #define RV_PHYS_INT_CALLEE(r)                               \
   {.reg = (r),                                              \
    .cls = NATIVE_REG_INT,                                   \
    .abi_index = 0xffu,                                      \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLEE_SAVED, \
    .spill_cost = 4u,                                        \
    .copy_cost = 1u}
 #define RV_PHYS_INT_RESERVED_ROLE(r, role) \
   {.reg = (r),                              \
    .cls = NATIVE_REG_INT,                   \
    .abi_index = 0xffu,                      \
    .flags = NATIVE_REG_RESERVED | (role),   \
    .spill_cost = 0u,                        \
    .copy_cost = 0u}
 #define RV_PHYS_INT_RESERVED(r) \
   RV_PHYS_INT_RESERVED_ROLE((r), NATIVE_REG_NONE)
 #define RV_PHYS_INT_RESERVED_CALLER(r) \
   RV_PHYS_INT_RESERVED_ROLE((r), NATIVE_REG_CALLER_SAVED)
 
 /* t0..t3 (x5,x6,x7,x28) are backend-private hook/asm temporaries
  * (RV_TMP0..RV_TMP3), reserved and never exposed as optimizer operand
  * locations. t4/t5/t6 form the O1 instruction-scoped operand-temp bank,
  * disjoint from backend temporaries so a hook cannot overwrite a leased
  * operand. Three are needed, not two: a load/store target can need an address
  * materialization alongside a 3-operand operation. aarch64 declares three for
  * the same reason; x64 needs only two because it folds the index into its
  * addressing mode. Location MIR retains spills as frame locations until the
  * consuming instruction opens this lease scope.
  *
  * rv_ndt_int_allocable is the NDT (-O0) value-cache / scratch pool and is read
  * only
  * by NativeDirectTarget; the optimizer allocates over the phys[] ALLOCABLE flags
  * (s1..s11, callee-saved, saved by its -O1 prologue) instead. The NDT pool is
  * the caller-saved argument registers a0..a7: the -O0 cache flushes to frame
  * homes at every call/branch/return, so nothing needs to survive in a register
  * across a call, and caller-saved regs keep the prologue free of callee-save
  * spills — which also keeps the single-pass prologue placeholder (a fixed
  * RV_PROLOGUE_WORDS region) from ever overflowing on large/variadic frames the
  * way callee-save offsets would. Incoming args are spilled to homes at entry
  * before any body op, so caching in a0..a7 cannot clobber a live parameter. */
 static const Reg rv_ndt_int_allocable[] = {10u, 11u, 12u, 13u, 14u,
                                            15u, 16u, 17u}; /* a0..a7 */
 static const Reg rv_int_scratch[] = {29u, 30u, 31u};  /* t4, t5, t6 */
 static const Reg rv_int_asm_temps[] = {RV_TMP2, RV_TMP3};
 static const Reg rv_direct_asm_int[] = {10u, 11u, 12u, 13u, 14u, 15u,
                                         16u, 17u, 29u, 30u, 31u};
 
 static const NativePhysRegInfo rv_int_phys[] = {
     RV_PHYS_INT_RESERVED(0u), /* zero */
     RV_PHYS_INT_RESERVED(1u), /* ra */
     RV_PHYS_INT_RESERVED(2u), /* sp */
     RV_PHYS_INT_RESERVED(3u), /* gp */
     RV_PHYS_INT_RESERVED(4u), /* tp */
     RV_PHYS_INT_RESERVED_CALLER(5u), /* t0 = TMP0 */
     RV_PHYS_INT_RESERVED_CALLER(6u), /* t1 = TMP1 */
     RV_PHYS_INT_RESERVED_CALLER(7u), /* t2 = TMP2 (backend/asm) */
     RV_PHYS_INT_RESERVED_ROLE(8u, NATIVE_REG_CALLEE_SAVED), /* s0/fp */
     RV_PHYS_INT_CALLEE(9u),   /* s1 */
     RV_PHYS_INT_ARG(10u, 0u),  RV_PHYS_INT_ARG(11u, 1u),
     RV_PHYS_INT_ARG(12u, 2u),  RV_PHYS_INT_ARG(13u, 3u),
     RV_PHYS_INT_ARG(14u, 4u),  RV_PHYS_INT_ARG(15u, 5u),
     RV_PHYS_INT_ARG(16u, 6u),  RV_PHYS_INT_ARG(17u, 7u),
     RV_PHYS_INT_CALLEE(18u),   RV_PHYS_INT_CALLEE(19u),
     RV_PHYS_INT_CALLEE(20u),   RV_PHYS_INT_CALLEE(21u),
     RV_PHYS_INT_CALLEE(22u),   RV_PHYS_INT_CALLEE(23u),
     RV_PHYS_INT_CALLEE(24u),   RV_PHYS_INT_CALLEE(25u),
     RV_PHYS_INT_CALLEE(26u),   RV_PHYS_INT_CALLEE(27u),
     RV_PHYS_INT_RESERVED_CALLER(28u), /* t3 = TMP3 (backend/asm) */
     RV_PHYS_INT_RESERVED_CALLER(29u), /* t4 = O1 operand temp / O0 scratch */
     RV_PHYS_INT_RESERVED_CALLER(30u), /* t5 = O1 operand temp / O0 scratch */
     RV_PHYS_INT_RESERVED_CALLER(31u), /* t6 = O1 operand temp / O0 scratch */
 };
 
 #define RV_PHYS_FP_ARG(r, idx)                         \
   {.reg = (r),                                         \
    .cls = NATIVE_REG_FP,                               \
    .abi_index = (idx),                                 \
    .flags = NATIVE_REG_CALLER_SAVED | NATIVE_REG_ARG | \
             ((idx) < 2u ? NATIVE_REG_RET : 0),         \
    .spill_cost = 1u,                                   \
    .copy_cost = 1u}
 #define RV_PHYS_FP_CALLER(r)                                \
   {.reg = (r),                                              \
    .cls = NATIVE_REG_FP,                                    \
    .abi_index = 0xffu,                                      \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLER_SAVED, \
    .spill_cost = 1u,                                        \
    .copy_cost = 1u}
 #define RV_PHYS_FP_CALLEE(r)                                \
   {.reg = (r),                                              \
    .cls = NATIVE_REG_FP,                                    \
    .abi_index = 0xffu,                                      \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLEE_SAVED, \
    .spill_cost = 4u,                                        \
    .copy_cost = 1u}
 #define RV_PHYS_FP_RESERVED(r)                   \
   {.reg = (r),                                   \
    .cls = NATIVE_REG_FP,                         \
    .abi_index = 0xffu,                           \
    .flags = NATIVE_REG_RESERVED |                \
             NATIVE_REG_CALLER_SAVED,             \
    .spill_cost = 0u,                             \
    .copy_cost = 0u}
 
 /* NDT (-O0) fp value-cache / scratch pool (read only by NativeDirectTarget; the
  * optimizer uses the phys[] ALLOCABLE flags, which also cover the callee-saved
  * fs0..fs11). Caller-saved only — the fa0..fa7 arg registers, then ft4..ft7 and
  * ft8..ft11 — for the same reasons as the int pool: the -O0 cache flushes at
  * every call/branch/return, so caller-saved suffices, and keeping callee-saves
  * out of the pool keeps the fixed single-pass prologue placeholder from
  * overflowing. fa0..fa7 come FIRST (mirroring the int pool's a0..a7):
  * nd_cache_alloc scans the pool in order, so fronting the arg registers lets a
  * producer materialize a call's fp args directly into their ABI registers
  * (Lever 1, via api_pack_call_args_in_order). ft0/ft1 are backend/asm
  * temporaries; ft2/ft3 form the O1 operand-temp bank and O0 scratch policy. */
 static const Reg rv_ndt_fp_allocable[] = {
     10u, 11u, 12u, 13u, 14u, 15u, 16u, 17u, 4u,  5u,  6u, 7u,
     28u, 29u, 30u, 31u}; /* fa0-7,ft4-7,ft8-11 */
 static const Reg rv_fp_scratch[] = {2u, 3u};               /* ft2, ft3 */
 static const Reg rv_fp_asm_temps[] = {RV_FTMP0, RV_FTMP1};
 static const Reg rv_direct_asm_fp[] = {10u, 11u, 12u, 13u, 14u, 15u,
                                        16u, 17u, 4u,  5u,  6u,  7u,
                                        28u, 29u, 30u, 31u};
 
 static const NativePhysRegInfo rv_fp_phys[] = {
     RV_PHYS_FP_RESERVED(0u), /* ft0 = FTMP0 */
     RV_PHYS_FP_RESERVED(1u), /* ft1 = FTMP1 */
     RV_PHYS_FP_RESERVED(2u), /* ft2 = O1 operand temp / O0 scratch */
     RV_PHYS_FP_RESERVED(3u), /* ft3 = O1 operand temp / O0 scratch */
     RV_PHYS_FP_CALLER(4u),   RV_PHYS_FP_CALLER(5u),   RV_PHYS_FP_CALLER(6u),
     RV_PHYS_FP_CALLER(7u),   RV_PHYS_FP_CALLEE(8u),   RV_PHYS_FP_CALLEE(9u),
     RV_PHYS_FP_ARG(10u, 0u), RV_PHYS_FP_ARG(11u, 1u), RV_PHYS_FP_ARG(12u, 2u),
     RV_PHYS_FP_ARG(13u, 3u), RV_PHYS_FP_ARG(14u, 4u), RV_PHYS_FP_ARG(15u, 5u),
     RV_PHYS_FP_ARG(16u, 6u), RV_PHYS_FP_ARG(17u, 7u), RV_PHYS_FP_CALLEE(18u),
     RV_PHYS_FP_CALLEE(19u),  RV_PHYS_FP_CALLEE(20u),  RV_PHYS_FP_CALLEE(21u),
     RV_PHYS_FP_CALLEE(22u),  RV_PHYS_FP_CALLEE(23u),  RV_PHYS_FP_CALLEE(24u),
     RV_PHYS_FP_CALLEE(25u),  RV_PHYS_FP_CALLEE(26u),  RV_PHYS_FP_CALLEE(27u),
     RV_PHYS_FP_CALLER(28u),  RV_PHYS_FP_CALLER(29u),  RV_PHYS_FP_CALLER(30u),
     RV_PHYS_FP_CALLER(31u),
 };
 
 static const NativeAllocClassInfo rv_classes[] = {
     {.cls = NATIVE_REG_INT,
      .ndt_allocable = rv_ndt_int_allocable,
      .ndt_allocable_count =
          sizeof rv_ndt_int_allocable / sizeof rv_ndt_int_allocable[0],
      .scratch = rv_int_scratch,
      .nscratch = sizeof rv_int_scratch / sizeof rv_int_scratch[0],
      .emit_temps = rv_int_scratch,
      .nemit_temps = sizeof rv_int_scratch / sizeof rv_int_scratch[0],
      .asm_temps = rv_int_asm_temps,
      .nasm_temps = sizeof rv_int_asm_temps / sizeof rv_int_asm_temps[0],
      .direct_asm_allocable = rv_direct_asm_int,
      .ndirect_asm_allocable =
          sizeof rv_direct_asm_int / sizeof rv_direct_asm_int[0],
      .emit_cache_mask = (1u << 29u) | (1u << 30u) | (1u << 31u),
      .phys = rv_int_phys,
      .nphys = sizeof rv_int_phys / sizeof rv_int_phys[0]},
     {.cls = NATIVE_REG_FP,
      .ndt_allocable = rv_ndt_fp_allocable,
      .ndt_allocable_count =
          sizeof rv_ndt_fp_allocable / sizeof rv_ndt_fp_allocable[0],
      .scratch = rv_fp_scratch,
      .nscratch = sizeof rv_fp_scratch / sizeof rv_fp_scratch[0],
      .emit_temps = rv_fp_scratch,
      .nemit_temps = sizeof rv_fp_scratch / sizeof rv_fp_scratch[0],
      .asm_temps = rv_fp_asm_temps,
      .nasm_temps = sizeof rv_fp_asm_temps / sizeof rv_fp_asm_temps[0],
      .direct_asm_allocable = rv_direct_asm_fp,
      .ndirect_asm_allocable =
          sizeof rv_direct_asm_fp / sizeof rv_direct_asm_fp[0],
      .emit_cache_mask = (1u << 2u) | (1u << 3u),
      .phys = rv_fp_phys,
      .nphys = sizeof rv_fp_phys / sizeof rv_fp_phys[0]},
 };
 
 /* Resolve a register name ("a7", "fa0", ...) to its (class, Reg). Powers the
  * optimizer's inline-asm clobber masks and explicit hard-register operands
  * ("{a7}" from a GNU local register variable). x0..x31 are DWARF 0..31; the
  * FP bank f0..f31 is DWARF 32..63. Returns non-zero for a non-register name
  * (cc/memory/unknown), which the caller skips. */
 static int rv_resolve_name(const NativeRegInfo* ri, Slice name, Reg* out,
                            NativeAllocClass* cls_out) {
   char buf[16];
   uint32_t dwarf;
   (void)ri;
   if (!name.s || !name.len || name.len >= sizeof buf) return 1;
   memcpy(buf, name.s, name.len);
   buf[name.len] = '\0';
   if (rv64_register_index(buf, &dwarf) != 0) return 1;
   if (dwarf <= 31u) {
     *cls_out = NATIVE_REG_INT;
     *out = (Reg)dwarf;
     return 0;
   }
   if (dwarf >= 32u && dwarf <= 63u) {
     *cls_out = NATIVE_REG_FP;
     *out = (Reg)(dwarf - 32u);
     return 0;
   }
   return 1;
 }
 
 static int rv_asm_operand_reg_ok(const NativeRegInfo* ri, NativeAllocClass cls,
                                  Reg reg) {
   (void)ri;
   if (cls == NATIVE_REG_INT) {
     if (reg == 9u) return 1;                /* s1 */
     if (reg >= 10u && reg <= 17u) return 1; /* a0..a7 */
     if (reg >= 18u && reg <= 27u) return 1; /* s2..s11 */
     if (reg == 31u) return 1;               /* t6 */
     return 0;
   }
   if (cls == NATIVE_REG_FP) return reg >= 4u && reg <= 31u;
   return 0;
 }
 
 static int rv_asm_constraint_reg(const NativeRegInfo* ri, const char* body,
                                  NativeAllocClass* cls_out, Reg* fixed_out,
                                  u32* allowed_mask_out) {
   (void)ri;
   if (!body || !body[0]) return 0;
   if (fixed_out) *fixed_out = REG_NONE;
   if (allowed_mask_out) *allowed_mask_out = 0;
   if (body[0] == 'r' && body[1] == '\0') {
     if (cls_out) *cls_out = NATIVE_REG_INT;
     return 1;
   }
   if (body[0] == 'f' && body[1] == '\0') {
     if (cls_out) *cls_out = NATIVE_REG_FP;
     return 1;
   }
   if (body[0] == 'c' && body[1] == 'r' && body[2] == '\0') {
     if (cls_out) *cls_out = NATIVE_REG_INT;
     if (allowed_mask_out) *allowed_mask_out = 0x0000ff00u; /* x8..x15 */
     return 1;
   }
   if (body[0] == 'c' && body[1] == 'f' && body[2] == '\0') {
     if (cls_out) *cls_out = NATIVE_REG_FP;
     if (allowed_mask_out) *allowed_mask_out = 0x0000ff00u; /* f8..f15 */
     return 1;
   }
   return 0;
 }
 
 static const NativeRegInfo rv_reg_info = {
     .classes = rv_classes,
     .nclasses = sizeof rv_classes / sizeof rv_classes[0],
     /* The NDT value-cache / scratch pool is the caller-saved a-regs (int) and
      * ft/fa regs (fp), so the -O0 path never needs a callee-saved register —
      * enabling the tcc-style deferred-`sub` prologue (see rv_func_begin). */
     .ndt_caller_saved_only = 1u,
     /* a0/fa0 are clobbered only by calls: rv div/rem/mul are R-type with
      * explicit operands (unlike x86-64's implicit-RAX div/mul), so a scalar
      * call result can stay cached in a0/fa0 across the following straight-line
      * ops and feed the next consumer with no mov — exactly as on aa64 (cf. the
      * aa64 ndt_result_reg_stable comment; doc/plan/PERF.md §4.2).
      * The NDT places the result directly in the ABI result reg post-call. */
     .ndt_result_reg_stable = 1u,
     .resolve_name = rv_resolve_name,
     .asm_operand_reg_ok = rv_asm_operand_reg_ok,
     .asm_constraint_reg = rv_asm_constraint_reg,
 };
 
 /* ============================ legality ============================ */
 
 static int rv_imm_legal(NativeTarget* t, NativeImmUse use, u32 op,
                         KitCgTypeId type, i64 imm) {
   /* SLLI/SRLI/SRAI shamt is shamt_bits wide: 6 bits (max 63) on rv64, 5 bits
    * (max 31) on rv32. */
   i64 shamt_max = (i64)((1u << rv_of(t)->variant->shamt_bits) - 1u);
   (void)type;
   switch (use) {
     case NATIVE_IMM_MOVE:
       return 1;
     case NATIVE_IMM_BINOP:
       switch ((BinOp)op) {
         case BO_IADD:
           return fits_i12(imm);
         case BO_ISUB:
           return fits_i12(-imm); /* emitted as ADDI with negated imm */
         case BO_AND:
         case BO_OR:
         case BO_XOR:
           return fits_i12(imm);
         case BO_SHL:
         case BO_SHR_S:
         case BO_SHR_U:
           return imm >= 0 && imm <= shamt_max;
         default:
           return 0;
       }
     case NATIVE_IMM_CMP:
       /* Only the compare-against-zero case is folded; a non-zero RHS is
        * materialized into a register first. This is a deliberate codegen
        * simplification, not an architectural constraint — SLTI/SLTIU do
        * accept a 12-bit signed immediate, but the comparison lowering does
        * not special-case immediate operands beyond zero. */
       return imm == 0;
     case NATIVE_IMM_ADDR_OFFSET:
       return fits_i12(imm);
   }
   return 0;
 }
 
 static int rv_addr_legal(NativeTarget* t, const NativeAddr* addr,
                          MemAccess mem) {
   (void)t;
   (void)mem;
   if (!addr) return 0;
   if (addr->index_kind != NATIVE_ADDR_INDEX_NONE) return 0;
   if (addr->base_kind != NATIVE_ADDR_BASE_REG &&
       addr->base_kind != NATIVE_ADDR_BASE_FRAME)
     return 0;
   return fits_i12(addr->offset);
 }
 
 /* ============================ memory ============================ */
 
 /* Emit the RISC-V PC-relative HI20/LO12 reloc pair:
  *   auipc(dst, 0)             ; R_RV_*_HI20 against `target_sym`
  *   .LpcrelHi local anchor at the auipc PC
  *   <follow_insn>             ; R_RV_PCREL_LO12_I against the anchor
  * The HI20 reloc kind (`hi20`) and the encoded follow-up instruction word
  * (`follow_insn` — e.g. addi dst,dst,0 for a direct addr or ld dst,dst,0 for a
  * GOT indirection) vary per call site; the anchor symbol, LO12 reloc kind, and
  * all addends/flags are fixed. The LO12 reloc is always *_LO12_I (I-type insn);
  * no caller currently needs the *_LO12_S form. Behavior-preserving extraction
  * of the auipc+anchor idiom shared by rv_emit_global_addr and
  * rv_load_label_addr. */
 static void rv_emit_pcrel_anchor(NativeTarget* t, u32 dst, ObjSymId target_sym,
                                  RelocKind hi20, u32 follow_insn) {
   MCEmitter* mc = t->mc;
   u32 sec = mc->section_id;
   u32 ap = mc_pos(mc);
   rv64_emit32(mc, rv_auipc(dst, 0));
   mc_emit_reloc_at(mc, sec, ap, hi20, target_sym, 0, 0, 0);
   {
     Sym an = pool_intern_slice(t->c->global, SLICE_LIT(".LpcrelHi"));
     ObjSymId anchor = obj_symbol(t->obj, an, SB_LOCAL, SK_OBJ, sec, (u64)ap, 0);
     u32 lp = mc_pos(mc);
     rv64_emit32(mc, follow_insn);
     mc_emit_reloc_at(mc, sec, lp, R_RV_PCREL_LO12_I, anchor, 0, 0, 0);
   }
 }
 
 /* Materialize the runtime address of a global into `dst`, including addend. */
 static void rv_emit_global_addr(RvNativeTarget* a, u32 dst, ObjSymId sym,
                                 i64 addend) {
   NativeTarget* t = &a->base;
   MCEmitter* mc = t->mc;
   if (obj_symbol_extern_via_got(t->c, t->obj, sym))
     rv_emit_pcrel_anchor(t, dst, sym, R_RV_GOT_HI20,
                          rv_ld_ptr(a->variant, dst, dst, 0));
   else
     rv_emit_pcrel_anchor(t, dst, sym, R_RV_PCREL_HI20, rv_addi(dst, dst, 0));
   if (addend) rv_emit_addr_adjust(a->variant, mc, dst, dst, (i32)addend);
 }
 
 /* Fold (base_reg << 0) + (index << scale) into RV_TMP0 via Zba. */
 static u32 rv_fold_index(RvNativeTarget* a, u32 base, u32 idx, u8 log2_scale) {
   MCEmitter* mc = a->base.mc;
   switch (log2_scale) {
     case 0:
       rv64_emit32(mc, rv_add(RV_TMP0, base, idx));
       break;
     case 1:
       rv64_emit32(mc, rv_sh1add(RV_TMP0, idx, base));
       break;
     case 2:
       rv64_emit32(mc, rv_sh2add(RV_TMP0, idx, base));
       break;
     default:
       rv64_emit32(mc, rv_sh3add(RV_TMP0, idx, base));
       break;
   }
   return RV_TMP0;
 }
 
 /* Resolve any NativeAddr to a base register + imm12 offset. RISC-V has no
  * indexed load/store, so an index is folded into RV_TMP0 via Zba; far offsets
  * and GLOBAL or exact-typed FRAME_VALUE components are materialized in the
  * backend-private RV_TMP0/RV_TMP1 bank. Plain FRAME bases remain s0-relative. */
 static void rv_resolve_mem_addr(RvNativeTarget* a, const NativeAddr* addr,
                                 u32* base_out, i32* off_out) {
   MCEmitter* mc = a->base.mc;
   u32 base;
   i32 off;
   switch (addr->base_kind) {
     case NATIVE_ADDR_BASE_REG:
       base = addr->base.reg & 0x1fu;
       off = addr->offset;
       break;
     case NATIVE_ADDR_BASE_FRAME: {
       RvNativeSlot* s = rv_slot_get(a, addr->base.frame);
       base = RV_S0;
       off = rv_s0_off_slot(s) + addr->offset;
       break;
     }
     case NATIVE_ADDR_BASE_FRAME_VALUE: {
       rv_load_frame_component(a, RV_TMP0, addr->base.frame, addr->base_type,
                               0u);
       base = RV_TMP0;
       off = addr->offset;
       break;
     }
     case NATIVE_ADDR_BASE_GLOBAL:
       rv_emit_global_addr(a, RV_TMP0, addr->base.global.sym,
                           addr->base.global.addend);
       base = RV_TMP0;
       off = addr->offset;
       break;
     default:
       rv_panic(a, "unsupported address base");
   }
   if (addr->index_kind == NATIVE_ADDR_INDEX_REG) {
     base = rv_fold_index(a, base, addr->index.reg & 0x1fu, addr->log2_scale);
   } else if (addr->index_kind == NATIVE_ADDR_INDEX_FRAME_VALUE) {
     u32 idx = rv_internal_tmp(a, 1u << base);
     rv_load_frame_component(a, idx, addr->index.frame, addr->index_type,
                             1u << base);
     base = rv_fold_index(a, base, idx, addr->log2_scale);
   }
   if (!fits_i12(off)) {
     u32 tmp = rv_internal_tmp(a, 1u << base);
     rv_emit_load_imm(a->variant, mc, 1, tmp, (i64)off);
     rv64_emit32(mc, rv_add(RV_TMP0, base, tmp));
     base = RV_TMP0;
     off = 0;
   }
   *base_out = base;
   *off_out = off;
 }
 
 /* Choose a backend-private integer register for an exact-width ABI transfer.
  * TMP0/TMP1 are considered last because address resolution normally owns that
  * pair; callers that choose before resolution therefore exclude them by
  * passing them as explicit avoids. */
 static u32 rv_part_tmp(RvNativeTarget* a, u32 avoid_a, u32 avoid_b,
                        u32 avoid_c) {
   static const u8 temps[] = {RV_TMP2, RV_TMP3, RV_TMP1, RV_TMP0};
   u32 i;
   for (i = 0; i < (u32)(sizeof temps / sizeof temps[0]); ++i) {
     u32 r = temps[i];
     if (r != avoid_a && r != avoid_b && r != avoid_c) return r;
   }
   rv_panic(a, "no temporary for exact-width ABI part");
   return RV_TMP2;
 }
 
 static u32 rv_addr_reg(const NativeAddr* addr, int index) {
   if (!index && addr->base_kind == NATIVE_ADDR_BASE_REG)
     return addr->base.reg & 0x1fu;
   if (index && addr->index_kind == NATIVE_ADDR_INDEX_REG)
     return addr->index.reg & 0x1fu;
   return REG_NONE;
 }
 
 /* TMP2/TMP3 are not borrowed by rv_resolve_mem_addr, so a packed store may
  * retain its source in either one while TMP0/TMP1 materialize the address. */
 static u32 rv_part_preserved_tmp(RvNativeTarget* a, const NativeAddr* addr) {
   u32 base = rv_addr_reg(addr, 0);
   u32 index = rv_addr_reg(addr, 1);
   if (RV_TMP2 != base && RV_TMP2 != index) return RV_TMP2;
   if (RV_TMP3 != base && RV_TMP3 != index) return RV_TMP3;
   rv_panic(a, "exact-width ABI address consumes transfer temporaries");
   return RV_TMP2;
 }
 
 /* RISC-V has no 3/5/6/7-byte scalar load/store. ABI parts of those semantic
  * sizes still occupy one packed integer register, so move them with bounded
  * low-to-high 4/2/1 (or RV32 2/1) accesses and explicitly pack/unpack the
  * register bits. This keeps every memory access within the semantic object;
  * the separately rounded ABI carrier/stack-slot size is not an access width. */
 static void rv_emit_exact_int_part(RvNativeTarget* a, int is_load, u32 reg,
                                    NativeAddr addr, u32 size) {
   const RiscvVariant* v = a->variant;
   MCEmitter* mc = a->base.mc;
   NativePartChunkIter chunks = native_part_chunks(size, v->ptr_bytes);
   u32 base;
   i32 off;
   u32 chunk_off, chunk_size;
 
   if (!is_load) {
     /* Preserve the packed source before address resolution, which may borrow
      * TMP0/TMP1. Avoid any explicit address components as well: a plain
      * BASE_REG address must remain intact until it has been resolved. */
     u32 value = rv_part_preserved_tmp(a, &addr);
     u32 consumed = 0u;
     if (value != reg) rv64_emit32(mc, rv_addi(value, reg, 0));
     rv_resolve_mem_addr(a, &addr, &base, &off);
     if (!fits_i12((i64)off + (i64)size - 1)) {
       u32 stable = rv_part_tmp(a, value, base, REG_NONE);
       rv_emit_addr_adjust(v, mc, stable, base, off);
       base = stable;
       off = 0;
     }
     while (native_part_chunk_next(&chunks, &chunk_off, &chunk_size)) {
       if (chunk_off != consumed) {
         rv64_emit32(mc, rv_srli(value, value,
                                 (chunk_off - consumed) * 8u));
         consumed = chunk_off;
       }
       rv64_emit32(mc,
                   enc_int_store(v, chunk_size, value, base,
                                 off + (i32)chunk_off));
     }
     return;
   }
 
   rv_resolve_mem_addr(a, &addr, &base, &off);
   if (base == reg || !fits_i12((i64)off + (i64)size - 1)) {
     u32 stable = rv_part_tmp(a, reg, base, REG_NONE);
     rv_emit_addr_adjust(v, mc, stable, base, off);
     base = stable;
     off = 0;
   }
   {
     u32 piece = rv_part_tmp(a, reg, base, REG_NONE);
     int first = 1;
     while (native_part_chunk_next(&chunks, &chunk_off, &chunk_size)) {
       u32 dst = first ? reg : piece;
       rv64_emit32(mc, enc_int_load(v, chunk_size, 0, dst, base,
                                    off + (i32)chunk_off));
       if (!first) {
         rv64_emit32(mc, rv_slli(piece, piece, chunk_off * 8u));
         rv64_emit32(mc, rv_or(reg, reg, piece));
       }
       first = 0;
     }
   }
 }
 
 /* Central load/store primitive. is_load: 1 load into reg, 0 store reg to mem.
  */
 static void rv_emit_mem(RvNativeTarget* a, int is_load, NativeLoc reg,
                         NativeAddr addr, MemAccess mem) {
   NativeTarget* t = &a->base;
   MCEmitter* mc = t->mc;
   u32 r = loc_reg(reg);
   int fp = native_loc_is_fp(reg);
   u32 sz = mem.size ? mem.size : loc_size32(t, reg);
   u32 base;
   i32 off;
 
   if (!fp && sz != 0u && sz <= a->variant->ptr_bytes &&
       (sz & (sz - 1u)) != 0u) {
     rv_emit_exact_int_part(a, is_load, r, addr, sz);
     return;
   }
 
   rv_resolve_mem_addr(a, &addr, &base, &off);
   if (fp) {
     rv64_emit32(
         mc, is_load ? (sz == 8u ? rv_fld(r, base, off) : rv_flw(r, base, off))
                     : (sz == 8u ? rv_fsd(r, base, off) : rv_fsw(r, base, off)));
   } else {
     rv64_emit32(mc, is_load ? enc_int_load(a->variant, sz, 0, r, base, off)
                             : enc_int_store(a->variant, sz, r, base, off));
   }
 }
 
 /* ============================ moves / data ============================ */
 
 static void rv_move(NativeTarget* t, NativeLoc dst, NativeLoc src) {
   MCEmitter* mc = t->mc;
   int dfp = native_loc_is_fp(dst), sfp = native_loc_is_fp(src);
   u32 rd = loc_reg(dst), rs = loc_reg(src);
   if (dfp && sfp) {
     u32 fmt = loc_size32(t, dst) == 8u ? RV_FMT_D : RV_FMT_S;
     if (rd == rs) return;
     rv64_emit32(mc, rv_fsgnj(fmt, rd, rs, rs));
     return;
   }
   if (!dfp && sfp) {
     u32 sz = loc_size32(t, src);
     rv64_emit32(mc, sz == 8u ? rv_fmv_x_d(rd, rs) : rv_fmv_x_w(rd, rs));
     return;
   }
   if (dfp && !sfp) {
     u32 sz = loc_size32(t, dst);
     rv64_emit32(mc, sz == 8u ? rv_fmv_d_x(rd, rs) : rv_fmv_w_x(rd, rs));
     return;
   }
   if (rd == rs) return;
   rv64_emit32(mc, rv_addi(rd, rs, 0));
 }
 
 static void rv_load_imm(NativeTarget* t, NativeLoc dst, i64 imm) {
   rv_emit_load_imm(rv_of(t)->variant, t->mc, loc_is_64(t, dst) ? 1u : 0u,
                    loc_reg(dst), imm);
 }
 
 static void rv_load_const(NativeTarget* t, NativeLoc dst, ConstBytes cb) {
   RvNativeTarget* a = rv_of(t);
   u64 v = 0;
   u32 i;
   if (!native_loc_is_fp(dst)) {
     for (i = 0; i < cb.size && i < 8u; ++i) v |= (u64)cb.bytes[i] << (i * 8u);
     rv_load_imm(t, dst, (i64)v);
     return;
   }
   /* FP constant: materialize the bit pattern in TMP0, bitcast into the FPR. */
   for (i = 0; i < cb.size && i < 8u; ++i) v |= (u64)cb.bytes[i] << (i * 8u);
   rv_emit_load_imm(a->variant, t->mc, 1, RV_TMP0, (i64)v);
   if (cb.size == 8u)
     rv64_emit32(t->mc, rv_fmv_d_x(loc_reg(dst), RV_TMP0));
   else
     rv64_emit32(t->mc, rv_fmv_w_x(loc_reg(dst), RV_TMP0));
   (void)a;
 }
 
 static void rv_load_addr(NativeTarget* t, NativeLoc dst, NativeAddr addr) {
   RvNativeTarget* a = rv_of(t);
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst);
   u32 base;
   i32 off;
   if (addr.base_kind == NATIVE_ADDR_BASE_GLOBAL) {
     rv_emit_global_addr(a, rd, addr.base.global.sym,
                         addr.base.global.addend + addr.offset);
     base = rd;
     off = 0;
   } else if (addr.base_kind == NATIVE_ADDR_BASE_FRAME_VALUE) {
     /* Load the pointer stored in the frame slot, then add the offset. */
     rv_load_frame_component(a, rd, addr.base.frame, addr.base_type, 0u);
     base = rd;
     off = addr.offset;
   } else if (addr.base_kind == NATIVE_ADDR_BASE_FRAME) {
     RvNativeSlot* s = rv_slot_get(a, addr.base.frame);
     base = RV_S0;
     off = rv_s0_off_slot(s) + addr.offset;
   } else if (addr.base_kind == NATIVE_ADDR_BASE_REG) {
     base = addr.base.reg & 0x1fu;
     off = addr.offset;
   } else {
     rv_panic(a, "unsupported address base in load_addr");
   }
   /* Fold any index via Zba sh{1,2,3}add (index << scale) + base. Resolve the
    * byte offset before loading a frame index so the far-offset helper's
    * scratch cannot overwrite that index. */
   if (addr.index_kind == NATIVE_ADDR_INDEX_REG ||
       addr.index_kind == NATIVE_ADDR_INDEX_FRAME_VALUE) {
     u32 idx;
     if (off != 0 || base != rd)
       rv_emit_addr_adjust(a->variant, mc, rd, base, off);
     if (addr.index_kind == NATIVE_ADDR_INDEX_REG) {
       idx = addr.index.reg & 0x1fu;
     } else {
       idx = rv_internal_tmp(a, 1u << rd);
       rv_load_frame_component(a, idx, addr.index.frame, addr.index_type,
                               1u << rd);
     }
     switch (addr.log2_scale) {
       case 0:
         rv64_emit32(mc, rv_add(rd, rd, idx));
         break;
       case 1:
         rv64_emit32(mc, rv_sh1add(rd, idx, rd));
         break;
       case 2:
         rv64_emit32(mc, rv_sh2add(rd, idx, rd));
         break;
       default:
         rv64_emit32(mc, rv_sh3add(rd, idx, rd));
         break;
     }
     return;
   }
   rv_emit_addr_adjust(a->variant, mc, rd, base, off);
 }
 
 static void rv_load(NativeTarget* t, NativeLoc dst, NativeAddr addr,
                     MemAccess mem) {
   rv_emit_mem(rv_of(t), 1, dst, addr, mem);
 }
 static void rv_store(NativeTarget* t, NativeAddr addr, NativeLoc src,
                      MemAccess mem) {
   rv_emit_mem(rv_of(t), 0, src, addr, mem);
 }
 
 /* copy_bytes: resolve dst and src to dedicated pointer regs (RV_TMP3 / RV_TMP0)
  * once, then copy granule-by-granule advancing both pointers. dst is resolved
  * first because its base may itself live in RV_TMP1 (the transfer reg, e.g. the
  * sret pointer from marshal_ret); capturing it into RV_TMP3 before src resolution
  * (which may clobber RV_TMP1 for far offsets) keeps it live. Advancing the
  * pointers keeps every load/store at offset 0, so no offset ever exceeds imm12
  * and the transfer reg never aliases a base. */
 static void rv_copy_bytes(NativeTarget* t, NativeAddr dst, NativeAddr src,
                           AggregateAccess access) {
   RvNativeTarget* a = rv_of(t);
   const RiscvVariant* v = a->variant;
   MCEmitter* mc = t->mc;
   KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
   u32 rem = access.size;
   u32 maxg = v->ptr_bytes; /* widest granule: 8 on rv64, 4 on rv32 */
   rv_load_addr(t, native_loc_reg(i64t, NATIVE_REG_INT, RV_TMP3), dst);
   rv_load_addr(t, native_loc_reg(i64t, NATIVE_REG_INT, RV_TMP0), src);
   while (rem) {
     u32 sz = rem >= 8u && maxg >= 8u ? 8u
              : rem >= 4u             ? 4u
              : rem >= 2u             ? 2u
                                      : 1u;
     rv64_emit32(mc, enc_int_load(v, sz, 0, RV_TMP1, RV_TMP0, 0));
     rv64_emit32(mc, enc_int_store(v, sz, RV_TMP1, RV_TMP3, 0));
     rv64_emit32(mc, rv_addi(RV_TMP0, RV_TMP0, (i32)sz));
     rv64_emit32(mc, rv_addi(RV_TMP3, RV_TMP3, (i32)sz));
     rem -= sz;
   }
 }
 
 static void rv_set_bytes(NativeTarget* t, NativeAddr dst, NativeLoc byte_value,
                          AggregateAccess access) {
   MCEmitter* mc = t->mc;
   KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
   u32 bv = loc_reg(byte_value);
   u32 rem = access.size;
   rv_load_addr(t, native_loc_reg(i64t, NATIVE_REG_INT, RV_TMP3), dst);
   while (rem) {
     rv64_emit32(mc, rv_sb(bv, RV_TMP3, 0));
     rv64_emit32(mc, rv_addi(RV_TMP3, RV_TMP3, 1));
     rem -= 1u;
   }
 }
 
 /* ============================ arithmetic ============================ */
 
 static void rv_binop(NativeTarget* t, BinOp op, NativeLoc dst, NativeLoc aop,
                      NativeLoc bop) {
   const RiscvVariant* v = rv_of(t)->variant;
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst);
   u32 ra = loc_reg(aop);
   int sf = loc_is_64(t, dst);
   /* The W-form ops (ADDW/SUBW/MULW/SLLW/...) are RV64-only and act on a 32-bit
    * value held in a 64-bit register. They are emitted only for a narrow value
    * on rv64; on rv32 the BASE ops ARE the 32-bit ops, so `w` is always 0 and we
    * fall to the base ops. */
   int w = !sf && v->has_w_forms;
   /* Immediate shamt mask: 5-bit (&31) for a W-form / rv32 op, else shamt_bits
    * (6-bit &63 on rv64) for the native-width op. */
   u32 shmask = w ? 31u : ((1u << v->shamt_bits) - 1u);
   int b_imm = bop.kind == NATIVE_LOC_IMM;
   u32 rb = b_imm ? 0u : loc_reg(bop);
   i64 imm = b_imm ? bop.v.imm : 0;
 
   switch (op) {
     case BO_FADD:
     case BO_FSUB:
     case BO_FMUL:
     case BO_FDIV: {
       u32 fmt = loc_size32(t, dst) == 8u ? RV_FMT_D : RV_FMT_S;
       switch (op) {
         case BO_FADD:
           rv64_emit32(mc, rv_fadd(fmt, rd, ra, rb));
           break;
         case BO_FSUB:
           rv64_emit32(mc, rv_fsub(fmt, rd, ra, rb));
           break;
         case BO_FMUL:
           rv64_emit32(mc, rv_fmul(fmt, rd, ra, rb));
           break;
         default:
           rv64_emit32(mc, rv_fdiv(fmt, rd, ra, rb));
           break;
       }
       return;
     }
     case BO_IADD:
       if (b_imm) {
         rv64_emit32(mc,
                     w ? rv_addiw(rd, ra, (i32)imm) : rv_addi(rd, ra, (i32)imm));
       } else {
         rv64_emit32(mc, w ? rv_addw(rd, ra, rb) : rv_add(rd, ra, rb));
       }
       return;
     case BO_ISUB:
       if (b_imm) {
         rv64_emit32(
             mc, w ? rv_addiw(rd, ra, (i32)-imm) : rv_addi(rd, ra, (i32)-imm));
       } else {
         rv64_emit32(mc, w ? rv_subw(rd, ra, rb) : rv_sub(rd, ra, rb));
       }
       return;
     case BO_IMUL:
       rv64_emit32(mc, w ? rv_mulw(rd, ra, rb) : rv_mul(rd, ra, rb));
       return;
     case BO_SDIV:
       rv64_emit32(mc, w ? rv_divw(rd, ra, rb) : rv_div(rd, ra, rb));
       return;
     case BO_UDIV:
       rv64_emit32(mc, w ? rv_divuw(rd, ra, rb) : rv_divu(rd, ra, rb));
       return;
     case BO_SREM:
       rv64_emit32(mc, w ? rv_remw(rd, ra, rb) : rv_rem(rd, ra, rb));
       return;
     case BO_UREM:
       rv64_emit32(mc, w ? rv_remuw(rd, ra, rb) : rv_remu(rd, ra, rb));
       return;
     case BO_AND:
       rv64_emit32(mc, b_imm ? rv_andi(rd, ra, (i32)imm) : rv_and(rd, ra, rb));
       return;
     case BO_OR:
       rv64_emit32(mc, b_imm ? rv_ori(rd, ra, (i32)imm) : rv_or(rd, ra, rb));
       return;
     case BO_XOR:
       rv64_emit32(mc, b_imm ? rv_xori(rd, ra, (i32)imm) : rv_xor(rd, ra, rb));
       return;
     case BO_SHL:
       if (b_imm)
         rv64_emit32(mc, w ? rv_slliw(rd, ra, (u32)imm & shmask)
                           : rv_slli(rd, ra, (u32)imm & shmask));
       else
         rv64_emit32(mc, w ? rv_sllw(rd, ra, rb) : rv_sll(rd, ra, rb));
       return;
     case BO_SHR_U:
       if (b_imm)
         rv64_emit32(mc, w ? rv_srliw(rd, ra, (u32)imm & shmask)
                           : rv_srli(rd, ra, (u32)imm & shmask));
       else
         rv64_emit32(mc, w ? rv_srlw(rd, ra, rb) : rv_srl(rd, ra, rb));
       return;
     case BO_SHR_S:
       if (b_imm)
         rv64_emit32(mc, w ? rv_sraiw(rd, ra, (u32)imm & shmask)
                           : rv_srai(rd, ra, (u32)imm & shmask));
       else
         rv64_emit32(mc, w ? rv_sraw(rd, ra, rb) : rv_sra(rd, ra, rb));
       return;
     default:
       rv_panic(rv_of(t), "unsupported binop");
   }
 }
 
 static void rv_unop(NativeTarget* t, UnOp op, NativeLoc dst, NativeLoc src) {
   const RiscvVariant* v = rv_of(t)->variant;
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst), rs = loc_reg(src);
   int sf = loc_is_64(t, dst);
   int w = !sf && v->has_w_forms; /* SUBW is RV64-only; base SUB on rv32 */
   switch (op) {
     case UO_NEG:
       rv64_emit32(mc, w ? rv_subw(rd, RV_ZERO, rs) : rv_sub(rd, RV_ZERO, rs));
       return;
     case UO_FNEG: {
       u32 fmt = loc_size32(t, dst) == 8u ? RV_FMT_D : RV_FMT_S;
       rv64_emit32(mc, rv_fsgnjn(fmt, rd, rs, rs));
       return;
     }
     case UO_BNOT:
       rv64_emit32(mc, rv_xori(rd, rs, -1));
       return;
     case UO_NOT:
       rv64_emit32(mc, rv_sltiu(rd, rs, 1));
       return;
     default:
       rv_panic(rv_of(t), "unsupported unop");
   }
 }
 
 /* Sign/zero-extend a 32-bit operand into a 64-bit register for comparison.
  * Returns the register to compare. */
 static u32 rv_cmp_ext(NativeTarget* t, int is_signed, NativeLoc op, u32 tmp) {
   const RiscvVariant* v = rv_of(t)->variant;
   MCEmitter* mc = t->mc;
   u32 r = loc_reg(op);
   /* On rv32 a 32-bit operand already fills the whole register — there is no
    * wider container to canonicalize into, so the extension is a no-op. */
   if (v->xlen == 32u) return r;
   if (loc_is_64(t, op)) return r;
   if (is_signed) {
     rv64_emit32(mc, rv_addiw(tmp, r, 0)); /* sign-extend low 32 */
   } else {
     rv64_emit32(mc, rv_slli(tmp, r, 32));
     rv64_emit32(mc, rv_srli(tmp, tmp, 32));
   }
   return tmp;
 }
 
 static int cmp_is_signed(CmpOp op) {
   switch (op) {
     case CMP_LT_U:
     case CMP_LE_U:
     case CMP_GT_U:
     case CMP_GE_U:
       return 0;
     default:
       return 1;
   }
 }
 
 /* Emit a 0/1 comparison result into rd from two integer registers. */
 static void rv_emit_icmp(NativeTarget* t, CmpOp op, u32 rd, u32 ra, u32 rb) {
   MCEmitter* mc = t->mc;
   switch (op) {
     case CMP_EQ:
       rv64_emit32(mc, rv_sub(rd, ra, rb));
       rv64_emit32(mc, rv_sltiu(rd, rd, 1));
       return;
     case CMP_NE:
       rv64_emit32(mc, rv_sub(rd, ra, rb));
       rv64_emit32(mc, rv_sltu(rd, RV_ZERO, rd));
       return;
     case CMP_LT_S:
       rv64_emit32(mc, rv_slt(rd, ra, rb));
       return;
     case CMP_LT_U:
       rv64_emit32(mc, rv_sltu(rd, ra, rb));
       return;
     case CMP_GT_S:
       rv64_emit32(mc, rv_slt(rd, rb, ra));
       return;
     case CMP_GT_U:
       rv64_emit32(mc, rv_sltu(rd, rb, ra));
       return;
     case CMP_GE_S:
       rv64_emit32(mc, rv_slt(rd, ra, rb));
       rv64_emit32(mc, rv_xori(rd, rd, 1));
       return;
     case CMP_GE_U:
       rv64_emit32(mc, rv_sltu(rd, ra, rb));
       rv64_emit32(mc, rv_xori(rd, rd, 1));
       return;
     case CMP_LE_S:
       rv64_emit32(mc, rv_slt(rd, rb, ra));
       rv64_emit32(mc, rv_xori(rd, rd, 1));
       return;
     case CMP_LE_U:
       rv64_emit32(mc, rv_sltu(rd, rb, ra));
       rv64_emit32(mc, rv_xori(rd, rd, 1));
       return;
     default:
       rv_panic(rv_of(t), "unsupported integer cmp");
   }
 }
 
 /* Format-dispatching wrappers over the ordered FP compares (feq/flt/fle are
  * ordered: they yield 0 on NaN; flt/fle are signaling, raising NV on NaN —
  * pre-existing for ordered ops, and the boolean result is still correct). */
 static u32 rv_feq_fmt(u32 fmt, u32 rd, u32 ra, u32 rb) {
   return fmt == RV_FMT_D ? rv_feq_d(rd, ra, rb) : rv_feq_s(rd, ra, rb);
 }
 static u32 rv_flt_fmt(u32 fmt, u32 rd, u32 ra, u32 rb) {
   return fmt == RV_FMT_D ? rv_flt_d(rd, ra, rb) : rv_flt_s(rd, ra, rb);
 }
 static u32 rv_fle_fmt(u32 fmt, u32 rd, u32 ra, u32 rb) {
   return fmt == RV_FMT_D ? rv_fle_d(rd, ra, rb) : rv_fle_s(rd, ra, rb);
 }
 
 static void rv_cmp(NativeTarget* t, CmpOp op, NativeLoc dst, NativeLoc aop,
                    NativeLoc bop) {
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst);
   /* FP-ness is self-describing from the opcode (FP block starts at CMP_OEQ_F).
    * Unordered predicates use unordered-R == NOT(ordered-not-R): the ordered
    * compare into rd, then `xori rd,rd,1`. ONE/UEQ have no single ordered
    * primitive and OR the two strict relations (a<b | a>b) via scratch RV_TMP2
    * (x7, reserved & never allocable, so it can't alias rd). */
   if (op >= CMP_OEQ_F) {
     u32 fmt = loc_size32(t, aop) == 8u ? RV_FMT_D : RV_FMT_S;
     u32 ra = loc_reg(aop), rb = loc_reg(bop);
     switch (op) {
       case CMP_OEQ_F:
         rv64_emit32(mc, rv_feq_fmt(fmt, rd, ra, rb));
         return;
       case CMP_UNE_F: /* !(OEQ) */
         rv64_emit32(mc, rv_feq_fmt(fmt, rd, ra, rb));
         rv64_emit32(mc, rv_xori(rd, rd, 1));
         return;
       case CMP_OLT_F:
         rv64_emit32(mc, rv_flt_fmt(fmt, rd, ra, rb));
         return;
       case CMP_OLE_F:
         rv64_emit32(mc, rv_fle_fmt(fmt, rd, ra, rb));
         return;
       case CMP_OGT_F:
         rv64_emit32(mc, rv_flt_fmt(fmt, rd, rb, ra));
         return;
       case CMP_OGE_F:
         rv64_emit32(mc, rv_fle_fmt(fmt, rd, rb, ra));
         return;
       case CMP_UGE_F: /* !(OLT) */
         rv64_emit32(mc, rv_flt_fmt(fmt, rd, ra, rb));
         rv64_emit32(mc, rv_xori(rd, rd, 1));
         return;
       case CMP_UGT_F: /* !(OLE) */
         rv64_emit32(mc, rv_fle_fmt(fmt, rd, ra, rb));
         rv64_emit32(mc, rv_xori(rd, rd, 1));
         return;
       case CMP_ULE_F: /* !(OGT) */
         rv64_emit32(mc, rv_flt_fmt(fmt, rd, rb, ra));
         rv64_emit32(mc, rv_xori(rd, rd, 1));
         return;
       case CMP_ULT_F: /* !(OGE) */
         rv64_emit32(mc, rv_fle_fmt(fmt, rd, rb, ra));
         rv64_emit32(mc, rv_xori(rd, rd, 1));
         return;
       case CMP_ONE_F: /* ordered & !=: (a<b) | (a>b) */
         rv64_emit32(mc, rv_flt_fmt(fmt, rd, ra, rb));
         rv64_emit32(mc, rv_flt_fmt(fmt, RV_TMP2, rb, ra));
         rv64_emit32(mc, rv_or(rd, rd, RV_TMP2));
         return;
       case CMP_UEQ_F: /* unordered | ==: !((a<b) | (a>b)) */
         rv64_emit32(mc, rv_flt_fmt(fmt, rd, ra, rb));
         rv64_emit32(mc, rv_flt_fmt(fmt, RV_TMP2, rb, ra));
         rv64_emit32(mc, rv_or(rd, rd, RV_TMP2));
         rv64_emit32(mc, rv_xori(rd, rd, 1));
         return;
       default:
         rv_panic(rv_of(t), "unsupported fp cmp");
     }
   }
   {
     int sg = cmp_is_signed(op);
     u32 ra = rv_cmp_ext(t, sg, aop, RV_TMP0);
     u32 rb = rv_cmp_ext(t, sg, bop, RV_TMP1);
     rv_emit_icmp(t, op, rd, ra, rb);
   }
 }
 
 static void rv_convert(NativeTarget* t, ConvKind op, NativeLoc dst,
                        NativeLoc src) {
   const RiscvVariant* v = rv_of(t)->variant;
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst), rs = loc_reg(src);
   u32 src_sz = loc_size32(t, src);
   u32 dst_sz = loc_size32(t, dst);
   /* `il` (int-side wide): the 64-bit-integer fcvt L-forms are RV64-only; on
    * rv32 only the w/wu forms exist and a 64-bit int<->fp is legalized to a
    * libcall before reaching here. */
   int il = v->has_w_forms;
   switch (op) {
     case CV_SEXT:
       if (src_sz >= 4u) {
         /* ADDIW sign-extends bits[31:0] into a 64-bit reg (RV64). On rv32 a
          * 4-byte value already spans the whole register, so a plain move (or
          * nothing when rd==rs) is the sign extension. */
         if (v->has_w_forms)
           rv64_emit32(mc, rv_addiw(rd, rs, 0));
         else if (rd != rs)
           rv64_emit32(mc, rv_addi(rd, rs, 0));
       } else {
         u32 sh = v->xlen - src_sz * 8u;
         rv64_emit32(mc, rv_slli(rd, rs, sh));
         rv64_emit32(mc, rv_srai(rd, rd, sh));
       }
       return;
     case CV_ZEXT: {
       u32 sh = v->xlen - src_sz * 8u;
       rv64_emit32(mc, rv_slli(rd, rs, sh));
       rv64_emit32(mc, rv_srli(rd, rd, sh));
       return;
     }
     case CV_TRUNC:
       if (rd != rs || dst_sz <= 4u)
         rv64_emit32(mc, rv_addi(rd, rs, 0)); /* low bits; users re-narrow */
       return;
     case CV_ITOF_S:
       if (dst_sz == 8u)
         rv64_emit32(
             mc, il && src_sz == 8u ? rv_fcvt_d_l(rd, rs) : rv_fcvt_d_w(rd, rs));
       else
         rv64_emit32(
             mc, il && src_sz == 8u ? rv_fcvt_s_l(rd, rs) : rv_fcvt_s_w(rd, rs));
       return;
     case CV_ITOF_U:
       if (dst_sz == 8u)
         rv64_emit32(mc, il && src_sz == 8u ? rv_fcvt_d_lu(rd, rs)
                                            : rv_fcvt_d_wu(rd, rs));
       else
         rv64_emit32(mc, il && src_sz == 8u ? rv_fcvt_s_lu(rd, rs)
                                            : rv_fcvt_s_wu(rd, rs));
       return;
     case CV_FTOI_S:
       if (src_sz == 8u)
         rv64_emit32(
             mc, il && dst_sz == 8u ? rv_fcvt_l_d(rd, rs) : rv_fcvt_w_d(rd, rs));
       else
         rv64_emit32(
             mc, il && dst_sz == 8u ? rv_fcvt_l_s(rd, rs) : rv_fcvt_w_s(rd, rs));
       return;
     case CV_FTOI_U:
       if (src_sz == 8u)
         rv64_emit32(mc, il && dst_sz == 8u ? rv_fcvt_lu_d(rd, rs)
                                            : rv_fcvt_wu_d(rd, rs));
       else
         rv64_emit32(mc, il && dst_sz == 8u ? rv_fcvt_lu_s(rd, rs)
                                            : rv_fcvt_wu_s(rd, rs));
       return;
     case CV_FEXT:
       rv64_emit32(mc, rv_fcvt_d_s(rd, rs));
       return;
     case CV_FTRUNC:
       rv64_emit32(mc, rv_fcvt_s_d(rd, rs));
       return;
     case CV_BITCAST:
       rv_move(t, dst, src);
       return;
     default:
       rv_panic(rv_of(t), "unsupported convert");
   }
 }
 
 /* §E.3 narrow register-only entry points. The NDT crosses 16 B NativeRegLoc
  * here; these reconstruct the NativeLoc the fat hook expects and delegate, so
  * the emitted bytes are identical. Used only on the -O0 NDT path; the opt
  * replay path calls rv_binop/rv_move/rv_cmp/rv_convert directly. */
 static void rv_binop_rr(NativeTarget* t, BinOp op, NativeRegLoc dst,
                         NativeRegLoc a, NativeRegLoc b) {
   rv_binop(t, op, native_loc_from_reg(dst), native_loc_from_reg(a),
            native_loc_from_reg(b));
 }
 
 static void rv_move_rr(NativeTarget* t, NativeRegLoc dst, NativeRegLoc src) {
   rv_move(t, native_loc_from_reg(dst), native_loc_from_reg(src));
 }
 
 static void rv_cmp_rr(NativeTarget* t, CmpOp op, NativeRegLoc dst,
                       NativeRegLoc a, NativeRegLoc b) {
   rv_cmp(t, op, native_loc_from_reg(dst), native_loc_from_reg(a),
          native_loc_from_reg(b));
 }
 
 static void rv_convert_rr(NativeTarget* t, ConvKind op, NativeRegLoc dst,
                           NativeRegLoc src) {
   rv_convert(t, op, native_loc_from_reg(dst), native_loc_from_reg(src));
 }
 
 /* ============================ spill / reload ============================ */
 
 static void rv_spill(NativeTarget* t, NativeLoc src, NativeFrameSlot slot,
                      MemAccess mem) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = slot;
   addr.base_type = src.type;
   rv_emit_mem(rv_of(t), 0, src, addr, mem);
 }
 static void rv_reload(NativeTarget* t, NativeLoc dst, NativeFrameSlot slot,
                       MemAccess mem) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = slot;
   addr.base_type = dst.type;
   rv_emit_mem(rv_of(t), 1, dst, addr, mem);
 }
 
 /* ============================ control flow ============================ */
 
 static MCLabel rv_label_new(NativeTarget* t) { return mc_label_new(t->mc); }
 static void rv_label_place(NativeTarget* t, MCLabel l) {
   mc_label_place(t->mc, l);
 }
 static void rv_jump(NativeTarget* t, MCLabel l) {
   rv64_emit32(t->mc, rv_jal(RV_ZERO, 0));
   mc_emit_label_ref(t->mc, l, R_RV_JAL, 4, 0);
 }
 
 static void rv_cmp_branch(NativeTarget* t, CmpOp op, NativeLoc aop,
                           NativeLoc bop, MCLabel l) {
   MCEmitter* mc = t->mc;
   /* RISC-V B-type branches reach only ±4 KiB, which a single (especially
    * -O0) function can exceed between a branch and its target. Rather than a
    * lone conditional branch to the label, emit a short *inverted* branch
    * that skips an unconditional `jal` (±1 MiB) to the target. The inverted
    * branch's displacement is the constant SKIP_JAL (skip just the jal) and
    * so is always in range; the jal carries the long reach. See rv_jump. */
   enum { SKIP_JAL = 8 }; /* branch over the 4-byte jal that follows it */
   /* FP compares have no register-register branch form: materialize the 0/1
    * into TMP0 via rv_cmp (handles all 12 predicates), then branch on nonzero.
    */
   if (op >= CMP_OEQ_F) {
     NativeLoc tmp =
         native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT, RV_TMP0);
     rv_cmp(t, op, tmp, aop, bop);
     /* Skip the jal when the result is 0 (condition false). */
     rv64_emit32(mc, rv_beq(RV_TMP0, RV_ZERO, SKIP_JAL));
     rv_jump(t, l);
     return;
   }
   {
     int sg = cmp_is_signed(op);
     u32 ra = rv_cmp_ext(t, sg, aop, RV_TMP0);
     u32 rb = rv_cmp_ext(t, sg, bop, RV_TMP1);
     u32 word;
     /* Encode the *inverse* of `op`, skipping the jal when NOT taken. */
     switch (op) {
       case CMP_EQ:
         word = rv_bne(ra, rb, SKIP_JAL);
         break;
       case CMP_NE:
         word = rv_beq(ra, rb, SKIP_JAL);
         break;
       case CMP_LT_S:
         word = rv_bge(ra, rb, SKIP_JAL);
         break;
       case CMP_GE_S:
         word = rv_blt(ra, rb, SKIP_JAL);
         break;
       case CMP_LT_U:
         word = rv_bgeu(ra, rb, SKIP_JAL);
         break;
       case CMP_GE_U:
         word = rv_bltu(ra, rb, SKIP_JAL);
         break;
       case CMP_GT_S:
         word = rv_bge(rb, ra, SKIP_JAL);
         break;
       case CMP_LE_S:
         word = rv_blt(rb, ra, SKIP_JAL);
         break;
       case CMP_GT_U:
         word = rv_bgeu(rb, ra, SKIP_JAL);
         break;
       case CMP_LE_U:
         word = rv_bltu(rb, ra, SKIP_JAL);
         break;
       default:
         rv_panic(rv_of(t), "unsupported cmp_branch");
     }
     rv64_emit32(mc, word);
     rv_jump(t, l);
   }
 }
 
 static void rv_indirect_branch(NativeTarget* t, NativeLoc addr,
                                const MCLabel* valid_targets, u32 ntargets) {
   (void)valid_targets;
   (void)ntargets;
   rv64_emit32(t->mc, rv_jalr(RV_ZERO, loc_reg(addr), 0));
 }
 
 static void rv_load_label_addr(NativeTarget* t, NativeLoc dst, MCLabel l) {
   /* `&&label` address-take: auipc/addi with a %pcrel_hi/%pcrel_lo relocation
    * pair against the label's per-block local symbol — the same form
    * rv_emit_global_addr uses for a global — so a compressing/re-encoding
    * assembler recomputes the displacement (a baked offset would break under
    * the C extension). */
   u32 rd = loc_reg(dst);
   ObjSymId sym = mc_label_symbol(t->mc, l);
   rv_emit_pcrel_anchor(t, rd, sym, R_RV_PCREL_HI20, rv_addi(rd, rd, 0));
 }
 
 /* ============================ frame / lifecycle ============================
  */
 
 static NativeFrameSlot rv_frame_slot(NativeTarget* t,
                                      const NativeFrameSlotDesc* d) {
   return native_frame_slot_alloc(&rv_of(t)->frame, d);
 }
 
 static void rv_release_frame_slot(NativeTarget* t, NativeFrameSlot slot) {
   native_frame_release_slot(&rv_of(t)->frame, slot);
 }
 
 static int rv_frame_slot_debug_loc(NativeTarget* t, NativeFrameSlot slot,
                                    CGDebugLoc* out) {
   RvNativeTarget* a = rv_of(t);
   RvNativeSlot* s;
   if (!out) return 0;
   memset(out, 0, sizeof *out);
   if (slot == NATIVE_FRAME_SLOT_NONE || slot > a->frame.nslots) return 0;
   s = rv_slot_get(a, slot);
   out->kind = CG_DEBUG_LOC_FRAME;
   /* rv64 slots are addressed s0/fp-relative (rv_s0_off_slot); the hosted dbg
    * snapshot seeds the frame base with s0, matching aa64's FP-relative
    * convention. */
   out->v.frame_ofs = rv_s0_off_slot(s);
   return 1;
 }
 
 static void rv_func_begin_common(NativeTarget* t, const CGFuncDesc* fd) {
   RvNativeTarget* a = rv_of(t);
   MCEmitter* mc = t->mc;
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fd->fn_type);
   a->func = fd;
   a->loc = fd->loc;
   /* Shared frame bookkeeping: clears the slot table, cum_off, max_outgoing,
    * callee-save set, and known_frame/has_alloca/frame_final. */
   native_frame_reset(&a->frame);
   a->incoming_stack_size = 0;
   a->next_param_int = 0;
   a->next_param_fp = 0;
   a->next_param_stack = 0;
   a->has_sret = (abi && abi->has_sret) ? 1u : 0u;
   a->is_variadic = (abi && abi->variadic) ? 1u : 0u;
   a->sret_ptr_slot = NATIVE_FRAME_SLOT_NONE;
   a->npatches = 0;
   a->nalloca = 0;
   a->minimal_prologue_words = 0;
   a->slim_prologue = 0;
 
   mc_set_section(mc, fd->text_section_id);
   mc_emit_align(mc, 4, 0);
   a->func_start = mc_pos(mc);
   mc_begin_function(mc, fd->sym, fd->text_section_id, a->func_start);
   mc_cfi_startproc(mc);
   a->epilogue_label = mc_label_new(mc);
 }
 
 /* sret: reserve a hidden slot for the incoming destination pointer (a0). */
 static void rv_reserve_entry_saves(RvNativeTarget* a) {
   NativeTarget* t = &a->base;
   if (a->has_sret) {
     NativeFrameSlotDesc sd;
     u32 ptr = a->variant->ptr_bytes;
     memset(&sd, 0, sizeof sd);
     sd.type = builtin_id(KIT_CG_BUILTIN_I64);
     sd.size = ptr; /* a pointer slot: 8 on rv64, 4 on rv32 */
     sd.align = ptr;
     sd.kind = NATIVE_FRAME_SLOT_SAVE;
     a->sret_ptr_slot = t->frame_slot(t, &sd);
     a->next_param_int = 1; /* a0 consumed by the sret pointer */
   }
 }
 
 /* Emit the s0-relative (frame-independent) entry stores live: the sret a0 spill
  * and, for variadics, the unconsumed-GP register save area. Both are anchored at
  * s0 (set by the fixed entry in rv_func_begin) and sit at non-negative offsets in
  * the saved-pair + variadic region the fixed entry already pre-decremented sp
  * for, so they run correctly after the deferred `sub sp`. Single-pass path only;
  * the known-frame path emits these inline in rv_build_prologue. */
 static void rv_emit_entry_save_stores(RvNativeTarget* a) {
   NativeTarget* t = &a->base;
   if (a->has_sret && a->sret_ptr_slot != NATIVE_FRAME_SLOT_NONE) {
     KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
     u32 ptr = a->variant->ptr_bytes;
     NativeAddr addr;
     memset(&addr, 0, sizeof addr);
     addr.base_kind = NATIVE_ADDR_BASE_FRAME;
     addr.base.frame = a->sret_ptr_slot;
     addr.base_type = i64t;
     rv_emit_mem(a, 0, native_loc_reg(i64t, NATIVE_REG_INT, RV_A0), addr,
                 native_mem_for_type(t, i64t, ptr));
   }
   /* variadic GP save area: home a0..a7 at [s0 + frame_save_size + i*gp_slot_bytes]
    * (positive, always within imm12). Emitted live in rv_func_begin, before the
    * params are bound, so next_param_int is not yet final — home ALL eight GP
    * registers (the named leading ones are homed too; va_start skips past them via
    * its next_param_int cursor, so the extra named-slot stores are dead-but-
    * harmless, mirroring aa64). The save-area offsets are byte-identical to the
    * historical rv_build_prologue layout. */
   if (a->is_variadic) {
     const RiscvVariant* v = a->variant;
     u32 fsz = v->frame_save_size;
     u32 gp_slot = v->gp_slot_bytes;
     u32 i;
     for (i = 0; i < 8u; ++i)
       rv64_emit32(t->mc, rv_sd_ptr(v, RV_A0 + i, RV_S0,
                                    (i32)fsz + (i32)i * (i32)gp_slot));
   }
 }
 
 /* Collect the callee-saves the body used (none at -O0). */
 static u32 rv_collect_int_saves(RvNativeTarget* a, u32* regs) {
   u32 n = 0, i;
   for (i = 0; i < a->frame.ncallee_saves; ++i)
     if (a->frame.callee_saves[i].cls == NATIVE_REG_INT)
       regs[n++] = a->frame.callee_saves[i].reg;
   return n;
 }
 static u32 rv_collect_fp_saves(RvNativeTarget* a, u32* regs) {
   u32 n = 0, i;
   for (i = 0; i < a->frame.ncallee_saves; ++i)
     if (a->frame.callee_saves[i].cls == NATIVE_REG_FP)
       regs[n++] = a->frame.callee_saves[i].reg;
   return n;
 }
 
 /* s0-relative offset of a saved register, below the locals. The flat index runs
  * 0..n_int-1 over integer saves (each ptr_bytes wide) then n_int..n_int+n_fp-1
  * over fp saves (each 8 bytes wide, fsd). On rv64 ptr_bytes==8 so this reduces
  * to the historical uniform -cum_off-8-8*idx layout, byte-for-byte. */
 static i32 rv_save_off(RvNativeTarget* a, u32 n_int, u32 idx) {
   i32 base = -(i32)(a->frame.cum_off);
   u32 ptr = a->variant->ptr_bytes;
   if (idx < n_int) return base - (i32)ptr * (i32)(idx + 1u);
   return base - (i32)(ptr * n_int) - 8 * (i32)(idx - n_int + 1u);
 }
 
 static void rv_load_s0(const RiscvVariant* v, MCEmitter* mc, int fp, u32 reg,
                        i32 off) {
   if (fits_i12(off)) {
     rv64_emit32(mc,
                 fp ? rv_fld(reg, RV_S0, off) : rv_ld_ptr(v, reg, RV_S0, off));
     return;
   }
   rv_emit_load_imm(v, mc, 1, RV_TMP0, (i64)off);
   rv64_emit32(mc, rv_add(RV_TMP0, RV_S0, RV_TMP0));
   rv64_emit32(mc, fp ? rv_fld(reg, RV_TMP0, 0) : rv_ld_ptr(v, reg, RV_TMP0, 0));
 }
 
 /* Build the prologue instruction sequence into words[]. Returns count. */
 static u32 rv_build_prologue(RvNativeTarget* a, u32* words, u32 cap,
                              u32 frame_size, u32 fp_pair_off,
                              const u32* int_regs, u32 n_int, const u32* fp_regs,
                              u32 n_fp) {
   const RiscvVariant* v = a->variant;
   u32 ptr = v->ptr_bytes;         /* saved-pair / int-save stride */
   u32 gp_slot = v->gp_slot_bytes; /* vararg GP-slot stride */
   u32 fsz = v->frame_save_size;   /* saved ra+s0 pair base offset */
   u32 wi = 0;
   /* lui+ADD{I,IW} materializes a 32-bit constant in TMP0; ADDIW is RV64-only so
    * use plain ADDI on rv32 (the value already fits 32 bits). */
 #define ADDI_LO(rd, lo) \
   (v->has_w_forms ? rv_addiw((rd), (rd), (lo)) : rv_addi((rd), (rd), (lo)))
 #define PUSH(w)                                                  \
   do {                                                           \
     if (wi >= cap) rv_panic(a, "prologue placeholder overflow"); \
     words[wi++] = (w);                                           \
   } while (0)
   /* sp -= frame_size */
   if (fits_i12(-(i32)frame_size)) {
     PUSH(rv_addi(RV_SP, RV_SP, -(i32)frame_size));
   } else {
     i32 neg = -(i32)frame_size;
     i32 hi = (i32)(((i64)neg + 0x800) >> 12);
     i32 lo = neg - (i32)((u32)hi << 12);
     PUSH(rv_lui(RV_TMP0, (u32)hi & 0xfffffu));
     if (lo) PUSH(ADDI_LO(RV_TMP0, lo));
     PUSH(rv_add(RV_SP, RV_SP, RV_TMP0));
   }
   /* save s0/ra at [sp + fp_pair_off], set s0 = sp + fp_pair_off. The saved-pair
    * internal stride is ptr_bytes (s0 at +0, ra at +ptr). */
   if (fits_i12((i32)fp_pair_off + (i32)ptr)) {
     PUSH(rv_sd_ptr(v, RV_S0, RV_SP, (i32)fp_pair_off));
     PUSH(rv_sd_ptr(v, RV_RA, RV_SP, (i32)fp_pair_off + (i32)ptr));
     PUSH(rv_addi(RV_S0, RV_SP, (i32)fp_pair_off));
   } else {
     i32 off = (i32)fp_pair_off;
     i32 hi = (i32)(((i64)off + 0x800) >> 12);
     i32 lo = off - (i32)((u32)hi << 12);
     PUSH(rv_lui(RV_TMP0, (u32)hi & 0xfffffu));
     if (lo) PUSH(ADDI_LO(RV_TMP0, lo));
     PUSH(rv_add(RV_TMP0, RV_SP, RV_TMP0));
     PUSH(rv_sd_ptr(v, RV_S0, RV_TMP0, 0));
     PUSH(rv_sd_ptr(v, RV_RA, RV_TMP0, (i32)ptr));
     PUSH(rv_addi(RV_S0, RV_TMP0, 0));
   }
   /* sret a0 spill */
   if (a->has_sret && a->sret_ptr_slot != NATIVE_FRAME_SLOT_NONE) {
     RvNativeSlot* s = rv_slot_get(a, a->sret_ptr_slot);
     PUSH(rv_sd_ptr(v, RV_A0, RV_S0, rv_s0_off_slot(s)));
   }
   /* variadic GP save area: spill unconsumed a-regs at
    * [s0 + frame_save_size + i*gp_slot_bytes] */
   if (a->is_variadic) {
     u32 i;
     for (i = a->next_param_int; i < 8u; ++i)
       PUSH(rv_sd_ptr(v, RV_A0 + i, RV_S0, (i32)fsz + (i32)i * (i32)gp_slot));
   }
   /* callee saves: integer with the pointer-width store (sw/sd), fp with fsd. */
   {
     u32 i;
     for (i = 0; i < n_int; ++i) {
       i32 off = rv_save_off(a, n_int, i);
       if (fits_i12(off)) {
         PUSH(rv_sd_ptr(v, int_regs[i], RV_S0, off));
       } else {
         /* rare; emitted directly is fine in the known-frame path, but the
          * single-pass placeholder must hold these too. Use the wide form. */
         i32 hi = (i32)(((i64)off + 0x800) >> 12);
         i32 lo = off - (i32)((u32)hi << 12);
         PUSH(rv_lui(RV_TMP0, (u32)hi & 0xfffffu));
         if (lo) PUSH(ADDI_LO(RV_TMP0, lo));
         PUSH(rv_add(RV_TMP0, RV_S0, RV_TMP0));
         PUSH(rv_sd_ptr(v, int_regs[i], RV_TMP0, 0));
       }
     }
     for (i = 0; i < n_fp; ++i) {
       i32 off = rv_save_off(a, n_int, n_int + i);
       if (fits_i12(off)) {
         PUSH(rv_fsd(fp_regs[i], RV_S0, off));
       } else {
         i32 hi = (i32)(((i64)off + 0x800) >> 12);
         i32 lo = off - (i32)((u32)hi << 12);
         PUSH(rv_lui(RV_TMP0, (u32)hi & 0xfffffu));
         if (lo) PUSH(ADDI_LO(RV_TMP0, lo));
         PUSH(rv_add(RV_TMP0, RV_S0, RV_TMP0));
         PUSH(rv_fsd(fp_regs[i], RV_TMP0, 0));
       }
     }
   }
 #undef PUSH
 #undef ADDI_LO
   return wi;
 }
 
 /* Build the deferred portion of the single-pass (-O0) prologue: the `sub sp`
  * that grows the frame below the saved s0/ra pair the live entry (rv_func_begin)
  * already saved and anchored s0 at. The fixed entry pre-decremented sp by the
  * saved-pair + variadic-GP-save area (frame_save_size + va_save_sz), so only
  * `frame_size - (frame_save_size + va_save_sz)` remains to subtract. RISC-V has
  * no stack probe, so this is the whole deferred region. Returns the word count
  * (0 when nothing remains to subtract; <= 3 for the far form). */
 static u32 rv_build_ndt_sub(RvNativeTarget* a, u32 frame_size, u32* words,
                             u32 cap) {
   const RiscvVariant* v = a->variant;
   u32 pair_area = v->frame_save_size + rv_va_save_sz(a);
   u32 sub_bytes = frame_size - pair_area;
   u32 wi = 0;
 #define ADDI_LO(rd, lo) \
   (v->has_w_forms ? rv_addiw((rd), (rd), (lo)) : rv_addi((rd), (rd), (lo)))
 #define PUSH(w)                                                  \
   do {                                                           \
     if (wi >= cap) rv_panic(a, "ndt sub placeholder overflow"); \
     words[wi++] = (w);                                           \
   } while (0)
   if (!sub_bytes) return 0;
   if (fits_i12(-(i32)sub_bytes)) {
     PUSH(rv_addi(RV_SP, RV_SP, -(i32)sub_bytes));
   } else {
     i32 neg = -(i32)sub_bytes;
     i32 hi = (i32)(((i64)neg + 0x800) >> 12);
     i32 lo = neg - (i32)((u32)hi << 12);
     PUSH(rv_lui(RV_TMP0, (u32)hi & 0xfffffu));
     if (lo) PUSH(ADDI_LO(RV_TMP0, lo));
     PUSH(rv_add(RV_SP, RV_SP, RV_TMP0));
   }
 #undef PUSH
 #undef ADDI_LO
   return wi;
 }
 
 static void rv_func_begin(NativeTarget* t, const CGFuncDesc* fd) {
   RvNativeTarget* a = rv_of(t);
   MCEmitter* mc = t->mc;
   const RiscvVariant* v;
   u32 pair_area, region, i;
   rv_func_begin_common(t, fd);
   v = a->variant;
   /* tcc-style prologue. Emit the frame-independent entry live: pre-decrement sp
    * by just the saved-pair + variadic-GP-save area, store the s0/ra pair at the
    * top of it, and anchor s0 there. The pair and s0 land at exactly the
    * addresses every s0-relative offset already assumes (s0 == old_sp -
    * frame_save_size - va_save_sz, identical to the old `sub frame_size` then
    * `addi s0,sp,fp_pair_off`), so the frame is byte-identical — only the
    * instruction sequence changes. pair_area is small (<=80 rv64 / <=40 rv32) and
    * always fits imm12. */
   pair_area = v->frame_save_size + rv_va_save_sz(a);
   rv64_emit32(mc, rv_addi(RV_SP, RV_SP, -(i32)pair_area));
   rv64_emit32(mc, rv_sd_ptr(v, RV_S0, RV_SP, 0));
   rv64_emit32(mc, rv_sd_ptr(v, RV_RA, RV_SP, (i32)v->ptr_bytes));
   rv64_emit32(mc, rv_addi(RV_S0, RV_SP, 0)); /* s0 = sp = saved-pair address */
   /* Reserve only the deferred `sub sp` region (patched in rv_func_end); record
    * it for the patch and CFI. RISC-V has no stack probe, so the region is just
    * the worst-case far `sub`. */
   region = RV_NDT_SUB_WORDS;
   a->prologue_pos = mc_pos(mc);
   /* The live fixed entry is exactly RV_NDT_FIXED_ENTRY_WORDS words; the CFI
    * advance in rv_func_end relies on prologue_pos sitting just past it. */
   if (a->prologue_pos - a->func_start != RV_NDT_FIXED_ENTRY_WORDS * 4u)
     rv_panic(a, "fixed prologue entry size drifted");
   a->prologue_region_words = region;
   for (i = 0; i < region; ++i) rv64_emit32(mc, RV_NOP);
   /* sret + variadic GP spills are s0-relative and run live after the region. */
   rv_reserve_entry_saves(a);
   rv_emit_entry_save_stores(a);
 }
 
 static void rv_func_end(NativeTarget* t) {
   RvNativeTarget* a = rv_of(t);
   MCEmitter* mc = t->mc;
   ObjBuilder* obj = t->obj;
   ObjSecId sec = a->func->text_section_id;
   u32 int_regs[RV_MAX_CALLEE_SAVES], fp_regs[RV_MAX_CALLEE_SAVES];
   u32 n_int = rv_collect_int_saves(a, int_regs);
   u32 n_fp = rv_collect_fp_saves(a, fp_regs);
   u32 frame_size = rv_frame_size(a);
   u32 fp_pair_off = rv_fp_pair_off(a, frame_size);
   u32 end;
   i32 i;
   a->frame_size_final = frame_size;
   a->fp_pair_off = fp_pair_off;
 
   /* epilogue */
   mc_label_place(mc, a->epilogue_label);
   if (a->slim_prologue) {
     /* Frameless leaf: no callee-saves, no s0/ra to reload, sp untouched. */
     rv64_emit32(mc, rv_jalr(RV_ZERO, RV_RA, 0));
   } else {
     const RiscvVariant* v = a->variant;
     for (i = (i32)n_int - 1; i >= 0; --i)
       rv_load_s0(v, mc, 0, int_regs[i], rv_save_off(a, n_int, (u32)i));
     for (i = (i32)n_fp - 1; i >= 0; --i)
       rv_load_s0(v, mc, 1, fp_regs[i], rv_save_off(a, n_int, n_int + (u32)i));
     if (a->frame.has_alloca)
       rv_emit_addr_adjust(v, mc, RV_SP, RV_S0, -(i32)fp_pair_off);
     /* Reload ra/s0 from the saved pair (s0 at +0, ra at +ptr_bytes), pointer
      * width. */
     rv64_emit32(mc, rv_ld_ptr(v, RV_RA, RV_S0, (i32)v->ptr_bytes));
     rv64_emit32(mc, rv_ld_ptr(v, RV_S0, RV_S0, 0));
     /* sp += frame_size */
     if (fits_i12((i32)frame_size)) {
       rv64_emit32(mc, rv_addi(RV_SP, RV_SP, (i32)frame_size));
     } else {
       rv_emit_load_imm(v, mc, 1, RV_TMP0, (i64)frame_size);
       rv64_emit32(mc, rv_add(RV_SP, RV_SP, RV_TMP0));
     }
     rv64_emit32(mc, rv_jalr(RV_ZERO, RV_RA, 0));
   }
 
   /* patch the deferred-`sub` region. The frame-independent entry (saved pair +
    * s0 anchor) was already emitted live in rv_func_begin; only the `sub sp` that
    * grows the frame below the pair is deferred here. Single-pass path only — the
    * known-frame path emits its prologue final and never patches. */
   if (!a->frame.known_frame) {
     u32 region = a->prologue_region_words;
     u32 words[RV_NDT_SUB_WORDS];
     u32 nwords, k;
     if (region > RV_NDT_SUB_WORDS) rv_panic(a, "prologue region too large");
     for (k = 0; k < region; ++k) words[k] = RV_NOP;
     nwords = rv_build_ndt_sub(a, frame_size, words, region);
     /* If the deferred `sub` is shorter than the reserved region, branch straight
      * to the entry saves rather than executing the trailing NOPs. */
     if (nwords < region) {
       words[nwords] = rv_jal(RV_ZERO, (i32)((region - nwords) * 4u));
       for (k = nwords + 1u; k < region; ++k) words[k] = RV_NOP;
     }
     for (k = 0; k < region; ++k)
       rv_patch32(obj, sec, a->prologue_pos + k * 4u, words[k]);
   }
   /* patch alloca sites: addi dst, sp, max_outgoing */
   {
     u32 mo = align_up_u32(a->frame.max_outgoing, 16u);
     u32 k;
     if (mo > 2047u) rv_panic(a, "max_outgoing too large for alloca patch");
     for (k = 0; k < a->npatches; ++k)
       rv_patch32(obj, sec, a->patches[k].pos,
                  rv_addi(a->patches[k].dst_reg, RV_SP, (i32)mo));
   }
 
   /* CFI: CFA = s0 + (frame_size - fp_pair_off) */
   {
     if (a->slim_prologue) {
       /* Frameless leaf: CFA = sp (unchanged from entry) and the return address
        * stays live in ra (the CIE default), so no saved-register rules. The
        * state holds from the first instruction (offset 0). */
       mc_cfi_set_next_pc_offset(mc, 0);
       mc_cfi_def_cfa(mc, RV_SP, 0);
     } else {
       i32 cfa = (i32)frame_size - (i32)fp_pair_off;
       /* CFI advance to the post-prologue PC. known-frame: prologue_pos +
        * minimal_prologue_words. single-pass: prologue_pos is past the live fixed
        * entry, so add the reserved deferred-`sub` region (the saved pair + s0
        * anchor are already established by the fixed entry; the deferred `sub` is
        * what completes the frame). The sret/variadic spills after the region do
        * not move sp/s0, so the CFA state holds from here. */
       u32 post = a->prologue_pos + (a->frame.known_frame
                                         ? a->minimal_prologue_words * 4u
                                         : a->prologue_region_words * 4u);
       u32 k;
       mc_cfi_set_next_pc_offset(mc, post - a->func_start);
       mc_cfi_def_cfa(mc, RV_S0, cfa);
       mc_cfi_offset(mc, RV_S0, -cfa);
       /* ra is saved at the saved-pair stride above s0 (ptr_bytes). */
       mc_cfi_offset(mc, RV_RA, -cfa + (i32)a->variant->ptr_bytes);
       for (k = 0; k < n_int; ++k)
         mc_cfi_offset(mc, int_regs[k], rv_save_off(a, n_int, k) - cfa);
       for (k = 0; k < n_fp; ++k)
         mc_cfi_offset(mc, 32u + fp_regs[k],
                        rv_save_off(a, n_int, n_int + k) - cfa);
     }
   }
 
   end = mc_pos(mc);
   obj_symbol_define(obj, a->func->sym, sec, (u64)a->func_start,
                     (u64)(end - a->func_start));
   if (a->func->atomize)
     obj_atom_define(obj, sec, a->func_start, end - a->func_start, a->func->sym,
                     0);
   if (mc->debug) debug_func_pc_range(mc->debug, sec, a->func_start, end);
   mc_cfi_endproc(mc);
   mc_end_function(mc);
   a->func = NULL;
 }
 
 /* rv64 homes its callee-saves below the locals at rv_save_off(idx) rather than
  * in frame slots, so alloc_slots=0: native_frame just records the {reg,cls} set
  * derived from the optimizer's per-class used-masks. */
 static void rv_reserve_callee_saves(NativeTarget* t, const u32* used,
                                     u32 nclasses) {
   native_frame_set_callee_saves(&rv_of(t)->frame, used, nclasses, NULL, 0, 0);
 }
 
 static int rv_reg_is_callee_int(Reg r);
 static int rv_reg_is_callee_fp(Reg r);
 static void rv_asm_clobber_masks(Compiler* c, SrcLoc loc, const Sym* clobbers,
                                  u32 nclob, u32* int_mask, u32* fp_mask);
 
 /* Expand the arch-neutral clobber-ABI sets (KitCgAsmClobberAbiSet bits) into
  * this target's per-class caller/callee-saved register masks. */
 /* abi_clobber_masks is shared as native_asm_abi_clobber_masks
  * (cg/native_asm.h); it reads the masks from t->regs->classes. */
 
 /* Build the callee-saved set the prologue must preserve: the allocator-assigned
  * callee-saved registers (frame->callee_saved_used) plus any an inline-asm
  * block clobbers. The latter are opaque to the optimizer's operand scan, so it
  * forwards the raw clobber names (frame->asm_clobbers) and the arch-neutral
  * clobber-ABI sets (frame->asm_clobber_abi_sets); we resolve both into masks
  * and keep only the callee-saved ones — rv_reg_is_callee_int excludes s0 (the
  * frame pointer, preserved by the prologue head, not as an ordinary
  * callee-save). This is the same register selection the per-block spill used,
  * hoisted into the prologue. Writes up to `cap` per-class masks into `out` and
  * returns the class count to reserve. */
 static int rv_asm_reg_is_callee_saved(NativeTarget* t, NativeAllocClass cls,
                                       Reg r) {
   (void)t;
   return (cls == NATIVE_REG_INT && rv_reg_is_callee_int(r)) ||
          (cls == NATIVE_REG_FP && rv_reg_is_callee_fp(r));
 }
 
 static u32 rv_known_callee_saves(NativeTarget* t,
                                  const NativeKnownFrameDesc* frame, u32* out,
                                  u32 cap) {
   RvNativeTarget* a = rv_of(t);
   SrcLoc loc = a->func ? a->func->loc : (SrcLoc){0, 0, 0};
   return native_asm_known_callee_saves(
       t, loc, frame, out, cap, rv_asm_clobber_masks,
       rv_asm_reg_is_callee_saved);
 }
 
 static u32 rv_signature_stack_bytes(NativeTarget* t, KitCgTypeId fn_type,
                                     int* variadic, u32* nparams);
 
 /* Optimizer entry point: the full frame is supplied up front, so the prologue
  * is emitted final the moment it is built — no NOP region, no func_end patch
  * (rv_func_end skips patching when known_frame). rv_build_prologue emits the
  * sret spill and the variadic register-save stores inline, so there is no
  * separate entry-save emission. Slot creation order matches the single-pass
  * path: callee-saves first (only recorded for rv64), then static slots, then
  * the sret entry-save slot. */
 static void rv_func_begin_known_frame(NativeTarget* t, const CGFuncDesc* fd,
                                       const NativeKnownFrameDesc* frame,
                                       NativeFrameSlot* out_slots) {
   RvNativeTarget* a = rv_of(t);
   MCEmitter* mc = t->mc;
   u32 int_regs[RV_MAX_CALLEE_SAVES], fp_regs[RV_MAX_CALLEE_SAVES];
   u32 n_int, n_fp, frame_size, fp_pair_off, nwords, i;
   u32 words[RV_KNOWN_PROLOGUE_WORDS];
   rv_func_begin_common(t, fd);
   a->frame.known_frame = 1;
   if (frame) {
     u32 cs[NATIVE_REG_CLASS_COUNT];
     u32 ncs = rv_known_callee_saves(t, frame, cs, NATIVE_REG_CLASS_COUNT);
     a->frame.has_alloca = frame->has_alloca;
     if (ncs) rv_reserve_callee_saves(t, cs, ncs);
     for (i = 0; i < frame->nslots; ++i) {
       NativeFrameSlot slot = rv_frame_slot(t, &frame->slots[i]);
       if (out_slots) out_slots[i] = slot;
     }
     rv_reserve_entry_saves(a);
     native_frame_note_outgoing(&a->frame, frame->max_outgoing);
   }
   /* Frame is final: size and offsets are settled, so emit the exact prologue.
    */
   frame_size = rv_frame_size(a);
   fp_pair_off = rv_fp_pair_off(a, frame_size);
   a->frame_size_final = frame_size;
   a->fp_pair_off = fp_pair_off;
   a->prologue_pos = mc_pos(mc);
   /* Leaf no-frame tier (aa64 slim_prologue equivalent): a leaf with no
    * callee-saves, no body slots, no outgoing args, no sret/variadic and
    * register-only params never reads s0 (no frame slots / stack args) nor
    * clobbers ra (no calls). Emit no prologue at all; rv_func_end emits a bare
    * `ret`. cum_off==0 already implies no sret slot and no param spills, but the
    * extra guards keep the intent explicit. Inline asm is excluded: it can
    * clobber ra opaquely, and without the saved record the bare `ret` would
    * return through the destroyed link register. */
   a->slim_prologue = frame && frame->is_leaf && !frame->has_asm &&
                      !frame->reads_frame && a->frame.ncallee_saves == 0 &&
                      !a->frame.has_alloca && a->frame.cum_off == 0 &&
                      a->frame.max_outgoing == 0 && !a->has_sret &&
                      !a->is_variadic &&
                      rv_signature_stack_bytes(t, fd->fn_type, NULL, NULL) == 0;
   if (a->slim_prologue) {
     a->minimal_prologue_words = 0;
     native_frame_set_final(&a->frame);
     return;
   }
   n_int = rv_collect_int_saves(a, int_regs);
   n_fp = rv_collect_fp_saves(a, fp_regs);
   nwords = rv_build_prologue(a, words, RV_KNOWN_PROLOGUE_WORDS, frame_size,
                              fp_pair_off, int_regs, n_int, fp_regs, n_fp);
   for (i = 0; i < nwords; ++i) rv64_emit32(mc, words[i]);
   a->minimal_prologue_words = nwords;
   native_frame_set_final(&a->frame);
 }
 
 /* ============================ params / ABI helpers
  * ============================ */
 
 static const ABIArgInfo* rv_param_abi(NativeTarget* t, const ABIFuncInfo* abi,
                                       const NativeCallDesc* desc, u32 i,
                                       ABIArgInfo* scratch) {
   /* Synthesized for unnamed (variadic) args, or untyped calls. RISC-V LP64D
    * passes variadic FP args in INTEGER registers (as their bit pattern), not
    * the FP pool — so a variadic float part is ABI_CLASS_INT. */
   int variadic = abi && i >= abi->nparams;
   u32 gpr = rv_of(t)->variant->ptr_bytes; /* GPR width: 4 ilp32 / 8 lp64 */
   u32 sz, align;
   int is_fp;
   if (abi && i < abi->nparams) return &abi->params[i];
   sz = native_type_size(t, desc->args[i].type);
   align = native_type_align(t, desc->args[i].type);
   /* A variadic FP arg rides the INTEGER pool as its bit pattern (RISC-V passes
    * unnamed FP args in GPRs), so it is INT-class here. */
   is_fp = !variadic && cg_type_is_float(t->c, desc->args[i].type);
   memset(scratch, 0, sizeof *scratch);
   scratch->kind = ABI_ARG_DIRECT;
   /* A scalar wider than one GPR (an 8-byte i64 / soft-double on ilp32) rides a
    * register pair, matching the named-arg classifier (abi_rv64.c). Synthesize
    * one INT part per GPR-word so the per-part marshaller fills both registers
    * (low word in the lower-numbered reg) instead of dropping the high half into
    * a single register. FP-class args (hardware-float, size<=GPR) stay single.
    */
   if (!is_fp && sz > gpr) {
     u32 nparts = (sz + gpr - 1u) / gpr, p;
     ABIArgPart* parts = arena_zarray(t->c->tu, ABIArgPart, nparts);
     for (p = 0; p < nparts; ++p) {
       u32 off = p * gpr;
       parts[p].cls = ABI_CLASS_INT;
       parts[p].loc = ABI_LOC_REG;
       parts[p].size = (sz - off) < gpr ? (sz - off) : gpr;
       parts[p].align = gpr;
       parts[p].src_offset = off;
     }
     scratch->nparts = nparts;
     scratch->parts = parts;
     return scratch;
   }
   scratch->nparts = 1;
   scratch->parts = arena_zarray(t->c->tu, ABIArgPart, 1);
   ((ABIArgPart*)scratch->parts)[0].cls = is_fp ? ABI_CLASS_FP : ABI_CLASS_INT;
   ((ABIArgPart*)scratch->parts)[0].loc = ABI_LOC_REG;
   ((ABIArgPart*)scratch->parts)[0].size = sz;
   ((ABIArgPart*)scratch->parts)[0].align = align;
   return scratch;
 }
 
 /* Outgoing stack-slot size/align: the xlen-word (gp_slot_bytes: 8 lp64d /
  * 4 ilp32) is the natural slot stride; stack ABI alignment caps at 16. */
 static u32 rv_part_stack_size(const RiscvVariant* v, const ABIArgPart* part) {
   u32 slot = v->gp_slot_bytes;
   return align_up_u32(part->size ? part->size : slot, slot);
 }
 static u32 rv_part_stack_align(const RiscvVariant* v, const ABIArgPart* part) {
   u32 slot = v->gp_slot_bytes;
   u32 al = part->align ? part->align : slot;
   if (al < slot) al = slot;
   if (al > 16u) al = 16u;
   return al;
 }
 
 static KitCgTypeId rv_part_scalar_type(const ABIArgPart* part) {
   if (part->cls == ABI_CLASS_FP) {
     if (part->size <= 4u) return builtin_id(KIT_CG_BUILTIN_F32);
     return builtin_id(KIT_CG_BUILTIN_F64);
   }
   switch (part->size) {
     case 1u:
       return builtin_id(KIT_CG_BUILTIN_I8);
     case 2u:
       return builtin_id(KIT_CG_BUILTIN_I16);
     case 4u:
       return builtin_id(KIT_CG_BUILTIN_I32);
     default:
       return builtin_id(KIT_CG_BUILTIN_I64);
   }
 }
 
 static u32 rv_class_stack_size(const RiscvVariant* v, const ABIArgInfo* ai) {
   u32 slot = v->gp_slot_bytes;
   u32 total = 0, p;
   if (!ai || ai->kind == ABI_ARG_IGNORE) return 0;
   if (ai->kind == ABI_ARG_INDIRECT) return v->ptr_bytes;
   for (p = 0; p < ai->nparts; ++p) {
     total = align_up_u32(total, rv_part_stack_align(v, &ai->parts[p]));
     total += rv_part_stack_size(v, &ai->parts[p]);
   }
   return align_up_u32(total ? total : slot, slot);
 }
 
 static u32 rv_call_stack_size(NativeTarget* t, const NativeCallDesc* desc) {
   const RiscvVariant* v = rv_of(t)->variant;
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, desc->fn_type);
   /* sret consumes a0 as the implicit first integer argument. */
   u32 next_int = (abi && abi->has_sret) ? 1u : 0u;
   u32 next_fp = 0, stack = 0, i, p;
   for (i = 0; i < desc->nargs; ++i) {
     ABIArgInfo tmp;
     const ABIArgInfo* ai = rv_param_abi(t, abi, desc, i, &tmp);
     int force_stack =
         abi && abi->variadic && abi->vararg_on_stack && i >= abi->nparams;
     if (ai->kind == ABI_ARG_IGNORE) continue;
     if (force_stack) {
       stack += rv_class_stack_size(v, ai);
       continue;
     }
     if (ai->kind == ABI_ARG_INDIRECT) {
       if (next_int < 8u)
         next_int++;
       else
         stack += v->ptr_bytes;
       continue;
     }
     for (p = 0; p < ai->nparts; ++p) {
       const ABIArgPart* part = &ai->parts[p];
       if (part->cls == ABI_CLASS_FP) {
         if (next_fp < 8u)
           next_fp++;
         else {
           stack = align_up_u32(stack, rv_part_stack_align(v, part));
           stack += rv_part_stack_size(v, part);
         }
       } else {
         if (next_int < 8u)
           next_int++;
         else {
           stack = align_up_u32(stack, rv_part_stack_align(v, part));
           stack += rv_part_stack_size(v, part);
         }
       }
     }
   }
   return align_up_u32(stack, 16u);
 }
 
 static u32 rv_signature_stack_bytes(NativeTarget* t, KitCgTypeId fn_type,
                                     int* variadic, u32* nparams) {
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fn_type);
   NativeCallDesc d;
   if (variadic) *variadic = abi ? (int)abi->variadic : 0;
   if (nparams) *nparams = abi ? abi->nparams : 0u;
   memset(&d, 0, sizeof d);
   d.fn_type = fn_type;
   d.nargs = abi ? abi->nparams : 0u;
   if (d.nargs) d.args = arena_zarray(t->c->tu, NativeLoc, d.nargs);
   return rv_call_stack_size(t, &d);
 }
 
 static u32 rv_call_stack_bytes(NativeTarget* t, const NativeCallDesc* desc) {
   return rv_call_stack_size(t, desc);
 }
 
 /* Resolve a dereferenceable NativeLoc to its storage address. */
 static NativeAddr rv_storage_addr(RvNativeTarget* a, NativeLoc loc,
                                   u32 offset) {
   NativeAddr addr;
   if (!native_loc_storage_addr(loc, (i32)offset, &addr))
     rv_panic(a, "location is not storage");
   return addr;
 }
 
 static void rv_load_part(NativeTarget* t, NativeLoc dst, NativeLoc src,
                          u32 offset, u32 size) {
   RvNativeTarget* a = rv_of(t);
   if (src.kind == NATIVE_LOC_REG) {
     rv_move(t, dst, src);
     return;
   }
   if (native_loc_addr_role(src) == NATIVE_LOC_ADDR_ROLE_VALUE) {
     NativeAddr addr;
     if (offset != 0u || size != t->c->target.ptr_size ||
         !native_loc_address_value(src, &addr))
       rv_panic(a, "split or invalid address-value argument");
     rv_load_addr(t, dst, addr);
     return;
   }
   if (native_loc_addr_role(src) == NATIVE_LOC_ADDR_ROLE_STORAGE) {
     NativeAddr addr = rv_storage_addr(a, src, offset);
     addr.base_type = dst.type;
     rv_emit_mem(a, 1, dst, addr, native_mem_for_type(t, dst.type, size));
     return;
   }
   if (src.kind == NATIVE_LOC_IMM) {
     i64 part;
     if (!native_loc_imm_part(src, offset, size, &part))
       rv_panic(a, "invalid immediate argument part");
     rv_emit_load_imm(a->variant, t->mc, loc_is_64(t, dst) ? 1u : 0u,
                      loc_reg(dst), part);
     return;
   }
   rv_panic(a, "unsupported part source");
 }
 
 static void rv_store_part(NativeTarget* t, NativeLoc dst, NativeLoc src,
                           u32 offset, u32 size) {
   RvNativeTarget* a = rv_of(t);
   if (native_loc_addr_role(dst) == NATIVE_LOC_ADDR_ROLE_STORAGE) {
     NativeAddr addr = rv_storage_addr(a, dst, offset);
     addr.base_type = src.type;
     rv_emit_mem(a, 0, src, addr, native_mem_for_type(t, src.type, size));
     return;
   }
   if (dst.kind == NATIVE_LOC_REG) {
     rv_move(t, dst, src);
     return;
   }
   rv_panic(a, "unsupported part destination");
 }
 
 static void rv_addr_of_loc(NativeTarget* t, NativeLoc dst, NativeLoc src) {
   NativeAddr addr = rv_storage_addr(rv_of(t), src, 0);
   rv_load_addr(t, dst, addr);
 }
 
 static void rv_store_outgoing_part(NativeTarget* t, int tail_call,
                                    u32 stack_off, NativeLoc src, u32 size) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_REG;
   addr.base_type = src.type;
   if (tail_call) {
     /* A sibling call reuses the caller's frame: its outgoing stack args land in
      * the caller's incoming-arg window ([s0 + 16 + va_save + off]) — physically
      * the same address the tail-callee will read at [sp+off] once the teardown
      * has restored sp to the caller's entry sp (the CFA). */
     addr.base.reg = RV_S0;
     addr.offset = rv_s0_off_in_arg(rv_of(t), stack_off);
   } else {
     addr.base.reg = RV_SP;
     addr.offset = (i32)stack_off;
   }
   rv_emit_mem(rv_of(t), 0, src, addr, native_mem_for_type(t, src.type, size));
 }
 
 /* Copy exactly the semantic bytes of a stack-routed value. The caller owns
  * the separately rounded physical ABI slot and advances its stack cursor by
  * that carrier size; padding is never treated as readable source storage. */
 static void rv_store_outgoing_value_bytes(NativeTarget* t, int tail_call,
                                           u32 stack_off, NativeLoc src,
                                           u32 value_size) {
   RvNativeTarget* a = rv_of(t);
   NativeLoc tmp = native_loc_reg(src.type, NATIVE_REG_INT, RV_TMP0);
   u32 off = 0;
   while (off < value_size) {
     u32 chunk = value_size - off;
     if (chunk > a->variant->ptr_bytes) chunk = a->variant->ptr_bytes;
     rv_load_part(t, tmp, src, off, chunk);
     rv_store_outgoing_part(t, tail_call, stack_off + off, tmp, chunk);
     off += chunk;
   }
 }
 
 /* Give every indirect target one lifetime that is disjoint from argument
  * transport. `ra` is reserved from allocation and from the t0-t3 backend
  * temporary bank, while its incoming value is already saved in the function
  * frame at any real call site. A normal JALR may read and write ra in the same
  * instruction. A tail site copies this parked value to t1 immediately before
  * teardown restores the incoming ra.
  *
  * Lowering normally presents REG/GLOBAL, but NativeCallDesc permits scalar
  * address values and dereferenceable storage too. Route every non-global shape
  * through the same checked part boundary so FRAME/STACK/ADDR/FRAME_ADDR/IMM
  * remain valid without acquiring an optimizer-owned register. NONE is kept for
  * planning-only queries that never emit a call. */
 static NativeLoc rv_stage_indirect_callee(NativeTarget* t, NativeLoc callee) {
   RvNativeTarget* a = rv_of(t);
   NativeLoc parked;
   if (callee.kind == NATIVE_LOC_NONE || callee.kind == NATIVE_LOC_GLOBAL)
     return callee;
   if (callee.kind == NATIVE_LOC_REG &&
       (NativeAllocClass)callee.cls != NATIVE_REG_INT)
     rv_panic(a, "indirect callee is not in an integer register");
   if (!callee.type) rv_panic(a, "indirect callee has no pointer type");
   parked = native_loc_reg(callee.type, NATIVE_REG_INT, RV_RA);
   rv_load_part(t, parked, callee, 0, a->variant->ptr_bytes);
   return parked;
 }
 
 /* NativeTarget bind_param: route incoming param (ABI loc) into dst. */
 static void rv_bind_native_param(NativeTarget* t, const CGParamDesc* p,
                                  NativeLoc dst) {
   RvNativeTarget* a = rv_of(t);
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, a->func->fn_type);
   const ABIArgInfo* ai =
       p->index < abi->nparams ? &abi->params[p->index] : NULL;
   int to_reg = dst.kind == NATIVE_LOC_REG;
   u32 i;
   if (!ai || ai->kind == ABI_ARG_IGNORE) return;
   if (ai->kind == ABI_ARG_INDIRECT) {
     NativeLoc src = native_loc_reg(
         builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT,
         a->next_param_int < 8u ? RV_A0 + a->next_param_int : RV_TMP0);
     NativeAddr d_addr, from;
     AggregateAccess access;
     if (a->next_param_int < 8u) {
       a->next_param_int++;
     } else {
       NativeAddr sa;
       memset(&sa, 0, sizeof sa);
       sa.base_kind = NATIVE_ADDR_BASE_REG;
       sa.base.reg = RV_S0;
       sa.offset = rv_s0_off_in_arg(a, a->next_param_stack);
       sa.base_type = src.type;
       rv_emit_mem(a, 1, src, sa,
                   native_mem_for_type(t, src.type, a->variant->ptr_bytes));
       a->next_param_stack += a->variant->ptr_bytes;
     }
     if (dst.kind != NATIVE_LOC_FRAME)
       rv_panic(a, "indirect parameter requires a frame destination");
     memset(&d_addr, 0, sizeof d_addr);
     d_addr.base_kind = NATIVE_ADDR_BASE_FRAME;
     d_addr.base.frame = dst.v.frame;
     d_addr.base_type = p->type;
     memset(&from, 0, sizeof from);
     from.base_kind = NATIVE_ADDR_BASE_REG;
     from.base.reg = loc_reg(src);
     from.base_type = p->type;
     memset(&access, 0, sizeof access);
     access.type = p->type;
     access.size = p->size ? p->size : (u32)cg_type_size(t->c, p->type);
     access.align = p->align ? p->align : native_type_align(t, p->type);
     rv_copy_bytes(t, d_addr, from, access);
     return;
   }
   for (i = 0; i < ai->nparts; ++i) {
     const ABIArgPart* part = &ai->parts[i];
     NativeAllocClass cls =
         part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
     NativeLoc src;
     if (cls == NATIVE_REG_FP && a->next_param_fp < 8u) {
       src = native_loc_reg(p->type, cls, RV_FA0 + a->next_param_fp++);
     } else if (cls == NATIVE_REG_INT && a->next_param_int < 8u) {
       src = native_loc_reg(p->type, cls, RV_A0 + a->next_param_int++);
     } else {
       Reg tmp = (cls == NATIVE_REG_FP) ? RV_FTMP0 : RV_TMP0;
       NativeAddr sa;
       src = native_loc_reg(p->type, cls, tmp);
       a->next_param_stack = align_up_u32(a->next_param_stack,
                                          rv_part_stack_align(a->variant, part));
       memset(&sa, 0, sizeof sa);
       sa.base_kind = NATIVE_ADDR_BASE_REG;
       sa.base.reg = RV_S0;
       sa.base_type = p->type;
       sa.offset = rv_s0_off_in_arg(a, a->next_param_stack);
       rv_emit_mem(a, 1, src, sa, native_mem_for_type(t, p->type, part->size));
       a->next_param_stack += rv_part_stack_size(a->variant, part);
     }
     if (dst.kind == NATIVE_LOC_NONE) {
       /* unused parameter; cursors already advanced */
     } else if (to_reg) {
       NativeLoc d = native_loc_reg(dst.type ? dst.type : p->type,
                                    (NativeAllocClass)dst.cls, (Reg)dst.v.reg);
       if (!(src.kind == NATIVE_LOC_REG && loc_reg(src) == loc_reg(d) &&
             (NativeAllocClass)src.cls == (NativeAllocClass)d.cls))
         rv_move(t, d, src);
     } else {
       rv_store_part(
           t, native_loc_stack(p->type, dst.v.frame, (i32)part->src_offset), src,
           0, part->size);
     }
   }
   a->incoming_stack_size = align_up_u32(a->next_param_stack, 16u);
 }
 
 /* ============================ calls / returns ============================ */
 
 typedef NativeArgMove RvArgMove;
 
 static void rv_emit_one_arg_move(NativeTarget* t, const NativeArgMove* m) {
   if (m->is_addr)
     rv_addr_of_loc(t, m->dst, m->src);
   else
     rv_load_part(t, m->dst, m->src, m->src_offset, m->size);
 }
 
 /* Parallel-copy register arg moves via the shared scheduler; cycles break
  * through backend-private t1/ft1. */
 static void rv_emit_reg_arg_moves(NativeTarget* t, NativeArgMove* moves,
                                   u32 n) {
   NativeArgShuffle s;
   if (n > RV_MAX_REG_ARG_MOVES) rv_panic(rv_of(t), "too many register args");
   memset(&s, 0, sizeof s);
   s.t = t;
   s.emit_one = rv_emit_one_arg_move;
   s.reg_move = rv_move;
   s.scratch[NATIVE_REG_INT] = RV_TMP1;
   s.scratch[NATIVE_REG_FP] = RV_FTMP1;
   s.scratch_class_mask = (1u << NATIVE_REG_INT) | (1u << NATIVE_REG_FP);
   native_arg_shuffle(&s, moves, n);
 }
 
 static void rv_marshal_call(NativeTarget* t, const NativeCallDesc* desc,
                             NativeCallPhase* plan) {
   RvNativeTarget* a = rv_of(t);
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, desc->fn_type);
   NativeCallPhaseRet* rets;
   KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
   /* Right-size the result scratch to the exact number of entries the ret loops
    * below write: nparts on a DIRECT register return, 1 on the !abi fallback,
    * 0 (NULL) otherwise (IGNORE / sret / no results). */
   u32 nrets_cap = (abi && abi->ret.kind == ABI_ARG_DIRECT && desc->nresults)
                       ? abi->ret.nparts
                       : ((!abi && desc->nresults) ? 1u : 0u);
   memset(plan, 0, sizeof *plan);
   rets = nrets_cap ? arena_zarray(t->c->tu, NativeCallPhaseRet, nrets_cap) : NULL;
   plan->callee = desc->callee;
   plan->rets = rets;
   plan->flags = desc->flags;
   plan->has_sret = abi && abi->has_sret;
   plan->is_variadic = abi && abi->variadic;
   plan->stack_arg_size = rv_call_stack_size(t, desc);
   if (plan->stack_arg_size > a->frame.max_outgoing)
     a->frame.max_outgoing = plan->stack_arg_size;
   plan->callee = rv_stage_indirect_callee(t, plan->callee);
   {
     /* sret returns pass the hidden destination pointer as the implicit first
      * integer argument (a0), so the real args start at a1. */
     u32 next_int = (abi && abi->has_sret) ? 1u : 0u;
     u32 next_fp = 0, stack = 0, nmoves = 0, i, p;
     int tail = (desc->flags & CG_CALL_TAIL) != 0;
     RvArgMove moves[RV_MAX_REG_ARG_MOVES];
     for (i = 0; i < desc->nargs; ++i) {
       ABIArgInfo tmp;
       const ABIArgInfo* ai = rv_param_abi(t, abi, desc, i, &tmp);
       int force_stack =
           abi && abi->variadic && abi->vararg_on_stack && i >= abi->nparams;
       if (ai->kind == ABI_ARG_IGNORE) continue;
       if (force_stack) {
         u32 value_size = native_type_size(t, desc->args[i].type);
         u32 n = rv_class_stack_size(a->variant, ai);
         rv_store_outgoing_value_bytes(t, tail, stack, desc->args[i],
                                       value_size);
         stack += n;
         continue;
       }
       if (ai->kind == ABI_ARG_INDIRECT) {
         u32 ptr_sz = a->variant->ptr_bytes;
         if (next_int < 8u) {
           RvArgMove* m = &moves[nmoves++];
           m->dst = native_loc_reg(i64t, NATIVE_REG_INT, RV_A0 + next_int++);
           m->src = desc->args[i];
           m->src_offset = 0;
           m->size = ptr_sz;
           m->is_addr = 1;
         } else {
           NativeLoc ptr = native_loc_reg(i64t, NATIVE_REG_INT, RV_TMP0);
           rv_addr_of_loc(t, ptr, desc->args[i]);
           rv_store_outgoing_part(t, tail, stack, ptr, ptr_sz);
           stack += ptr_sz;
         }
         continue;
       }
       for (p = 0; p < ai->nparts; ++p) {
         const ABIArgPart* part = &ai->parts[p];
         NativeAllocClass cls =
             part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
         if ((cls == NATIVE_REG_FP && next_fp < 8u) ||
             (cls == NATIVE_REG_INT && next_int < 8u)) {
           RvArgMove* m = &moves[nmoves++];
           Reg areg =
               cls == NATIVE_REG_FP ? RV_FA0 + next_fp++ : RV_A0 + next_int++;
           m->dst = native_loc_reg(desc->args[i].type, cls, areg);
           m->src = desc->args[i];
           m->src_offset = part->src_offset;
           m->size = part->size;
           m->is_addr = 0;
         } else {
           Reg tmp = cls == NATIVE_REG_FP ? RV_FTMP0 : RV_TMP0;
           NativeLoc tmpreg = native_loc_reg(desc->args[i].type, cls, tmp);
           rv_load_part(t, tmpreg, desc->args[i], part->src_offset, part->size);
           stack = align_up_u32(stack, rv_part_stack_align(a->variant, part));
           rv_store_outgoing_part(t, tail, stack, tmpreg, part->size);
           stack += rv_part_stack_size(a->variant, part);
         }
       }
     }
     rv_emit_reg_arg_moves(t, moves, nmoves);
     if (abi && abi->has_sret) {
       /* sret pointer goes in a0; arg loads have completed. A tail call forwards
        * the caller's own incoming sret pointer (spilled at entry) so the
        * sibling writes the result into the caller's caller's destination;
        * otherwise pass the address of this call's result slot. */
       NativeLoc a0 = native_loc_reg(i64t, NATIVE_REG_INT, RV_A0);
       if (tail)
         rv_load_part(t, a0, native_loc_stack(i64t, a->sret_ptr_slot, 0), 0,
                      a->variant->ptr_bytes);
       else if (desc->nresults)
         rv_addr_of_loc(t, a0, desc->results[0]);
     }
   }
   if (abi && abi->ret.kind == ABI_ARG_DIRECT && desc->nresults) {
     u32 nr = 0, ni = 0, nf = 0, p;
     for (p = 0; p < abi->ret.nparts; ++p) {
       const ABIArgPart* part = &abi->ret.parts[p];
       NativeAllocClass cls =
           part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
       KitCgTypeId pty = rv_part_scalar_type(part);
       Reg rreg = cls == NATIVE_REG_FP ? RV_FA0 + nf++ : RV_A0 + ni++;
       rets[nr].src = native_loc_reg(pty, cls, rreg);
       rets[nr].dst = desc->results[0];
       if (rets[nr].dst.kind == NATIVE_LOC_FRAME)
         rets[nr].dst = native_loc_stack(pty, desc->results[0].v.frame,
                                         (i32)part->src_offset);
       else if (rets[nr].dst.kind == NATIVE_LOC_STACK) {
         rets[nr].dst.v.stack.offset += (i32)part->src_offset;
         rets[nr].dst.type = pty;
       }
       rets[nr].mem = native_mem_for_type(t, pty, part->size);
       nr++;
     }
     plan->nrets = nr;
   } else if (abi && abi->ret.kind == ABI_ARG_IGNORE) {
     plan->nrets = 0;
   } else if (!abi && desc->nresults) {
     rets[0].src = native_loc_reg(desc->results[0].type, NATIVE_REG_INT, RV_A0);
     rets[0].dst = desc->results[0];
     rets[0].mem = native_mem_for_type(t, desc->results[0].type, 0);
     plan->nrets = 1;
   }
 }
 
 /* Emit a sibling (tail) call: tear the frame down to the caller's entry state
  * and jump (no link) to the callee. Outgoing args are already in the arg regs /
  * the caller's incoming-arg window. At -O0 there are no callee-saves, and the
  * sp restore uses the CFA offset (s0 + 16 + va_save), which is independent of
  * the not-yet-final frame_size — so no func_end patching is needed. */
 static void rv_emit_tail_site(NativeTarget* t, NativeLoc callee) {
   RvNativeTarget* a = rv_of(t);
   const RiscvVariant* v = a->variant;
   MCEmitter* mc = t->mc;
   i32 cfa = (i32)(v->frame_save_size + rv_va_save_sz(a));
   int indirect = callee.kind == NATIVE_LOC_REG;
   u32 int_regs[RV_MAX_CALLEE_SAVES], fp_regs[RV_MAX_CALLEE_SAVES];
   u32 n_int = rv_collect_int_saves(a, int_regs);
   u32 n_fp = rv_collect_fp_saves(a, fp_regs);
   i32 i;
   /* Stage the target parked in ra into a reserved scratch BEFORE teardown: the
    * callee-save / s0 / ra restores below would otherwise overwrite it. t1 is
    * reserved (never allocable) and untouched by the restore loop (which only
    * uses t0 for far offsets). */
   if (indirect) rv64_emit32(mc, rv_addi(RV_TMP1, loc_reg(callee), 0));
   /* Restore callee-saves before tearing the frame down (O1 path; none at -O0).
    * Their save offsets are s0-relative via rv_save_off, so the restore is
    * frame-size- and teardown-order-independent. */
   for (i = (i32)n_int - 1; i >= 0; --i)
     rv_load_s0(v, mc, 0, int_regs[i], rv_save_off(a, n_int, (u32)i));
   for (i = (i32)n_fp - 1; i >= 0; --i)
     rv_load_s0(v, mc, 1, fp_regs[i], rv_save_off(a, n_int, n_int + (u32)i));
   rv64_emit32(mc, rv_ld_ptr(v, RV_RA, RV_S0, (i32)v->ptr_bytes));
   rv64_emit32(mc, rv_addi(RV_SP, RV_S0, cfa));
   rv64_emit32(mc, rv_ld_ptr(v, RV_S0, RV_S0, 0));
   if (callee.kind == NATIVE_LOC_GLOBAL) {
     u32 pos = mc_pos(mc);
     rv64_emit32(mc, rv_auipc(RV_TMP0, 0));
     rv64_emit32(mc, rv_jalr(RV_ZERO, RV_TMP0, 0));
     mc_emit_reloc_at(mc, mc->section_id, pos, R_RV_CALL, callee.v.global.sym,
                       callee.v.global.addend, 0, 0);
   } else if (indirect) {
     rv64_emit32(mc, rv_jalr(RV_ZERO, RV_TMP1, 0));
   } else {
     rv_panic(a, "unsupported tail call target");
   }
 }
 
 static void rv_emit_call(NativeTarget* t, const NativeCallPhase* plan) {
   MCEmitter* mc = t->mc;
   ObjSecId sec = mc->section_id;
   if (plan->flags & CG_CALL_TAIL) {
     rv_emit_tail_site(t, plan->callee);
     return;
   }
   if (plan->callee.kind == NATIVE_LOC_GLOBAL) {
     u32 pos = mc_pos(mc);
     rv64_emit32(mc, rv_auipc(RV_RA, 0));
     rv64_emit32(mc, rv_jalr(RV_RA, RV_RA, 0));
     mc_emit_reloc_at(mc, sec, pos, R_RV_CALL, plan->callee.v.global.sym,
                       plan->callee.v.global.addend, 0, 0);
     return;
   }
   if (plan->callee.kind == NATIVE_LOC_REG) {
     rv64_emit32(mc, rv_jalr(RV_RA, loc_reg(plan->callee), 0));
     return;
   }
   rv_panic(rv_of(t), "unsupported call target");
 }
 
 static void rv_marshal_ret(NativeTarget* t, const CGFuncDesc* fd,
                            const NativeLoc* value,
                            NativeCallPhaseRet** out_rets, u32* out_nrets) {
   RvNativeTarget* a = rv_of(t);
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fd->fn_type);
   NativeCallPhaseRet* rets = NULL;
   u32 nr = 0;
   if (value) rets = arena_zarray(t->c->tu, NativeCallPhaseRet, 4);
   if (value && abi && abi->ret.kind == ABI_ARG_INDIRECT) {
     KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
     NativeLoc dstp = native_loc_reg(i64t, NATIVE_REG_INT, RV_TMP1);
     NativeLoc saved = native_loc_stack(i64t, a->sret_ptr_slot, 0);
     NativeAddr dst_addr, src_addr;
     AggregateAccess access;
     rv_load_part(t, dstp, saved, 0, a->variant->ptr_bytes);
     memset(&dst_addr, 0, sizeof dst_addr);
     dst_addr.base_kind = NATIVE_ADDR_BASE_REG;
     dst_addr.base.reg = RV_TMP1;
     dst_addr.base_type = value->type;
     src_addr = rv_storage_addr(a, *value, 0);
     src_addr.base_type = value->type;
     memset(&access, 0, sizeof access);
     access.type = value->type;
     access.size = (u32)cg_type_size(t->c, value->type);
     access.align = native_type_align(t, value->type);
     rv_copy_bytes(t, dst_addr, src_addr, access);
     *out_rets = NULL;
     *out_nrets = 0;
     return;
   }
   if (value && abi && abi->ret.kind == ABI_ARG_DIRECT) {
     u32 ni = 0, nf = 0, p;
     for (p = 0; p < abi->ret.nparts; ++p) {
       const ABIArgPart* part = &abi->ret.parts[p];
       NativeAllocClass cls =
           part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
       KitCgTypeId pty = rv_part_scalar_type(part);
       Reg rreg = cls == NATIVE_REG_FP ? RV_FA0 + nf++ : RV_A0 + ni++;
       NativeLoc dst = native_loc_reg(pty, cls, rreg);
       /* Returning a memory-homed aggregate through the generic move phase
        * loses ABIArgPart.size while materializing its scalar carrier type (a
        * six-byte part becomes an eight-byte load). Marshal here through the
        * same exact-width boundary as outgoing arguments instead. */
       rv_load_part(t, dst, *value, part->src_offset, part->size);
     }
     *out_rets = NULL;
     *out_nrets = 0;
     return;
   } else if (value) {
     rets[0].src = *value;
     rets[0].dst = native_loc_reg(value->type, NATIVE_REG_INT, RV_A0);
     rets[0].mem = native_mem_for_type(t, value->type, 0);
     nr = 1;
   }
   *out_rets = rets;
   *out_nrets = nr;
 }
 
 static void rv_ret(NativeTarget* t) {
   RvNativeTarget* a = rv_of(t);
   rv_jump(t, a->epilogue_label);
 }
 
 /* ============================ alloca ============================ */
 
 static void rv_alloca(NativeTarget* t, NativeLoc dst, NativeLoc size,
                       u32 align) {
   RvNativeTarget* a = rv_of(t);
   MCEmitter* mc = t->mc;
   u32 rsz = loc_reg(size);
   u32 rd = loc_reg(dst);
   u32 al = align ? align : 16u;
   if (al < 16u) al = 16u;
   /* round up: t0 = (size + (al-1)) & ~(al-1) */
   rv64_emit32(mc, rv_addi(RV_TMP0, rsz, (i32)(al - 1u)));
   rv_emit_load_imm(a->variant, mc, 1, RV_TMP1, -(i64)al);
   rv64_emit32(mc, rv_and(RV_TMP0, RV_TMP0, RV_TMP1));
   rv64_emit32(mc, rv_sub(RV_SP, RV_SP, RV_TMP0));
   a->frame.has_alloca = 1;
   /* dst = sp + max_outgoing (patched in func_end) */
   if (a->npatches == a->patches_cap) {
     u32 cap = a->patches_cap ? a->patches_cap * 2u : 8u;
     RvPatch* nb = arena_zarray(t->c->tu, RvPatch, cap);
     if (a->patches) memcpy(nb, a->patches, sizeof(*nb) * a->npatches);
     a->patches = nb;
     a->patches_cap = cap;
   }
   a->patches[a->npatches].kind = RV_PATCH_ALLOCA;
   a->patches[a->npatches].pos = mc_pos(mc);
   a->patches[a->npatches].dst_reg = rd;
   a->npatches++;
   a->nalloca++;
   rv64_emit32(mc, RV_NOP); /* placeholder for addi dst, sp, max_outgoing */
 }
 
 /* ============================ TLS / bitfield / atomics
  * ============================ */
 
 static void rv_tls_addr_of(NativeTarget* t, NativeLoc dst, ObjSymId sym,
                            i64 addend) {
   MCEmitter* mc = t->mc;
   u32 sec = mc->section_id;
   u32 rd = loc_reg(dst);
   /* Local-Exec only, matching aa64 (aa_tls_addr_of) and x64 (x64_tls_addr_of):
    * kit links the whole module statically, so every _Thread_local symbol is
    * resolved within the image and TPREL is always valid. An Initial-Exec GOT
    * path (R_RV_TLS_GOT_HI20) used to be emitted for extern-via-GOT symbols
    * under -fPIE (the hosted default), but the linker has no layout/apply for
    * that reloc, so it produced a hard "unsupported reloc kind" link failure
    * rather than a working binary. */
   /* lui t0, %tprel_hi(sym); add t0, tp, t0; addi dst, t0, %tprel_lo(sym). */
   {
     u32 hp = mc_pos(mc);
     rv64_emit32(mc, rv_lui(RV_TMP0, 0));
     mc_emit_reloc_at(mc, sec, hp, R_RV_TPREL_HI20, sym, addend, 0, 0);
     rv64_emit32(mc, rv_add(RV_TMP0, RV_TP, RV_TMP0));
     {
       u32 lp = mc_pos(mc);
       rv64_emit32(mc, rv_addi(rd, RV_TMP0, 0));
       mc_emit_reloc_at(mc, sec, lp, R_RV_TPREL_LO12_I, sym, addend, 0, 0);
     }
   }
 }
 static void rv_bitfield_load(NativeTarget* t, NativeLoc dst, NativeAddr ra,
                              BitFieldAccess bf) {
   RvNativeTarget* a = rv_of(t);
   const RiscvVariant* v = a->variant;
   MCEmitter* mc = t->mc;
   u32 storage_bytes = bf.storage.size ? bf.storage.size : 4u;
   u32 rd = loc_reg(dst);
   u32 base;
   i32 off;
   u32 lsb = bf.bit_offset;
   u32 width = bf.bit_width ? bf.bit_width : 1u;
   /* Shift left so the field's MSB lands at the register top (XLEN-1), then
    * shift right to sign/zero extend it down. Shifts are XLEN-wide. */
   u32 sh_left = v->xlen - (lsb + width);
   u32 sh_right = v->xlen - width;
   ra.offset += (i32)bf.storage_offset;
   rv_resolve_mem_addr(a, &ra, &base, &off);
   rv64_emit32(mc, enc_int_load(v, storage_bytes, 0, rd, base, off));
   rv64_emit32(mc, rv_slli(rd, rd, sh_left));
   if (bf.signed_)
     rv64_emit32(mc, rv_srai(rd, rd, sh_right));
   else
     rv64_emit32(mc, rv_srli(rd, rd, sh_right));
 }
 static void rv_bitfield_store(NativeTarget* t, NativeAddr ra, NativeLoc src,
                               BitFieldAccess bf) {
   RvNativeTarget* a = rv_of(t);
   const RiscvVariant* v = a->variant;
   MCEmitter* mc = t->mc;
   u32 storage_bytes = bf.storage.size ? bf.storage.size : 4u;
   u32 src_reg = loc_reg(src);
   u32 base;
   i32 off;
   u32 lsb = bf.bit_offset;
   u32 width = bf.bit_width ? bf.bit_width : 1u;
   u64 ones = width >= 64u ? ~(u64)0 : (((u64)1 << width) - 1u);
   u64 mask_in = ones << lsb;
   ra.offset += (i32)bf.storage_offset;
   /* Resolve the field address; rv_resolve_mem_addr may use RV_TMP0/RV_TMP1, so
    * stabilize the base into RV_TMP1 before consuming the scratch temps. */
   rv_resolve_mem_addr(a, &ra, &base, &off);
   if (base != RV_S0 && base != RV_TMP1) {
     rv_emit_addr_adjust(v, mc, RV_TMP1, base, off);
     base = RV_TMP1;
     off = 0;
   } else if (base == RV_TMP1 && off != 0) {
     rv_emit_addr_adjust(v, mc, RV_TMP1, RV_TMP1, off);
     off = 0;
   }
   /* word in RV_TMP2; merged via RV_TMP0 (clear mask, then shifted src). */
   rv64_emit32(mc, enc_int_load(v, storage_bytes, 0, RV_TMP2, base, off));
   rv_emit_load_imm(v, mc, 1, RV_TMP0, (i64)~mask_in);
   rv64_emit32(mc, rv_and(RV_TMP2, RV_TMP2, RV_TMP0));
   rv_emit_load_imm(v, mc, 1, RV_TMP0, (i64)ones);
   rv64_emit32(mc, rv_and(RV_TMP0, src_reg, RV_TMP0));
   if (lsb) rv64_emit32(mc, rv_slli(RV_TMP0, RV_TMP0, lsb));
   rv64_emit32(mc, rv_or(RV_TMP2, RV_TMP2, RV_TMP0));
   rv64_emit32(mc, enc_int_store(v, storage_bytes, RV_TMP2, base, off));
 }
 static int rv_order_acquire(KitCgMemOrder o) {
   return o == KIT_CG_MO_CONSUME || o == KIT_CG_MO_ACQUIRE ||
          o == KIT_CG_MO_ACQ_REL || o == KIT_CG_MO_SEQ_CST;
 }
 static int rv_order_release(KitCgMemOrder o) {
   return o == KIT_CG_MO_RELEASE || o == KIT_CG_MO_ACQ_REL ||
          o == KIT_CG_MO_SEQ_CST;
 }
 
 /* Materialize the atomic operand address into RV_TMP0 (a bare pointer, since
  * LR/SC and AMO take a base register with no offset) and return it. */
 static u32 rv_atomic_addr_reg(RvNativeTarget* a, NativeAddr addr) {
   NativeLoc dst =
       native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT, RV_TMP0);
   rv_load_addr(&a->base, dst, addr);
   return RV_TMP0;
 }
 
 static void rv_atomic_load(NativeTarget* t, NativeLoc dst, NativeAddr addr,
                            MemAccess mem, KitCgMemOrder mo) {
   RvNativeTarget* a = rv_of(t);
   MCEmitter* mc = t->mc;
   u32 sf = (mem.size ? mem.size : loc_size32(t, dst)) == 8u ? 1u : 0u;
   u32 base = rv_atomic_addr_reg(a, addr);
   if (mo == KIT_CG_MO_SEQ_CST) rv64_emit32(mc, rv_fence_rw_rw());
   if (rv_order_acquire(mo)) {
     /* lr.w/d as an ordered load (aq=1). */
     rv64_emit32(mc, sf ? rv_lr_d(loc_reg(dst), base, 1, 0)
                        : rv_lr_w(loc_reg(dst), base, 1, 0));
   } else {
     rv64_emit32(mc, enc_int_load(a->variant,
                                  mem.size ? mem.size : loc_size32(t, dst), 0,
                                  loc_reg(dst), base, 0));
   }
 }
 
 static void rv_atomic_store(NativeTarget* t, NativeAddr addr, NativeLoc src,
                             MemAccess mem, KitCgMemOrder mo) {
   RvNativeTarget* a = rv_of(t);
   MCEmitter* mc = t->mc;
   u32 sz = mem.size ? mem.size : loc_size32(t, src);
   /* RV_TMP0 holds the address; never collides with src (an allocable reg). */
   u32 base = rv_atomic_addr_reg(a, addr);
   if (rv_order_release(mo)) rv64_emit32(mc, rv_fence_rw_rw());
   rv64_emit32(mc, enc_int_store(a->variant, sz, loc_reg(src), base, 0));
   if (mo == KIT_CG_MO_SEQ_CST) rv64_emit32(mc, rv_fence_rw_rw());
 }
 
 static void rv_atomic_rmw(NativeTarget* t, KitCgAtomicOp op, NativeLoc dst,
                           NativeAddr addr, NativeLoc val, MemAccess mem,
                           KitCgMemOrder mo) {
   RvNativeTarget* a = rv_of(t);
   const RiscvVariant* v = a->variant;
   MCEmitter* mc = t->mc;
   u32 sf = (mem.size ? mem.size : loc_size32(t, dst)) == 8u ? 1u : 0u;
   /* W-form add/sub apply only to a 32-bit value on rv64; on rv32 the base ops
    * are the 32-bit ops. */
   int w = !sf && v->has_w_forms;
   u32 base = rv_atomic_addr_reg(a, addr); /* RV_TMP0 */
   u32 vreg = loc_reg(val);
   u32 rd = loc_reg(dst);
   u32 aq = (u32)rv_order_acquire(mo);
   u32 rl = (u32)rv_order_release(mo);
   MCLabel retry = mc_label_new(mc);
   /* LR/SC loop: dst = *base; new = dst op val; sc new; retry on failure.
    * RV_TMP1 carries the SC status, RV_TMP3 the computed new value. */
   mc_label_place(mc, retry);
   rv64_emit32(mc, sf ? rv_lr_d(rd, base, aq, 0) : rv_lr_w(rd, base, aq, 0));
   switch (op) {
     case KIT_CG_ATOMIC_XCHG:
       rv64_emit32(mc, rv_addi(RV_TMP3, vreg, 0));
       break;
     case KIT_CG_ATOMIC_ADD:
       rv64_emit32(mc,
                   w ? rv_addw(RV_TMP3, rd, vreg) : rv_add(RV_TMP3, rd, vreg));
       break;
     case KIT_CG_ATOMIC_SUB:
       rv64_emit32(mc,
                   w ? rv_subw(RV_TMP3, rd, vreg) : rv_sub(RV_TMP3, rd, vreg));
       break;
     case KIT_CG_ATOMIC_AND:
       rv64_emit32(mc, rv_and(RV_TMP3, rd, vreg));
       break;
     case KIT_CG_ATOMIC_OR:
       rv64_emit32(mc, rv_or(RV_TMP3, rd, vreg));
       break;
     case KIT_CG_ATOMIC_XOR:
       rv64_emit32(mc, rv_xor(RV_TMP3, rd, vreg));
       break;
     case KIT_CG_ATOMIC_NAND:
       rv64_emit32(mc, rv_and(RV_TMP3, rd, vreg));
       rv64_emit32(mc, rv_xori(RV_TMP3, RV_TMP3, -1));
       break;
     default:
       rv_panic(a, "unsupported atomic rmw op");
   }
   rv64_emit32(mc, sf ? rv_sc_d(RV_TMP1, base, RV_TMP3, 0, rl)
                      : rv_sc_w(RV_TMP1, base, RV_TMP3, 0, rl));
   rv64_emit32(mc, rv_bne(RV_TMP1, RV_ZERO, 0));
   mc_emit_label_ref(mc, retry, R_RV_BRANCH, 4, 0);
 }
 
 static void rv_atomic_cas(NativeTarget* t, NativeLoc prior, NativeLoc ok,
                           NativeAddr addr, NativeLoc expected,
                           NativeLoc desired, MemAccess mem,
                           KitCgMemOrder success, KitCgMemOrder failure) {
   RvNativeTarget* a = rv_of(t);
   MCEmitter* mc = t->mc;
   u32 sf = (mem.size ? mem.size : loc_size32(t, prior)) == 8u ? 1u : 0u;
   u32 base = rv_atomic_addr_reg(a, addr); /* RV_TMP0 */
   u32 rprior = loc_reg(prior);
   u32 rexp = loc_reg(expected);
   u32 rdes = loc_reg(desired);
   u32 rok = loc_reg(ok);
   u32 aq = (u32)rv_order_acquire(success);
   u32 rl = (u32)rv_order_release(success);
   MCLabel retry = mc_label_new(mc);
   MCLabel fail = mc_label_new(mc);
   MCLabel done = mc_label_new(mc);
   (void)failure;
   mc_label_place(mc, retry);
   rv64_emit32(mc,
               sf ? rv_lr_d(rprior, base, aq, 0) : rv_lr_w(rprior, base, aq, 0));
   /* if (prior != expected) -> fail */
   rv64_emit32(mc, rv_bne(rprior, rexp, 0));
   mc_emit_label_ref(mc, fail, R_RV_BRANCH, 4, 0);
   /* sc.w/d status, desired, (base); retry on failure. */
   rv64_emit32(mc, sf ? rv_sc_d(RV_TMP1, base, rdes, 0, rl)
                      : rv_sc_w(RV_TMP1, base, rdes, 0, rl));
   rv64_emit32(mc, rv_bne(RV_TMP1, RV_ZERO, 0));
   mc_emit_label_ref(mc, retry, R_RV_BRANCH, 4, 0);
   /* ok = 1; jump done. */
   rv_emit_load_imm(a->variant, mc, 0, rok, 1);
   rv64_emit32(mc, rv_jal(RV_ZERO, 0));
   mc_emit_label_ref(mc, done, R_RV_JAL, 4, 0);
   mc_label_place(mc, fail);
   rv_emit_load_imm(a->variant, mc, 0, rok, 0);
   mc_label_place(mc, done);
 }
 
 static void rv_fence(NativeTarget* t, KitCgMemOrder mo) {
   if (mo == KIT_CG_MO_RELAXED) return;
   rv64_emit32(t->mc, rv_fence_rw_rw());
 }
 /* ---- variadics (LP64D ABI_VA_LIST_POINTER) ----
  * va_list is a single void* to the next argument slot. The prologue spilled
  * unconsumed a-regs into the 64-byte save area at [s0+16); incoming stack args
  * follow contiguously, so a uniform 8-byte stride covers both. `ap` is a
  * NativeAddr that addresses the va_list object itself. */
 
 static void rv_va_start_core(RvNativeTarget* a, NativeAddr ap) {
   NativeTarget* t = &a->base;
   const RiscvVariant* v = a->variant;
   MCEmitter* mc = t->mc;
   ABIVaListInfo vai = abi_va_list_layout(t->c->abi);
   KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
   u32 slot = vai.gp_slot_size ? vai.gp_slot_size : v->gp_slot_bytes;
   if (vai.kind != ABI_VA_LIST_POINTER)
     rv_panic(a, "unsupported va_list layout");
   if (!a->is_variadic) rv_panic(a, "va_start: function not variadic");
   /* *ap = s0 + frame_save + next_param_int*gp_slot (skip named-int slots). */
   rv64_emit32(
       mc, rv_addi(RV_TMP1, RV_S0,
                   (i32)v->frame_save_size + (i32)(a->next_param_int * slot)));
   rv_emit_mem(a, 0, native_loc_reg(i64t, NATIVE_REG_INT, RV_TMP1), ap,
               native_mem_for_type(t, i64t, v->ptr_bytes));
 }
 
 /* Wide / aggregate va_arg: a value too large for a single GPR (an 8-byte
  * i64 / soft-double on ilp32) occupies consecutive GP slots in the save area
  * and cannot move through one register. Read the cursor, advance it past the
  * whole span, then byte-copy the value from the (saved) cursor into the
  * destination memory. RV_TMP2 holds the cursor across the rv_copy_bytes call,
  * which itself uses RV_TMP0/RV_TMP1/RV_TMP3. */
 static void rv_va_arg_wide(RvNativeTarget* a, NativeAddr dst, NativeAddr ap,
                            u32 sz) {
   NativeTarget* t = &a->base;
   const RiscvVariant* v = a->variant;
   MCEmitter* mc = t->mc;
   KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
   ABIVaListInfo vai = abi_va_list_layout(t->c->abi);
   u32 slot = vai.gp_slot_size ? vai.gp_slot_size : v->gp_slot_bytes;
   u32 span = align_up_u32(sz, slot);
   NativeLoc cur = native_loc_reg(i64t, NATIVE_REG_INT, RV_TMP2);
   NativeLoc nxt = native_loc_reg(i64t, NATIVE_REG_INT, RV_TMP1);
   NativeAddr src;
   AggregateAccess acc;
   /* cur = *ap; *ap = cur + span. */
   rv_emit_mem(a, 1, cur, ap, native_mem_for_type(t, i64t, v->ptr_bytes));
   rv64_emit32(mc, rv_addi(RV_TMP1, RV_TMP2, (i32)span));
   rv_emit_mem(a, 0, nxt, ap, native_mem_for_type(t, i64t, v->ptr_bytes));
   /* Copy sz bytes from [cur] to the destination. */
   memset(&src, 0, sizeof src);
   src.base_kind = NATIVE_ADDR_BASE_REG;
   src.base.reg = RV_TMP2;
   src.base_type = i64t;
   memset(&acc, 0, sizeof acc);
   acc.type = i64t;
   acc.size = sz;
   acc.align = slot;
   rv_copy_bytes(t, dst, src, acc);
 }
 
 static void rv_va_arg_core(RvNativeTarget* a, NativeLoc dst, NativeAddr ap,
                            KitCgTypeId type) {
   NativeTarget* t = &a->base;
   const RiscvVariant* v = a->variant;
   MCEmitter* mc = t->mc;
   ABIVaListInfo vai = abi_va_list_layout(t->c->abi);
   KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
   u32 sz = native_type_size(t, type);
   u32 slot = vai.gp_slot_size ? vai.gp_slot_size : v->gp_slot_bytes;
   NativeLoc cur = native_loc_reg(i64t, NATIVE_REG_INT, RV_TMP1);
   NativeAddr from;
   if (vai.kind != ABI_VA_LIST_POINTER)
     rv_panic(a, "unsupported va_list layout");
   if (dst.kind != NATIVE_LOC_REG) rv_panic(a, "va_arg destination must be reg");
   /* cur = *ap; load value from [cur]; *ap = cur + slot (one GP-slot stride). */
   rv_emit_mem(a, 1, cur, ap, native_mem_for_type(t, i64t, v->ptr_bytes));
   memset(&from, 0, sizeof from);
   from.base_kind = NATIVE_ADDR_BASE_REG;
   from.base.reg = RV_TMP1;
   from.base_type = type;
   if (native_loc_is_fp(dst)) {
     /* Variadic FP args sit in the integer save area as their bit pattern;
      * load into RV_TMP2 and bitcast into the FPR. The fmv_d_x (double) path is
      * RV64-only — on rv32 doubles are passed soft and never reach here. */
     NativeLoc itmp = native_loc_reg(type, NATIVE_REG_INT, RV_TMP2);
     rv_emit_mem(a, 1, itmp, from, native_mem_for_type(t, type, sz));
     rv64_emit32(mc, sz == 8u ? rv_fmv_d_x(loc_reg(dst), RV_TMP2)
                              : rv_fmv_w_x(loc_reg(dst), RV_TMP2));
   } else {
     rv_emit_mem(a, 1, dst, from, native_mem_for_type(t, type, sz));
   }
   rv64_emit32(mc, rv_addi(RV_TMP1, RV_TMP1, (i32)slot));
   rv_emit_mem(a, 0, cur, ap, native_mem_for_type(t, i64t, v->ptr_bytes));
 }
 
 static void rv_va_copy_core(RvNativeTarget* a, NativeAddr dst_ap,
                             NativeAddr src_ap) {
   NativeTarget* t = &a->base;
   u32 ptr = a->variant->ptr_bytes;
   KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
   NativeLoc tmp = native_loc_reg(i64t, NATIVE_REG_INT, RV_TMP1);
   /* va_list is a single pointer-width slot. */
   rv_emit_mem(a, 1, tmp, src_ap, native_mem_for_type(t, i64t, ptr));
   rv_emit_mem(a, 0, tmp, dst_ap, native_mem_for_type(t, i64t, ptr));
 }
 
 static NativeAddr rv_va_addr_from_ptr(NativeLoc ap_ptr) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_REG;
   addr.cls = NATIVE_REG_INT;
   addr.base.reg = ap_ptr.v.reg;
   addr.base_type = ap_ptr.type;
   return addr;
 }
 
 static void rv_va_start_native(NativeTarget* t, NativeLoc ap_ptr) {
   rv_va_start_core(rv_of(t), rv_va_addr_from_ptr(ap_ptr));
 }
 /* A scalar whose value cannot move through one GPR (size > GPR width, e.g. an
  * 8-byte i64 / soft-double on ilp32). pass_native_emit hands such a va_arg its
  * memory destination directly rather than a scratch register. */
 static int rv_va_arg_is_wide(NativeTarget* t, KitCgTypeId type) {
   return native_type_size(t, type) > rv_of(t)->variant->ptr_bytes;
 }
 
 static void rv_va_arg_native(NativeTarget* t, NativeLoc dst, NativeLoc ap_ptr,
                              KitCgTypeId type) {
   RvNativeTarget* a = rv_of(t);
   if (rv_va_arg_is_wide(t, type)) {
     rv_va_arg_wide(a, rv_storage_addr(a, dst, 0), rv_va_addr_from_ptr(ap_ptr),
                    native_type_size(t, type));
     return;
   }
   rv_va_arg_core(a, dst, rv_va_addr_from_ptr(ap_ptr), type);
 }
 static void rv_va_end_native(NativeTarget* t, NativeLoc ap_ptr) {
   (void)t;
   (void)ap_ptr;
 }
 static void rv_va_copy_native(NativeTarget* t, NativeLoc dst, NativeLoc src) {
   rv_va_copy_core(rv_of(t), rv_va_addr_from_ptr(dst), rv_va_addr_from_ptr(src));
 }
 /* Software popcount of RV_TMP1 (already width-normalized) into rd, using
  * RV_TMP1/RV_TMP2/RV_TMP3 as scratch. Mirrors the legacy bit-twiddling. On rv32
  * only the 32-bit (is64==0) path is reachable for a single register. */
 static void rv_emit_popcount(const RiscvVariant* v, MCEmitter* mc, u32 rd,
                              int is64) {
   rv64_emit32(mc, rv_srli(RV_TMP2, RV_TMP1, 1));
   rv_emit_load_imm(v, mc, 1, RV_TMP3,
                    is64 ? (i64)0x5555555555555555ll : (i64)0x55555555);
   rv64_emit32(mc, rv_and(RV_TMP2, RV_TMP2, RV_TMP3));
   rv64_emit32(mc, rv_sub(RV_TMP1, RV_TMP1, RV_TMP2));
   rv_emit_load_imm(v, mc, 1, RV_TMP3,
                    is64 ? (i64)0x3333333333333333ll : (i64)0x33333333);
   rv64_emit32(mc, rv_and(RV_TMP2, RV_TMP1, RV_TMP3));
   rv64_emit32(mc, rv_srli(RV_TMP1, RV_TMP1, 2));
   rv64_emit32(mc, rv_and(RV_TMP1, RV_TMP1, RV_TMP3));
   rv64_emit32(mc, rv_add(RV_TMP1, RV_TMP1, RV_TMP2));
   rv64_emit32(mc, rv_srli(RV_TMP2, RV_TMP1, 4));
   rv64_emit32(mc, rv_add(RV_TMP1, RV_TMP1, RV_TMP2));
   rv_emit_load_imm(v, mc, 1, RV_TMP3,
                    is64 ? (i64)0x0f0f0f0f0f0f0f0fll : (i64)0x0f0f0f0f);
   rv64_emit32(mc, rv_and(RV_TMP1, RV_TMP1, RV_TMP3));
   rv_emit_load_imm(v, mc, 1, RV_TMP3,
                    is64 ? (i64)0x0101010101010101ll : (i64)0x01010101);
   rv64_emit32(mc, rv_mul(RV_TMP1, RV_TMP1, RV_TMP3));
   rv64_emit32(mc, rv_srli(rd, RV_TMP1, is64 ? 56u : 24u));
   /* The 32-bit SWAR sum lives in product bits [24,32); since the multiply is
    * 64-bit, bits [32,64) survive the >>24 and must be masked off. (The 64-bit
    * path's >>56 already isolates the top byte, so it needs no mask.) */
   if (!is64) rv64_emit32(mc, rv_andi(rd, rd, 0xff));
 }
 
 /* Inline byte-granule copy/set between bare base registers (memcpy/memmove/
  * memset intrinsics). dir<0 copies high-to-low (memmove backward). The 8-byte
  * granule (ld/sd) and zero-extending lwu are RV64-only; on rv32 the widest
  * granule is 4 bytes via lw/sw. */
 static void rv_intrin_copy(const RiscvVariant* v, MCEmitter* mc, u32 dr, u32 sr,
                            u32 n, int backward) {
   int wide = v->ptr_bytes == 8u;
   if (!backward) {
     u32 i = 0;
     while (wide && i + 8u <= n) {
       rv64_emit32(mc, rv_ld(RV_TMP3, sr, (i32)i));
       rv64_emit32(mc, rv_sd(RV_TMP3, dr, (i32)i));
       i += 8u;
     }
     while (i + 4u <= n) {
       rv64_emit32(
           mc, wide ? rv_lwu(RV_TMP3, sr, (i32)i) : rv_lw(RV_TMP3, sr, (i32)i));
       rv64_emit32(mc, rv_sw(RV_TMP3, dr, (i32)i));
       i += 4u;
     }
     while (i + 2u <= n) {
       rv64_emit32(mc, rv_lhu(RV_TMP3, sr, (i32)i));
       rv64_emit32(mc, rv_sh(RV_TMP3, dr, (i32)i));
       i += 2u;
     }
     while (i < n) {
       rv64_emit32(mc, rv_lbu(RV_TMP3, sr, (i32)i));
       rv64_emit32(mc, rv_sb(RV_TMP3, dr, (i32)i));
       i += 1u;
     }
   } else {
     u32 i = n;
     while (wide && i >= 8u) {
       i -= 8u;
       rv64_emit32(mc, rv_ld(RV_TMP3, sr, (i32)i));
       rv64_emit32(mc, rv_sd(RV_TMP3, dr, (i32)i));
     }
     while (i >= 4u) {
       i -= 4u;
       rv64_emit32(
           mc, wide ? rv_lwu(RV_TMP3, sr, (i32)i) : rv_lw(RV_TMP3, sr, (i32)i));
       rv64_emit32(mc, rv_sw(RV_TMP3, dr, (i32)i));
     }
     while (i >= 2u) {
       i -= 2u;
       rv64_emit32(mc, rv_lhu(RV_TMP3, sr, (i32)i));
       rv64_emit32(mc, rv_sh(RV_TMP3, dr, (i32)i));
     }
     while (i >= 1u) {
       i -= 1u;
       rv64_emit32(mc, rv_lbu(RV_TMP3, sr, (i32)i));
       rv64_emit32(mc, rv_sb(RV_TMP3, dr, (i32)i));
     }
   }
 }
 
 static void rv_intrinsic(NativeTarget* t, IntrinKind kind,
                          const NativeLoc* dsts, u32 ndst, const NativeLoc* args,
                          u32 narg) {
   RvNativeTarget* a = rv_of(t);
   const RiscvVariant* v = a->variant;
   MCEmitter* mc = t->mc;
   (void)ndst;
   (void)narg;
   switch (kind) {
     case INTRIN_NONE:
       break;
     case INTRIN_EXPECT:
     case INTRIN_ASSUME_ALIGNED: {
       /* dst = val (hint dropped). */
       if (args[0].kind == NATIVE_LOC_IMM)
         rv_emit_load_imm(v, mc, rv_is_64(t, dsts[0].type) ? 1u : 0u,
                          loc_reg(dsts[0]), args[0].v.imm);
       else
         rv_move(t, dsts[0], args[0]);
       return;
     }
     case INTRIN_PREFETCH:
       return;
     case INTRIN_TRAP:
       rv64_emit32(mc, rv_ebreak());
       return;
     case INTRIN_READCYCLECOUNTER:
       /* RDCYCLE rd = csrrs rd, cycle, x0 — reads the 64-bit cycle CSR on rv64
        * (rv32 is gated out in rv64_supports_intrinsic). */
       rv64_emit32(mc, rv_csrrs(loc_reg(dsts[0]), RV_CSR_CYCLE, RV_ZERO));
       return;
     case INTRIN_SYSCALL:
       if (ndst == 1u && narg >= 1u && narg <= 7u) {
         static const u32 syscall_regs[7] = {RV_A7, RV_A0, RV_A1, RV_A2,
                                             RV_A3, RV_A4, RV_A5};
         RvArgMove moves[7];
         for (u32 i = 0; i < narg; ++i) {
           RvArgMove* m = &moves[i];
           memset(m, 0, sizeof *m);
           m->dst =
               native_loc_reg(dsts[0].type, NATIVE_REG_INT, syscall_regs[i]);
           m->src = args[i];
           m->size = t->c->target.ptr_size;
         }
         rv_emit_reg_arg_moves(t, moves, narg);
         rv64_emit32(mc, rv_ecall());
         rv_move(t, dsts[0],
                 native_loc_reg(dsts[0].type, NATIVE_REG_INT, RV_A0));
       }
       return;
     case INTRIN_BSWAP: {
       u32 width = abi_cg_sizeof(t->c->abi, dsts[0].type);
       switch (width) {
         case 2: {
           u32 rd = loc_reg(dsts[0]), rs = loc_reg(args[0]);
           /* rd = ((rs & 0xff) << 8) | ((rs >> 8) & 0xff). */
           rv64_emit32(mc, rv_addi(RV_TMP2, RV_ZERO, 0xff));
           rv64_emit32(mc, rv_slli(RV_TMP2, RV_TMP2, 8)); /* 0xff00 */
           rv64_emit32(mc, rv_slli(RV_TMP1, rs, 8));
           rv64_emit32(mc, rv_and(RV_TMP1, RV_TMP1, RV_TMP2));
           rv64_emit32(mc, rv_srli(RV_TMP3, rs, 8));
           rv64_emit32(mc, rv_andi(RV_TMP3, RV_TMP3, 0xff));
           rv64_emit32(mc, rv_or(rd, RV_TMP1, RV_TMP3));
           return;
         }
         case 4: {
           u32 rd = loc_reg(dsts[0]), rs = loc_reg(args[0]);
           /* SRLIW is RV64-only; on rv32 SRLI on a 32-bit reg is equivalent. */
           int w = v->has_w_forms;
           rv64_emit32(mc,
                       w ? rv_srliw(RV_TMP1, rs, 24) : rv_srli(RV_TMP1, rs, 24));
           rv64_emit32(mc, rv_andi(RV_TMP1, RV_TMP1, 0xff));
           rv64_emit32(mc,
                       w ? rv_srliw(RV_TMP2, rs, 16) : rv_srli(RV_TMP2, rs, 16));
           rv64_emit32(mc, rv_andi(RV_TMP2, RV_TMP2, 0xff));
           rv64_emit32(mc, rv_slli(RV_TMP2, RV_TMP2, 8));
           rv64_emit32(mc, rv_or(RV_TMP1, RV_TMP1, RV_TMP2));
           rv64_emit32(mc,
                       w ? rv_srliw(RV_TMP2, rs, 8) : rv_srli(RV_TMP2, rs, 8));
           rv64_emit32(mc, rv_andi(RV_TMP2, RV_TMP2, 0xff));
           rv64_emit32(mc, rv_slli(RV_TMP2, RV_TMP2, 16));
           rv64_emit32(mc, rv_or(RV_TMP1, RV_TMP1, RV_TMP2));
           rv64_emit32(mc, rv_andi(RV_TMP2, rs, 0xff));
           rv64_emit32(mc, rv_slli(RV_TMP2, RV_TMP2, 24));
           rv64_emit32(mc, rv_or(rd, RV_TMP1, RV_TMP2));
           /* Canonicalize to a 32-bit value in a 64-bit reg (RV64 only); on rv32
            * the result already occupies the whole register. */
           if (w) {
             rv64_emit32(mc, rv_slli(rd, rd, 32));
             rv64_emit32(mc, rv_srli(rd, rd, 32));
           }
           return;
         }
         case 8: {
           u32 rd = loc_reg(dsts[0]), rs = loc_reg(args[0]);
           int i;
           rv64_emit32(mc, rv_addi(RV_TMP1, RV_ZERO, 0));
           for (i = 0; i < 8; ++i) {
             int sh = 56 - 8 * i;
             if (i == 0) {
               rv64_emit32(mc, rv_andi(RV_TMP2, rs, 0xff));
             } else {
               rv64_emit32(mc, rv_srli(RV_TMP2, rs, (u32)(8 * i)));
               rv64_emit32(mc, rv_andi(RV_TMP2, RV_TMP2, 0xff));
             }
             if (sh) rv64_emit32(mc, rv_slli(RV_TMP2, RV_TMP2, (u32)sh));
             rv64_emit32(mc, rv_or(RV_TMP1, RV_TMP1, RV_TMP2));
           }
           rv64_emit32(mc, rv_addi(rd, RV_TMP1, 0));
           return;
         }
         default:
           break;
       }
       return;
     }
     case INTRIN_POPCOUNT: {
       u32 rd = loc_reg(dsts[0]), rs = loc_reg(args[0]);
       int is64 = rv_is_64(t, args[0].type);
       /* The narrow-in-wide normalization clears the high 32 bits of a 64-bit
        * reg; on rv32 there are none, so it is skipped. */
       int nrm = !is64 && v->xlen == 64u;
       rv64_emit32(mc, rv_addi(RV_TMP1, rs, 0));
       if (nrm) {
         rv64_emit32(mc, rv_slli(RV_TMP1, RV_TMP1, 32));
         rv64_emit32(mc, rv_srli(RV_TMP1, RV_TMP1, 32));
       }
       rv_emit_popcount(v, mc, rd, is64);
       return;
     }
     case INTRIN_CTZ: {
       /* ctz(x) = popcount((x & -x) - 1) for x != 0. */
       u32 rd = loc_reg(dsts[0]), rs = loc_reg(args[0]);
       int is64 = rv_is_64(t, args[0].type);
       int nrm = !is64 && v->xlen == 64u;
       rv64_emit32(mc, rv_sub(RV_TMP1, RV_ZERO, rs));
       rv64_emit32(mc, rv_and(RV_TMP1, RV_TMP1, rs));
       rv64_emit32(mc, rv_addi(RV_TMP1, RV_TMP1, -1));
       if (nrm) {
         rv64_emit32(mc, rv_slli(RV_TMP1, RV_TMP1, 32));
         rv64_emit32(mc, rv_srli(RV_TMP1, RV_TMP1, 32));
       }
       rv_emit_popcount(v, mc, rd, is64);
       return;
     }
     case INTRIN_CLZ: {
       /* Fold the high bit downward, then clz = popcount(~folded). */
       u32 rd = loc_reg(dsts[0]), rs = loc_reg(args[0]);
       int is64 = rv_is_64(t, args[0].type);
       int nrm = !is64 && v->xlen == 64u;
       u32 shifts[6] = {1, 2, 4, 8, 16, 32};
       u32 ns = is64 ? 6u : 5u, i;
       rv64_emit32(mc, rv_addi(RV_TMP1, rs, 0));
       if (nrm) {
         rv64_emit32(mc, rv_slli(RV_TMP1, RV_TMP1, 32));
         rv64_emit32(mc, rv_srli(RV_TMP1, RV_TMP1, 32));
       }
       for (i = 0; i < ns; ++i) {
         rv64_emit32(mc, rv_srli(RV_TMP2, RV_TMP1, shifts[i]));
         rv64_emit32(mc, rv_or(RV_TMP1, RV_TMP1, RV_TMP2));
       }
       rv64_emit32(mc, rv_xori(RV_TMP1, RV_TMP1, -1));
       if (nrm) {
         rv64_emit32(mc, rv_slli(RV_TMP1, RV_TMP1, 32));
         rv64_emit32(mc, rv_srli(RV_TMP1, RV_TMP1, 32));
       }
       rv_emit_popcount(v, mc, rd, is64);
       return;
     }
     case INTRIN_SADD_OVERFLOW:
     case INTRIN_SSUB_OVERFLOW: {
       /* dsts: [val, ovf]. ADD: ovf=((a^r)&(b^r))>>(w-1);
        * SUB: ovf=((a^b)&(a^r))>>(w-1). */
       int is64 = rv_is_64(t, dsts[0].type);
       int w = !is64 && v->has_w_forms; /* narrow op on rv64 -> W-form */
       u32 ra = loc_reg(args[0]), rb = loc_reg(args[1]);
       u32 rd = loc_reg(dsts[0]), rovf = loc_reg(dsts[1]);
       u32 sh = is64 ? 63u : 31u;
       if (kind == INTRIN_SADD_OVERFLOW)
         rv64_emit32(mc, w ? rv_addw(RV_TMP2, ra, rb) : rv_add(RV_TMP2, ra, rb));
       else
         rv64_emit32(mc, w ? rv_subw(RV_TMP2, ra, rb) : rv_sub(RV_TMP2, ra, rb));
       rv64_emit32(mc, rv_xor(RV_TMP3, ra, RV_TMP2)); /* a ^ r */
       if (kind == INTRIN_SADD_OVERFLOW) {
         rv64_emit32(mc, rv_xor(rovf, rb, RV_TMP2)); /* b ^ r */
         rv64_emit32(mc, rv_and(rovf, rovf, RV_TMP3));
       } else {
         rv64_emit32(mc, rv_xor(rovf, ra, rb)); /* a ^ b */
         rv64_emit32(mc, rv_and(rovf, rovf, RV_TMP3));
       }
       rv64_emit32(mc, w ? rv_srliw(rovf, rovf, sh) : rv_srli(rovf, rovf, sh));
       rv64_emit32(mc, rv_andi(rovf, rovf, 1));
       rv64_emit32(mc, rv_addi(rd, RV_TMP2, 0));
       return;
     }
     case INTRIN_UADD_OVERFLOW:
     case INTRIN_USUB_OVERFLOW: {
       int is64 = rv_is_64(t, dsts[0].type);
       /* `single`: the value fills the whole native register (rv64 i64 or any
        * rv32 value), so the native carry/borrow sequence applies directly; the
        * `!single` branch is the rv64 32-bit-in-64-bit-register implementation
        * (zero-extend + srli-32), reachable only on rv64. */
       int single = is64 || v->xlen == 32u;
       u32 ra = loc_reg(args[0]), rb = loc_reg(args[1]);
       u32 rd = loc_reg(dsts[0]), rovf = loc_reg(dsts[1]);
       if (!single) {
         rv64_emit32(mc, rv_slli(RV_TMP2, ra, 32));
         rv64_emit32(mc, rv_srli(RV_TMP2, RV_TMP2, 32));
         rv64_emit32(mc, rv_slli(RV_TMP3, rb, 32));
         rv64_emit32(mc, rv_srli(RV_TMP3, RV_TMP3, 32));
         ra = RV_TMP2;
         rb = RV_TMP3;
       }
       if (kind == INTRIN_UADD_OVERFLOW) {
         if (single) {
           rv64_emit32(mc, rv_add(RV_TMP2, ra, rb));
           rv64_emit32(mc, rv_sltu(rovf, RV_TMP2, ra));
         } else {
           rv64_emit32(mc, rv_add(RV_TMP2, ra, rb));
           rv64_emit32(mc, rv_srli(rovf, RV_TMP2, 32));
           rv64_emit32(mc, rv_sltu(rovf, RV_ZERO, rovf));
           rv64_emit32(mc, rv_addiw(RV_TMP2, RV_TMP2, 0));
         }
       } else {
         rv64_emit32(mc, rv_sltu(rovf, ra, rb));
         rv64_emit32(
             mc, single ? rv_sub(RV_TMP2, ra, rb) : rv_subw(RV_TMP2, ra, rb));
       }
       rv64_emit32(mc, rv_addi(rd, RV_TMP2, 0));
       return;
     }
     case INTRIN_SMUL_OVERFLOW: {
       int is64 = rv_is_64(t, dsts[0].type);
       /* `single`: native-width product overflow via MUL + MULH and a sign-bit
        * compare (shift xlen-1). rv64 i64 and any rv32 value take this path; the
        * `!single` branch is the rv64 32-bit-in-64-bit-register sequence. */
       int single = is64 || v->xlen == 32u;
       u32 sh = is64 ? 63u : 31u;
       u32 ra = loc_reg(args[0]), rb = loc_reg(args[1]);
       u32 rd = loc_reg(dsts[0]), rovf = loc_reg(dsts[1]);
       if (single) {
         rv64_emit32(mc, rv_mul(RV_TMP2, ra, rb));
         rv64_emit32(mc, rv_mulh(RV_TMP3, ra, rb));
         rv64_emit32(mc, rv_srai(rovf, RV_TMP2, sh));
         rv64_emit32(mc, rv_xor(rovf, RV_TMP3, rovf));
         rv64_emit32(mc, rv_sltu(rovf, RV_ZERO, rovf));
         rv64_emit32(mc, rv_addi(rd, RV_TMP2, 0));
       } else {
         rv64_emit32(mc, rv_addiw(RV_TMP2, ra, 0));
         rv64_emit32(mc, rv_addiw(RV_TMP3, rb, 0));
         rv64_emit32(mc, rv_mul(RV_TMP2, RV_TMP2, RV_TMP3));
         rv64_emit32(mc, rv_addiw(RV_TMP3, RV_TMP2, 0));
         rv64_emit32(mc, rv_xor(rovf, RV_TMP2, RV_TMP3));
         rv64_emit32(mc, rv_sltu(rovf, RV_ZERO, rovf));
         rv64_emit32(mc, rv_addiw(rd, RV_TMP2, 0));
       }
       return;
     }
     case INTRIN_UMUL_OVERFLOW: {
       int is64 = rv_is_64(t, dsts[0].type);
       /* `single`: native-width product, overflow = (high word != 0) via MULHU.
        * rv64 i64 and any rv32 value take this path; `!single` is the rv64
        * 32-bit-in-64-bit-register sequence. */
       int single = is64 || v->xlen == 32u;
       u32 ra = loc_reg(args[0]), rb = loc_reg(args[1]);
       u32 rd = loc_reg(dsts[0]), rovf = loc_reg(dsts[1]);
       if (single) {
         rv64_emit32(mc, rv_mulhu(rovf, ra, rb));
         rv64_emit32(mc, rv_mul(rd, ra, rb));
         rv64_emit32(mc, rv_sltu(rovf, RV_ZERO, rovf));
       } else {
         rv64_emit32(mc, rv_slli(RV_TMP2, ra, 32));
         rv64_emit32(mc, rv_srli(RV_TMP2, RV_TMP2, 32));
         rv64_emit32(mc, rv_slli(RV_TMP3, rb, 32));
         rv64_emit32(mc, rv_srli(RV_TMP3, RV_TMP3, 32));
         rv64_emit32(mc, rv_mul(RV_TMP2, RV_TMP2, RV_TMP3));
         rv64_emit32(mc, rv_srli(rovf, RV_TMP2, 32));
         rv64_emit32(mc, rv_sltu(rovf, RV_ZERO, rovf));
         rv64_emit32(mc, rv_addiw(rd, RV_TMP2, 0));
       }
       return;
     }
     case INTRIN_SMUL_HIGH:
     case INTRIN_UMUL_HIGH: {
       int is64 = rv_is_64(t, dsts[0].type);
       int single = is64 || v->xlen == 32u;
       u32 ra = loc_reg(args[0]), rb = loc_reg(args[1]);
       u32 rd = loc_reg(dsts[0]);
       if (single) {
         rv64_emit32(mc, kind == INTRIN_SMUL_HIGH ? rv_mulh(rd, ra, rb)
                                                  : rv_mulhu(rd, ra, rb));
       } else if (kind == INTRIN_SMUL_HIGH) {
         rv64_emit32(mc, rv_addiw(RV_TMP2, ra, 0));
         rv64_emit32(mc, rv_addiw(RV_TMP3, rb, 0));
         rv64_emit32(mc, rv_mul(RV_TMP2, RV_TMP2, RV_TMP3));
         rv64_emit32(mc, rv_srai(rd, RV_TMP2, 32));
       } else {
         rv64_emit32(mc, rv_slli(RV_TMP2, ra, 32));
         rv64_emit32(mc, rv_srli(RV_TMP2, RV_TMP2, 32));
         rv64_emit32(mc, rv_slli(RV_TMP3, rb, 32));
         rv64_emit32(mc, rv_srli(RV_TMP3, RV_TMP3, 32));
         rv64_emit32(mc, rv_mul(RV_TMP2, RV_TMP2, RV_TMP3));
         rv64_emit32(mc, rv_srli(rd, RV_TMP2, 32));
       }
       return;
     }
     case INTRIN_MEMMOVE: {
       u32 dr, sr, n;
       if (narg != 3u || args[0].kind != NATIVE_LOC_REG ||
           args[1].kind != NATIVE_LOC_REG || args[2].kind != NATIVE_LOC_IMM)
         rv_panic(a, "unsupported memory intrinsic operands");
       if (args[2].v.imm < 0 || args[2].v.imm > 0xffffffffll)
         rv_panic(a, "unsupported memory intrinsic size");
       dr = loc_reg(args[0]);
       sr = loc_reg(args[1]);
       n = (u32)args[2].v.imm;
       rv_intrin_copy(v, mc, dr, sr, n, /*reverse (overlap-safe)=*/1);
       return;
     }
     case INTRIN_CPU_NOP:
       rv64_emit32(mc, rv_nop());
       return;
     case INTRIN_CPU_YIELD:
       rv64_emit32(mc, rv_pause());
       return;
     case INTRIN_ISB:
       rv64_emit32(mc, rv_fence_i());
       return;
     case INTRIN_DMB:
     case INTRIN_DSB:
       rv64_emit32(mc, rv_fence_rw_rw());
       return;
     case INTRIN_WFI:
       rv64_emit32(mc, rv_wfi());
       return;
     case INTRIN_FRAME_ADDRESS:
     case INTRIN_RETURN_ADDRESS:
       /* Walk the s0 frame-record chain. kit's RISC-V prologue anchors s0 at the
        * saved pair: [s0] = caller's s0, [s0 + ptr_bytes] = saved ra (this
        * frame's return address). NOTE: this differs from the psABI's
        * ra@s0-8 / fp@s0-16 layout — kit stores the pair at and above s0. A
        * function that reads its frame is forced off the frameless-leaf tier
        * (see NativeKnownFrameDesc.reads_frame), so s0 is always valid here. The
        * level is constant, so the walk unrolls to `level` dependent loads. */
       if (ndst == 1u) {
         u32 level = (narg >= 1u && args[0].kind == NATIVE_LOC_IMM)
                         ? (u32)args[0].v.imm
                         : 0u;
         u32 rd = loc_reg(dsts[0]);
         rv64_emit32(mc, rv_addi(rd, RV_S0, 0)); /* rd = s0 */
         for (u32 i = 0; i < level; ++i)
           rv64_emit32(mc, rv_ld_ptr(v, rd, rd, 0)); /* rd = *(rd) */
         if (kind == INTRIN_RETURN_ADDRESS)
           rv64_emit32(mc, rv_ld_ptr(v, rd, rd, (i32)v->ptr_bytes));
       }
       return;
     default:
       break;
   }
   rv_panic(a, "unsupported compiler intrinsic");
 }
 
 /* Optimized-hook fixed effects. RISC-V arithmetic, bitfields, aggregate
  * memory operations, atomics, varargs and Local-Exec TLS use only declared
  * operands/results plus the reserved t0..t3 / ft0..ft3 backend temporaries.
  * A syscall is the one exception: argument placement writes the Linux syscall
  * ABI registers even though the semantic IR operands are ordinary virtual
  * values. Keep live-across allocations out of those fixed destinations. */
 static int rv_machine_op_clobbers(NativeTarget* t, const NativeMachineOp* op,
                                   u32 mask[NATIVE_REG_CLASS_COUNT]) {
   mask[0] = mask[1] = mask[2] = 0;
   switch ((NativeMachineOpKind)op->kind) {
     case NATIVE_MOP_INTRINSIC:
       if ((IntrinKind)op->intrin != INTRIN_SYSCALL) return 0;
       /* a7 = syscall number; a0..a5 = arguments / a0 result. */
       mask[NATIVE_REG_INT] = (1u << RV_A0) | (1u << RV_A1) | (1u << RV_A2) |
                              (1u << RV_A3) | (1u << RV_A4) | (1u << RV_A5) |
                              (1u << RV_A7);
       return 1;
     case NATIVE_MOP_BINOP:
     case NATIVE_MOP_BITFIELD_LOAD:
     case NATIVE_MOP_BITFIELD_STORE:
     case NATIVE_MOP_VA_START:
     case NATIVE_MOP_VA_ARG:
     case NATIVE_MOP_ATOMIC_CAS:
     case NATIVE_MOP_ATOMIC_RMW:
     case NATIVE_MOP_TLS_ADDR:
       return 0;
     case NATIVE_MOP_COUNT:
       break;
   }
   rv_panic(rv_of(t), "invalid machine-effect operation");
   return 0;
 }
 
 /* ============================ inline asm ============================ */
 
 _Noreturn static void rv_asm_panic_at(Compiler* c, SrcLoc loc,
                                       const char* msg) {
   compiler_panic(c, loc, "rv64 inline asm: %s", msg);
 }
 _Noreturn static void rv_asm_panic(NativeDirectTarget* d, const char* msg) {
   rv_asm_panic_at(d->base.c, d->loc, msg);
 }
 
 /* constraint_body / constraint_early / match_index are shared
  * (cg/native_asm.h). */
 
 /* Build a bound register pseudo-operand in the rv64 inline shape. */
 static void rv_asm_bound_reg(Operand* out, KitCgTypeId type,
                              NativeAllocClass cls, Reg reg) {
   memset(out, 0, sizeof *out);
   out->kind = RV64_INLINE_OPK_REG;
   out->pad[0] =
       (cls == NATIVE_REG_FP) ? RV64_INLINE_OPCLS_FP : RV64_INLINE_OPCLS_INT;
   out->type = type;
   out->v.local = (CGLocal)reg;
 }
 static void rv_asm_bound_mem(Operand* out, KitCgTypeId type, Reg base) {
   memset(out, 0, sizeof *out);
   out->kind = OPK_INDIRECT;
   out->type = type;
   out->v.ind.base = (CGLocal)base;
   out->v.ind.index = CG_LOCAL_NONE;
   out->v.ind.ofs = 0;
 }
 
 /* Parse a clobber register name into (class, reg). Returns 0 for the special
  * "cc"/"memory" clobbers and panics on an unknown register. RV64 dwarf: int
  * x0..x31 = 0..31, fp f0..f31 = 32..63. */
 static int rv_asm_parse_reg_clobber(Compiler* c, SrcLoc loc, Sym name,
                                     NativeAllocClass* cls_out, Reg* reg_out) {
   Slice s = pool_slice(c->global, name);
   char buf[16];
   uint32_t dwarf;
   if (!s.s || !s.len) return 0;
   if (s.len == 2 && s.s[0] == 'c' && s.s[1] == 'c') return 0;
   if (s.len == 6 && memcmp(s.s, "memory", 6) == 0) return 0;
   if (s.len >= sizeof buf) rv_asm_panic_at(c, loc, "clobber name is too long");
   memcpy(buf, s.s, s.len);
   buf[s.len] = '\0';
   if (rv64_register_index(buf, &dwarf) != 0)
     rv_asm_panic_at(c, loc, "unknown clobber register");
   if (dwarf <= 31u) {
     *cls_out = NATIVE_REG_INT;
     *reg_out = (Reg)dwarf;
     return 1;
   }
   if (dwarf >= 32u && dwarf <= 63u) {
     *cls_out = NATIVE_REG_FP;
     *reg_out = (Reg)(dwarf - 32u);
     return 1;
   }
   rv_asm_panic_at(c, loc, "unsupported clobber register");
   return 0;
 }
 
 static void rv_asm_clobber_masks(Compiler* c, SrcLoc loc, const Sym* clobbers,
                                  u32 nclob, u32* int_mask, u32* fp_mask) {
   u32 i;
   *int_mask = 0;
   *fp_mask = 0;
   for (i = 0; i < nclob; ++i) {
     NativeAllocClass cls;
     Reg reg;
     if (!rv_asm_parse_reg_clobber(c, loc, clobbers[i], &cls, &reg)) continue;
     if (cls == NATIVE_REG_INT)
       *int_mask |= 1u << reg;
     else
       *fp_mask |= 1u << reg;
   }
 }
 
 /* Pin resolution + panic is the shared native_asm_bind_direct_operands path. */
 
 /* Direct (-O0) path: resolve a semantic Operand to a NativeAddr. */
 static NativeAddr rv_direct_addr(NativeDirectTarget* d, Operand op) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   switch ((OpKind)op.kind) {
     case OPK_LOCAL:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = d->locals[op.v.local - 1u].home;
       addr.base_type = op.type;
       return addr;
     case OPK_INDIRECT:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME_VALUE;
       addr.base.frame = d->locals[op.v.ind.base - 1u].home;
       addr.cls = d->locals[op.v.ind.base - 1u].cls;
       addr.base_type = d->locals[op.v.ind.base - 1u].type;
       addr.offset = op.v.ind.ofs;
       return addr;
     default:
       rv_asm_panic(d, "operand is not addressable");
   }
 }
 
 /* Materialize an OPK_INDIRECT (frame-value) base into a register, returning a
  * plain register-based NativeAddr. */
 static NativeAddr rv_direct_materialize_addr(NativeDirectTarget* d,
                                              Operand op) {
   RvNativeTarget* a = rv_of(d->native);
   NativeAddr addr = rv_direct_addr(d, op);
   if (addr.base_kind == NATIVE_ADDR_BASE_FRAME_VALUE) {
     NativeLoc base = native_loc_reg(addr.base_type, NATIVE_REG_INT, RV_TMP1);
     NativeAddr load;
     memset(&load, 0, sizeof load);
     load.base_kind = NATIVE_ADDR_BASE_FRAME;
     load.base.frame = addr.base.frame;
     load.base_type = addr.base_type;
     rv_emit_mem(a, 1, base, load,
                 native_mem_for_type(d->native, addr.base_type, 8));
     addr.base_kind = NATIVE_ADDR_BASE_REG;
     addr.base.reg = RV_TMP1;
   }
   return addr;
 }
 
 static void rv_direct_load_operand_to_reg(NativeDirectTarget* d, Operand op,
                                           NativeLoc dst) {
   RvNativeTarget* a = rv_of(d->native);
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   switch ((OpKind)op.kind) {
     case OPK_IMM:
       if ((NativeAllocClass)dst.cls != NATIVE_REG_INT)
         rv_asm_panic(d, "floating-point immediate asm input is unsupported");
       d->native->load_imm(d->native, dst, op.v.imm);
       return;
     case OPK_LOCAL:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = d->locals[op.v.local - 1u].home;
       addr.base_type = op.type;
       rv_emit_mem(a, 1, dst, addr, native_mem_for_type(d->native, op.type, 0));
       return;
     case OPK_GLOBAL:
       addr.base_kind = NATIVE_ADDR_BASE_GLOBAL;
       addr.base.global.sym = op.v.global.sym;
       addr.base.global.addend = op.v.global.addend;
       addr.base_type = op.type;
       d->native->load_addr(d->native, dst, addr);
       return;
     case OPK_INDIRECT:
       addr = rv_direct_materialize_addr(d, op);
       rv_emit_mem(a, 1, dst, addr, native_mem_for_type(d->native, op.type, 0));
       return;
   }
   rv_asm_panic(d, "unsupported asm input operand");
 }
 
 static void rv_direct_load_address_to_reg(NativeDirectTarget* d, Operand op,
                                           NativeLoc dst) {
   d->native->load_addr(d->native, dst, rv_direct_addr(d, op));
 }
 
 static void rv_direct_store_reg_to_operand(NativeDirectTarget* d, Operand op,
                                            NativeLoc src) {
   RvNativeTarget* a = rv_of(d->native);
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   if (op.kind == OPK_LOCAL) {
     addr.base_kind = NATIVE_ADDR_BASE_FRAME;
     addr.base.frame = d->locals[op.v.local - 1u].home;
     addr.base_type = op.type;
   } else {
     addr = rv_direct_materialize_addr(d, op);
   }
   rv_emit_mem(a, 0, src, addr, native_mem_for_type(d->native, op.type, 0));
 }
 
 /* Callee-saved registers an asm block clobbers must be spilled/restored around
  * the block (the only ABI duty the allocator cannot discharge itself). */
 typedef struct RvAsmSavedClobber {
   NativeFrameSlot slot;
   NativeAllocClass cls;
   Reg reg;
   KitCgTypeId type;
 } RvAsmSavedClobber;
 
 /* A clobber save slot is register-width: ptr_bytes for an integer reg (4 on
  * rv32, 8 on rv64) but always 8 for an FP reg (fsd, even on rv32d). */
 static u32 rv_asm_save_bytes(const RvNativeTarget* a,
                              const RvAsmSavedClobber* s) {
   return s->cls == NATIVE_REG_FP ? 8u : a->variant->ptr_bytes;
 }
 static void rv_asm_save_one(RvNativeTarget* a, RvAsmSavedClobber* s) {
   NativeFrameSlotDesc desc;
   NativeAddr addr;
   u32 sz = rv_asm_save_bytes(a, s);
   memset(&desc, 0, sizeof desc);
   desc.type = s->type;
   desc.size = sz;
   desc.align = sz;
   desc.kind = NATIVE_FRAME_SLOT_SAVE;
   s->slot = a->base.frame_slot(&a->base, &desc);
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = s->slot;
   addr.base_type = s->type;
   rv_emit_mem(a, 0, native_loc_reg(s->type, s->cls, s->reg), addr,
               native_mem_for_type(&a->base, s->type, sz));
 }
 static void rv_asm_restore_one(RvNativeTarget* a, const RvAsmSavedClobber* s) {
   NativeAddr addr;
   u32 sz = rv_asm_save_bytes(a, s);
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = s->slot;
   addr.base_type = s->type;
   rv_emit_mem(a, 1, native_loc_reg(s->type, s->cls, s->reg), addr,
               native_mem_for_type(&a->base, s->type, sz));
 }
 
 /* psABI callee-saved: integer s0..s11 (x8,x9,x18..x27), fp fs0..fs11
  * (f8,f9,f18..f27). x8 is the frame pointer and never asm-clobbered. */
 static int rv_reg_is_callee_int(Reg r) {
   return r == 9u || (r >= 18u && r <= 27u);
 }
 static int rv_reg_is_callee_fp(Reg r) {
   return r == 8u || r == 9u || (r >= 18u && r <= 27u);
 }
 
 static RvAsmSavedClobber* rv_asm_save_callee_clobbers(RvNativeTarget* a,
                                                       u32 int_mask, u32 fp_mask,
                                                       u32* nsaved_out) {
   RvAsmSavedClobber* saved =
       arena_zarray(a->base.c->tu, RvAsmSavedClobber, 24u);
   KitCgTypeId i64 = builtin_id(KIT_CG_BUILTIN_I64);
   KitCgTypeId f64 = builtin_id(KIT_CG_BUILTIN_F64);
   u32 n = 0;
   Reg r;
   for (r = 0; r <= 31u; ++r) {
     if ((int_mask & (1u << r)) == 0 || !rv_reg_is_callee_int(r)) continue;
     saved[n].cls = NATIVE_REG_INT;
     saved[n].reg = r;
     saved[n].type = i64;
     rv_asm_save_one(a, &saved[n++]);
   }
   for (r = 0; r <= 31u; ++r) {
     if ((fp_mask & (1u << r)) == 0 || !rv_reg_is_callee_fp(r)) continue;
     saved[n].cls = NATIVE_REG_FP;
     saved[n].reg = r;
     saved[n].type = f64;
     rv_asm_save_one(a, &saved[n++]);
   }
   *nsaved_out = n;
   return saved;
 }
 
 /* ---- NativeTarget (optimizer) asm hook ----
  * The optimized emitter owns register placement, staging, and output
  * writeback. This hook binds its concrete locations and only materializes
  * memory-constraint bases in backend-private registers. */
 
 static NativeAddr rv_asm_loc_to_addr(RvNativeTarget* a, SrcLoc loc,
                                      NativeLoc src) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_type = src.type;
   switch ((NativeLocKind)src.kind) {
     case NATIVE_LOC_FRAME:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = src.v.frame;
       return addr;
     case NATIVE_LOC_ADDR:
       return src.v.addr;
     case NATIVE_LOC_GLOBAL:
       addr.base_kind = NATIVE_ADDR_BASE_GLOBAL;
       addr.base.global.sym = src.v.global.sym;
       addr.base.global.addend = src.v.global.addend;
       return addr;
     case NATIVE_LOC_REG:
       addr.base_kind = NATIVE_ADDR_BASE_REG;
       addr.cls = NATIVE_REG_INT;
       addr.base.reg = src.v.reg;
       return addr;
     default:
       rv_asm_panic_at(a->base.c, loc, "unsupported memory asm operand");
   }
 }
 
 /* Resolve a memory-constraint operand to a single base register with zero
  * offset, folding any frame/global/offset into a reserved scratch register. */
 static Reg rv_asm_native_mem_base(RvNativeTarget* a, SrcLoc loc, NativeLoc src,
                                   u32* ntmp) {
   NativeAddr addr = rv_asm_loc_to_addr(a, loc, src);
   u32 base;
   i32 off;
   Reg dst;
   if (addr.index_kind != NATIVE_ADDR_INDEX_NONE)
     rv_asm_panic_at(a->base.c, loc, "indexed memory asm operand unsupported");
   rv_resolve_mem_addr(a, &addr, &base, &off);
   if (off == 0 && base != RV_TMP0 && base != RV_TMP1) return (Reg)base;
   if (*ntmp >= 2u)
     rv_asm_panic_at(a->base.c, loc, "too many memory asm operands");
   dst = (*ntmp == 0u) ? RV_TMP0 : RV_TMP1;
   (*ntmp)++;
   rv_emit_addr_adjust(a->variant, a->base.mc, dst, base, off);
   return dst;
 }
 
 static void rv_asm_native_panic(NativeTarget* t, SrcLoc loc, const char* msg) {
   rv_asm_panic_at(t->c, loc, msg);
 }
 
 static Reg rv_asm_native_mem_base_hook(NativeTarget* t, SrcLoc loc,
                                        NativeLoc src, u32* ntmp) {
   return rv_asm_native_mem_base(rv_of(t), loc, src, ntmp);
 }
 
 static void rv_asm_native_run_template_hook(
     NativeTarget* t, const char* tmpl, const AsmConstraint* outs, u32 nout,
     Operand* bound_outs, const AsmConstraint* ins, u32 nin, Operand* bound_ins,
     const Sym* clobbers, u32 nclob) {
   Rv64Asm* asmh = rv64_asm_open(t->c);
   rv64_inline_bind(asmh, outs, nout, bound_outs, ins, nin, bound_ins, clobbers,
                    nclob);
   rv64_asm_run_template(asmh, t->mc, tmpl);
   rv64_asm_close(asmh);
 }
 
 static void rv_asm_block_native(NativeTarget* t, const char* tmpl,
                                 const AsmConstraint* outs, u32 nout,
                                 NativeLoc* out_locs, const AsmConstraint* ins,
                                 u32 nin, const NativeLoc* in_locs,
                                 const Sym* clobbers, u32 nclob) {
   RvNativeTarget* a = rv_of(t);
   SrcLoc loc = a->func ? a->func->loc : (SrcLoc){0, 0, 0};
   static const NativeAsmNativeHooks hooks = {
       .panic = rv_asm_native_panic,
       .bound_reg = rv_asm_bound_reg,
       .bound_mem = rv_asm_bound_mem,
       .mem_base = rv_asm_native_mem_base_hook,
       .run_template = rv_asm_native_run_template_hook,
   };
   native_asm_bind_native_operands(t, loc, tmpl, outs, nout, out_locs, ins, nin,
                                   in_locs, clobbers, nclob, &hooks);
 }
 /* file_scope_asm + finalize are shared (cg/native_asm.h). */
 
 static void rv_trap(NativeTarget* t) { rv64_emit32(t->mc, rv_ebreak()); }
 static void rv_set_loc(NativeTarget* t, SrcLoc loc) {
   rv_of(t)->loc = loc;
   mc_set_loc(t->mc, loc);
 }
 
 /* ============================ construction ============================ */
 
 NativeTarget* rv64_native_target_new(Compiler* c, ObjBuilder* obj,
                                      MCEmitter* mc) {
   RvNativeTarget* a = arena_znew(c->tu, RvNativeTarget);
   NativeTarget* t;
   if (!a) return NULL;
   t = &a->base;
   t->c = c;
   t->obj = obj;
   t->mc = mc;
   a->variant = riscv_variant_for_kind(c->target.arch);
   native_frame_init(&a->frame, c);
   t->regs = &rv_reg_info;
   t->class_for_type = native_class_for_type_fp_le8;
   t->imm_legal = rv_imm_legal;
   t->addr_legal = rv_addr_legal;
   t->machine_op_clobbers = rv_machine_op_clobbers;
   t->func_begin = rv_func_begin;
   t->func_begin_known_frame = rv_func_begin_known_frame;
   t->note_frame_state = NULL;
   /* Non-NULL so the optimizer emit path (plan_frame) computes the callee-saved
    * set; rv_func_begin_known_frame derives the records from the masks. */
   t->reserve_callee_saves = rv_reserve_callee_saves;
   t->signature_stack_bytes = rv_signature_stack_bytes;
   t->call_stack_bytes = rv_call_stack_bytes;
   t->has_store_zero_reg = 1;
   t->store_zero_reg = RV_ZERO;
   t->func_end = rv_func_end;
   t->frame_slot = rv_frame_slot;
   t->release_frame_slot = rv_release_frame_slot;
   t->frame_slot_debug_loc = rv_frame_slot_debug_loc;
   t->bind_param = rv_bind_native_param;
   t->label_new = rv_label_new;
   t->label_place = rv_label_place;
   t->jump = rv_jump;
   t->cmp_branch = rv_cmp_branch;
   t->indirect_branch = rv_indirect_branch;
   t->load_label_addr = rv_load_label_addr;
   t->move = rv_move;
   t->load_imm = rv_load_imm;
   t->load_const = rv_load_const;
   t->load_addr = rv_load_addr;
   t->load = rv_load;
   t->store = rv_store;
   t->tls_addr_of = rv_tls_addr_of;
   t->copy_bytes = rv_copy_bytes;
   t->set_bytes = rv_set_bytes;
   t->bitfield_load = rv_bitfield_load;
   t->bitfield_store = rv_bitfield_store;
   t->binop = rv_binop;
   t->unop = rv_unop;
   t->cmp = rv_cmp;
   t->convert = rv_convert;
   t->binop_rr = rv_binop_rr;
   t->move_rr = rv_move_rr;
   t->cmp_rr = rv_cmp_rr;
   t->convert_rr = rv_convert_rr;
   t->alloca_ = rv_alloca;
   t->spill = rv_spill;
   t->reload = rv_reload;
   t->marshal_call = rv_marshal_call;
   t->emit_call = rv_emit_call;
   t->marshal_ret = rv_marshal_ret;
   t->ret = rv_ret;
   t->atomic_load = rv_atomic_load;
   t->atomic_store = rv_atomic_store;
   t->atomic_rmw = rv_atomic_rmw;
   t->atomic_cas = rv_atomic_cas;
   t->fence = rv_fence;
   t->va_start_ = rv_va_start_native;
   t->va_arg_ = rv_va_arg_native;
   t->va_end_ = rv_va_end_native;
   t->va_copy_ = rv_va_copy_native;
   t->intrinsic = rv_intrinsic;
   t->asm_block = rv_asm_block_native;
   t->file_scope_asm = native_file_scope_asm;
   t->trap = rv_trap;
   t->set_loc = rv_set_loc;
   t->finalize = native_finalize;
   return t;
 }
 
 /* ============================ NativeOps (-O0) ============================ */
 
 static void rv_bind_param(NativeDirectTarget* d, const CGParamDesc* p,
                           CGLocal local, NativeDirectLocal* l) {
   NativeLoc dst;
   (void)local;
   memset(&dst, 0, sizeof dst);
   dst.kind = NATIVE_LOC_FRAME;
   dst.type = p->type;
   dst.v.frame = l->home;
   rv_bind_native_param(d->native, p, dst);
 }
 
 /* A sibling call is realizable when its outgoing stack-argument area fits the
  * window the caller itself received (so the args land in the caller's incoming
  * slots without overflowing into the caller's caller's frame). Register-only
  * calls (the common case) always qualify. Mirrors aa64's aa_no_tail. */
 static const char* rv_no_tail(NativeDirectTarget* d, const CGCallDesc* call) {
   RvNativeTarget* a = rv_of(d->native);
   NativeCallDesc nd;
   u32 stack;
   if (a->frame.ncallee_saves)
     return "rv64 tail call: callee-saved registers in use";
   native_direct_project_tail_call_desc(d, call, &nd);
   stack = rv_call_stack_size(d->native, &nd);
   if (stack > a->incoming_stack_size)
     return "rv64 tail call: stack argument area too small";
   return NULL;
 }
 
 /* Resolve a pointer-typed Operand (the address of a va_list object) into `reg`
  * and return a register-based NativeAddr. An OPK_LOCAL holds the va_list object
  * itself, so we take its frame address; an OPK_INDIRECT holds the pointer in
  * memory and must be loaded. The va cores use TMP1/TMP2 internally, so `reg`
  * must be distinct from those (callers pass TMP0 / TMP3). */
 /* ap_addr is the pointer value &ap (the va_list object's address). For an
  * OPK_LOCAL the local HOLDS that pointer, so load its home value; an
  * OPK_INDIRECT names *(base+ofs), whose address base+ofs is the pointer.
  * Mirrors aa64's aa_direct_pointer_addr. */
 static NativeAddr rv_direct_pointer_addr(NativeDirectTarget* d, Operand op) {
   RvNativeTarget* a = rv_of(d->native);
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   if (op.kind == OPK_LOCAL) {
     NativeLoc base = native_loc_reg(op.type, NATIVE_REG_INT, RV_TMP1);
     NativeAddr load;
     memset(&load, 0, sizeof load);
     load.base_kind = NATIVE_ADDR_BASE_FRAME;
     load.base.frame = d->locals[op.v.local - 1u].home;
     load.base_type = op.type;
     rv_emit_mem(a, 1, base, load, native_mem_for_type(d->native, op.type, 8));
     addr.base_kind = NATIVE_ADDR_BASE_REG;
     addr.base.reg = RV_TMP1;
     addr.base_type = op.type;
     return addr;
   }
   return rv_direct_materialize_addr(d, op);
 }
 
 static NativeAddr rv_direct_va_base(NativeDirectTarget* d, Operand ap_addr,
                                     Reg reg) {
   NativeLoc dst =
       native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT, reg);
   NativeAddr addr;
   d->native->load_addr(d->native, dst, rv_direct_pointer_addr(d, ap_addr));
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_REG;
   addr.cls = NATIVE_REG_INT;
   addr.base.reg = reg;
   addr.base_type = builtin_id(KIT_CG_BUILTIN_I64);
   return addr;
 }
 
 static void rv_va_start_(NativeDirectTarget* d, Operand ap_addr) {
   rv_va_start_core(rv_of(d->native), rv_direct_va_base(d, ap_addr, RV_TMP3));
 }
 static void rv_va_arg_(NativeDirectTarget* d, Operand dst, Operand ap_addr,
                        KitCgTypeId type) {
   RvNativeTarget* a = rv_of(d->native);
   NativeAllocClass cls;
   /* A value too wide for one GPR (8-byte i64 / soft-double on ilp32) is copied
    * straight from the save area into its destination memory. */
   if (rv_va_arg_is_wide(d->native, type)) {
     rv_va_arg_wide(a, rv_direct_addr(d, dst),
                    rv_direct_va_base(d, ap_addr, RV_TMP3),
                    native_type_size(d->native, type));
     return;
   }
   /* Float-ABI-aware class: a soft (or wider-than-flen) float is INT-class so
    * the va_arg fetch never lands a double in an FP register on rv32. */
   cls = native_class_for_type_fp_le8(d->native, type);
   NativeLoc res =
       native_loc_reg(type, cls, cls == NATIVE_REG_FP ? RV_FTMP0 : RV_TMP0);
   NativeAddr dst_addr;
   rv_va_arg_core(a, res, rv_direct_va_base(d, ap_addr, RV_TMP3), type);
   /* Store the fetched value back into the semantic destination. */
   dst_addr = rv_direct_addr(d, dst);
   if (dst_addr.base_kind == NATIVE_ADDR_BASE_FRAME_VALUE) {
     NativeLoc base =
         native_loc_reg(dst_addr.base_type, NATIVE_REG_INT, RV_TMP1);
     NativeAddr load;
     memset(&load, 0, sizeof load);
     load.base_kind = NATIVE_ADDR_BASE_FRAME;
     load.base.frame = dst_addr.base.frame;
     load.base_type = dst_addr.base_type;
     rv_emit_mem(a, 1, base, load,
                 native_mem_for_type(d->native, dst_addr.base_type, 8));
     dst_addr.base_kind = NATIVE_ADDR_BASE_REG;
     dst_addr.base.reg = RV_TMP1;
   }
   rv_emit_mem(
       a, 0, res, dst_addr,
       native_mem_for_type(d->native, type, native_type_size(d->native, type)));
 }
 static void rv_va_end_(NativeDirectTarget* d, Operand ap_addr) {
   (void)d;
   (void)ap_addr;
 }
 static void rv_va_copy_(NativeDirectTarget* d, Operand dst, Operand src) {
   RvNativeTarget* a = rv_of(d->native);
   NativeAddr src_ap = rv_direct_va_base(d, src, RV_TMP0);
   NativeAddr dst_ap = rv_direct_va_base(d, dst, RV_TMP3);
   rv_va_copy_core(a, dst_ap, src_ap);
 }
 
 /* Hook adapters bridging the arch-typed save/restore + assembler entry to the
  * shared NativeAsmDirectHooks signatures. */
 static void* rv_asm_hook_save_callee_clobbers(NativeDirectTarget* d,
                                               u32 int_mask, u32 fp_mask,
                                               u32* nsaved_out) {
   return rv_asm_save_callee_clobbers(rv_of(d->native), int_mask, fp_mask,
                                      nsaved_out);
 }
 static void rv_asm_hook_restore_one(NativeDirectTarget* d, void* saved,
                                     u32 idx) {
   rv_asm_restore_one(rv_of(d->native), &((RvAsmSavedClobber*)saved)[idx]);
 }
 static void rv_asm_hook_run_template(NativeDirectTarget* d, const char* tmpl,
                                      const AsmConstraint* outs, u32 nout,
                                      Operand* bound_outs,
                                      const AsmConstraint* ins, u32 nin,
                                      Operand* bound_ins, const Sym* clobbers,
                                      u32 nclob) {
   Rv64Asm* asmh = rv64_asm_open(d->base.c);
   rv64_inline_bind(asmh, outs, nout, bound_outs, ins, nin, bound_ins, clobbers,
                    nclob);
   rv64_asm_run_template(asmh, d->native->mc, tmpl);
   rv64_asm_close(asmh);
 }
 
 static void rv_direct_asm_block(NativeDirectTarget* d, const char* tmpl,
                                 const AsmConstraint* outs, u32 nout,
                                 Operand* out_ops, const AsmConstraint* ins,
                                 u32 nin, const Operand* in_ops,
                                 const Sym* clobbers, u32 nclob,
                                 u32 clobber_abi_sets) {
   static const NativeAsmDirectHooks hooks = {
       .opk_reg = RV64_INLINE_OPK_REG,
       .opcls_fp = RV64_INLINE_OPCLS_FP,
       .panic = rv_asm_panic,
       .bound_reg = rv_asm_bound_reg,
       .bound_mem = rv_asm_bound_mem,
       .clobber_masks = rv_asm_clobber_masks,
       .save_callee_clobbers = rv_asm_hook_save_callee_clobbers,
       .restore_one = rv_asm_hook_restore_one,
       .load_operand_to_reg = rv_direct_load_operand_to_reg,
       .load_address_to_reg = rv_direct_load_address_to_reg,
       .store_reg_to_operand = rv_direct_store_reg_to_operand,
       .run_template = rv_asm_hook_run_template,
   };
   native_asm_bind_direct_operands(d, tmpl, outs, nout, out_ops, ins, nin,
                                   in_ops, clobbers, nclob, clobber_abi_sets,
                                   &hooks);
 }
 
 static const NativeOps rv_direct_ops = {
     .bind_param = rv_bind_param,
     .tail_call_unrealizable_reason = rv_no_tail,
     .va_start_ = rv_va_start_,
     .va_arg_ = rv_va_arg_,
     .va_end_ = rv_va_end_,
     .va_copy_ = rv_va_copy_,
     .asm_block = rv_direct_asm_block,
 };
 
 const NativeOps* rv64_native_direct_ops(void) { return &rv_direct_ops; }