kit

asm.c (55290B)

---

 /* RV64 assembler — descriptor-table driven.
  *
  * Mnemonic → Rv64InsnDesc via rv64_asm_find; operand parsing dispatches
  * on the format kind. The descriptor's `match` field already carries
  * the funct3/funct7/opcode bits; the parser only needs to fill in the
  * register operands and immediate.
  *
  * Aliases (li, mv, ret, jr, j, nop, sext.w, beqz, bnez) are recognized
  * by their alias rows in the descriptor table and rewritten to the
  * canonical encoding here. Inline rv_* encoders in isa.h remain the
  * hot path for codegen; the assembler uses them to assemble the
  * machine word once it has the operand values. */
 
 #include "arch/riscv/asm.h"
 
 #include <string.h>
 
 #include "arch/riscv/isa.h"
 #include "arch/riscv/regs.h"
 #include "arch/riscv/rv64.h"
 #include "arch/riscv/variant.h"
 #include "asm/asm_helpers.h"
 #include "core/arena.h"
 #include "core/pool.h"
 #include "core/slice.h"
 #include "core/strbuf.h"
 #include "obj/obj.h"
 
 struct Rv64Asm {
   ArchAsm base;
   Compiler* c;
 
   /* Inline-asm bound state (set by rv64_inline_bind, cleared otherwise).
    * Operand indexing per GCC convention: 0..nout-1 are outputs, then
    * nout..nout+nin-1 are inputs. Templates address into this combined
    * list via %N / %zN / %aN / %w[name] / %x[name]. */
   const AsmConstraint* outs;
   Operand* out_ops;
   const AsmConstraint* ins;
   const Operand* in_ops;
   const Sym* clobbers;
   u32 nout;
   u32 nin;
   u32 nclob;
 };
 
 typedef struct Rv64Asm Rv64Asm;
 
 /* Relocation modifier on a 12-bit immediate offset (`%lo`/`%pcrel_lo`).
  * RV_MEMMOD_NONE means a plain numeric displacement in `disp`. */
 typedef enum RvMemMod {
   RV_MEMMOD_NONE = 0,
   RV_MEMMOD_LO,
   RV_MEMMOD_PCREL_LO,
 } RvMemMod;
 
 typedef struct Rv64Mem {
   i32 disp;
   u32 base;
   RvMemMod mod; /* reloc modifier on the offset, or RV_MEMMOD_NONE */
   ObjSymId sym; /* symbol when mod != NONE */
   i64 off;      /* addend when mod != NONE */
 } Rv64Mem;
 
 static int sym_to_cstr(AsmDriver* d, Sym s, char* out, size_t cap) {
   Slice sl = pool_slice(asm_driver_pool(d), s);
   if (!sl.s || sl.len >= cap) return 0;
   memcpy(out, sl.s, sl.len);
   out[sl.len] = '\0';
   return 1;
 }
 
 /* True if `s` begins with the NUL-terminated literal `pfx` (length-explicit).
  */
 static bool slice_has_prefix_cstr(Slice s, const char* pfx, size_t n) {
   return s.len >= n && memcmp(s.s, pfx, n) == 0;
 }
 
 static int rv_reg_from_name(AsmDriver* d, Sym s, u32* reg_out, int* fp_out) {
   char name[16];
   uint32_t dwarf = 0;
   if (!sym_to_cstr(d, s, name, sizeof name)) return 0;
   if (rv64_register_index(name, &dwarf) != 0) return 0;
   if (reg_out) *reg_out = dwarf & 31u;
   if (fp_out) *fp_out = dwarf >= 32u;
   return 1;
 }
 
 static u32 parse_reg(AsmDriver* d, int* fp_out) {
   AsmTok t = asm_driver_next(d);
   u32 r;
   if (t.kind != ASM_TOK_IDENT || !rv_reg_from_name(d, t.v.ident, &r, fp_out))
     asm_driver_panic(d, "rv64 asm: bad register");
   return r;
 }
 
 static u32 parse_xreg(AsmDriver* d) {
   int fp = 0;
   u32 r = parse_reg(d, &fp);
   if (fp) asm_driver_panic(d, "rv64 asm: expected integer register");
   return r;
 }
 
 static u32 parse_freg(AsmDriver* d) {
   int fp = 0;
   u32 r = parse_reg(d, &fp);
   if (!fp) asm_driver_panic(d, "rv64 asm: expected float register");
   return r;
 }
 
 static void expect_comma(AsmDriver* d) {
   if (!asm_driver_eat_comma(d)) asm_driver_panic(d, "rv64 asm: expected ','");
 }
 
 /* Parse a CSR operand: a standard CSR name (mstatus, mtvec, ...) or a bare
  * numeric expression. Returns the 12-bit CSR number. */
 static u32 parse_csr(AsmDriver* d) {
   AsmTok t = asm_driver_peek(d);
   if (t.kind == ASM_TOK_IDENT) {
     u16 num;
     if (rv64_csr_num_from_name(pool_slice(asm_driver_pool(d), t.v.ident),
                                &num)) {
       (void)asm_driver_next(d); /* consume the name */
       return (u32)num & 0xfffu;
     }
   }
   return (u32)asm_driver_parse_const(d) & 0xfffu;
 }
 
 /* Position of a `%mod(sym)` relocation operand: the 20-bit upper field of
  * lui/auipc, or a 12-bit I-type (addi/load) or S-type (store) immediate. */
 typedef enum RvModPos {
   RV_MODPOS_HI20,
   RV_MODPOS_LO_I,
   RV_MODPOS_LO_S,
 } RvModPos;
 
 /* Map a relocation-modifier name (`hi`, `lo`, `pcrel_hi`, `pcrel_lo`,
  * `got_pcrel_hi`) to the RelocKind appropriate for `pos`. Panics on a name
  * that is not valid at this operand position. */
 static RelocKind rv_mod_to_reloc(AsmDriver* d, Slice name, RvModPos pos) {
   if (pos == RV_MODPOS_HI20) {
     if (slice_eq_cstr(name, "hi")) return R_RV_HI20;
     if (slice_eq_cstr(name, "pcrel_hi")) return R_RV_PCREL_HI20;
     if (slice_eq_cstr(name, "got_pcrel_hi")) return R_RV_GOT_HI20;
   } else {
     int store = (pos == RV_MODPOS_LO_S);
     if (slice_eq_cstr(name, "lo")) return store ? R_RV_LO12_S : R_RV_LO12_I;
     if (slice_eq_cstr(name, "pcrel_lo"))
       return store ? R_RV_PCREL_LO12_S : R_RV_PCREL_LO12_I;
   }
   asm_driver_panic(d, "rv64 asm: relocation modifier not valid here");
 }
 
 /* If the next token is `%`, parse a `%mod(sym{+off})` relocation operand,
  * emit the relocation at the current emit position (where the about-to-be-
  * returned instruction word will land), and return 1. The caller encodes a
  * zero placeholder in the immediate field. Returns 0 if there is no modifier
  * (leaving the stream untouched for normal constant parsing). A leading `%`
  * is unambiguous here: modulo is infix and never starts an operand. */
 static int rv_parse_mod_reloc(AsmDriver* d, RvModPos pos, ObjSymId* sym_out,
                               i64* off_out, RelocKind* kind_out) {
   if (!asm_driver_tok_is_punct(asm_driver_peek(d), '%')) return 0;
   (void)asm_driver_next(d); /* eat '%' */
   AsmTok name = asm_driver_next(d);
   if (name.kind != ASM_TOK_IDENT)
     asm_driver_panic(d, "rv64 asm: expected relocation modifier name");
   Slice nm = pool_slice(asm_driver_pool(d), name.v.ident);
   asm_driver_expect_punct(d, '(', "'(' after relocation modifier");
   ObjSymId sym = OBJ_SYM_NONE;
   i64 off = 0;
   asm_driver_parse_sym_expr(d, &sym, &off);
   asm_driver_expect_punct(d, ')', "')' after %mod(sym)");
   RelocKind k = rv_mod_to_reloc(d, nm, pos);
   if (sym_out) *sym_out = sym;
   if (off_out) *off_out = off;
   if (kind_out) *kind_out = k;
   return 1;
 }
 
 /* Parse a RISC-V rounding-mode mnemonic (the comma is already consumed) into
  * its 3-bit funct3 value. cc -S emits this suffix on fcvt/fsqrt when the mode
  * isn't the default `dyn`, so the round-trip (and clang) re-encode the exact
  * mode rather than guessing a default. */
 static u32 rv_parse_rm_name(AsmDriver* d) {
   AsmTok t = asm_driver_next(d);
   Slice s;
   if (t.kind != ASM_TOK_IDENT)
     asm_driver_panic(d, "rv64 asm: expected rounding mode");
   s = pool_slice(asm_driver_pool(d), t.v.ident);
   if (slice_eq_cstr(s, "rne")) return 0u;
   if (slice_eq_cstr(s, "rtz")) return 1u;
   if (slice_eq_cstr(s, "rdn")) return 2u;
   if (slice_eq_cstr(s, "rup")) return 3u;
   if (slice_eq_cstr(s, "rmm")) return 4u;
   if (slice_eq_cstr(s, "dyn")) return 7u;
   asm_driver_panic(d, "rv64 asm: unknown rounding mode");
 }
 
 /* Emit a relocation for a U-type / I-type immediate `%mod(sym)` operand at
  * the current instruction position; returns 1 if one was present. */
 static int rv_emit_imm_mod_reloc(AsmDriver* d, RvModPos pos) {
   ObjSymId sym;
   i64 off;
   RelocKind k;
   if (!rv_parse_mod_reloc(d, pos, &sym, &off, &k)) return 0;
   MCEmitter* mc = asm_driver_mc(d);
   mc_emit_reloc_at(mc, mc->section_id, mc_pos(mc), k, sym, off, 0, 0);
   return 1;
 }
 
 static Rv64Mem parse_mem(AsmDriver* d) {
   Rv64Mem m;
   m.disp = 0;
   m.mod = RV_MEMMOD_NONE;
   m.sym = OBJ_SYM_NONE;
   m.off = 0;
   if (asm_driver_tok_is_punct(asm_driver_peek(d), '%')) {
     /* `%lo(sym)(base)` / `%pcrel_lo(label)(base)` — record the modifier; the
      * load/store caller emits the I- or S-type relocation. */
     ObjSymId sym;
     i64 off;
     RelocKind k;
     (void)rv_parse_mod_reloc(d, RV_MODPOS_LO_I, &sym, &off, &k);
     m.mod = (k == R_RV_PCREL_LO12_I) ? RV_MEMMOD_PCREL_LO : RV_MEMMOD_LO;
     m.sym = sym;
     m.off = off;
   } else {
     m.disp = (i32)asm_driver_parse_const(d);
   }
   asm_driver_expect_punct(d, '(', "'(' in rv64 memory operand");
   m.base = parse_xreg(d);
   asm_driver_expect_punct(d, ')', "')' in rv64 memory operand");
   return m;
 }
 
 /* Emit the I/S-type relocation recorded by parse_mem for a `%lo`/`%pcrel_lo`
  * memory offset, picking the S-type variant for stores. */
 static void rv_emit_mem_mod_reloc(AsmDriver* d, const Rv64Mem* m,
                                   int is_store) {
   if (m->mod == RV_MEMMOD_NONE) return;
   RelocKind k = (m->mod == RV_MEMMOD_PCREL_LO)
                     ? (is_store ? R_RV_PCREL_LO12_S : R_RV_PCREL_LO12_I)
                     : (is_store ? R_RV_LO12_S : R_RV_LO12_I);
   MCEmitter* mc = asm_driver_mc(d);
   mc_emit_reloc_at(mc, mc->section_id, mc_pos(mc), k, m->sym, m->off, 0, 0);
 }
 
 /* Fence pred/succ parser — accepts a string like "rw" / "iorw" / "0" /
  * a numeric literal. Returns the 4-bit mask: bit3=i, bit2=o, bit1=r,
  * bit0=w. */
 static u32 parse_fence_mask(AsmDriver* d) {
   AsmTok t = asm_driver_peek(d);
   if (t.kind == ASM_TOK_NUM) {
     (void)asm_driver_next(d);
     return (u32)asm_driver_parse_const(d) & 0xfu;
   }
   if (t.kind == ASM_TOK_IDENT) {
     char name[8];
     AsmTok tt = asm_driver_next(d);
     if (!sym_to_cstr(d, tt.v.ident, name, sizeof name))
       asm_driver_panic(d, "rv64 asm: bad fence mask");
     u32 mask = 0;
     for (const char* p = name; *p; ++p) {
       switch (*p) {
         case 'i':
           mask |= 8u;
           break;
         case 'o':
           mask |= 4u;
           break;
         case 'r':
           mask |= 2u;
           break;
         case 'w':
           mask |= 1u;
           break;
         default:
           asm_driver_panic(d, "rv64 asm: bad fence char");
       }
     }
     return mask;
   }
   asm_driver_panic(d, "rv64 asm: bad fence operand");
 }
 
 /* The XLEN variant for the assembly target. Reached off the AsmDriver's
  * Compiler so the stateless encoders can gate rv32-vs-rv64 behavior
  * (shamt width, addiw availability). */
 static const RiscvVariant* rv_asm_variant(AsmDriver* d) {
   return riscv_variant_for_kind(asm_driver_compiler(d)->target.arch);
 }
 
 /* Field overlay onto a descriptor's `match` word.
  *
  * For most formats the descriptor's match already pins opcode +
  * funct3 + funct7. We OR in the per-operand fields. For shift-imm and
  * AMO families the layouts diverge from the basic R/I templates — we
  * handle those explicitly below. */
 
 static u32 enc_r(u32 match, u32 rd, u32 rs1, u32 rs2) {
   return match | ((rs2 & 0x1fu) << 20) | ((rs1 & 0x1fu) << 15) |
          ((rd & 0x1fu) << 7);
 }
 static u32 enc_i(u32 match, u32 rd, u32 rs1, i32 imm12) {
   return match | (((u32)imm12 & 0xfffu) << 20) | ((rs1 & 0x1fu) << 15) |
          ((rd & 0x1fu) << 7);
 }
 static u32 enc_s(u32 match, u32 rs2, u32 rs1, i32 imm12) {
   return match | rv_imm_s(imm12) | ((rs2 & 0x1fu) << 20) |
          ((rs1 & 0x1fu) << 15);
 }
 static u32 enc_b(u32 match, u32 rs1, u32 rs2, i32 imm13) {
   return match | rv_imm_b(imm13) | ((rs2 & 0x1fu) << 20) |
          ((rs1 & 0x1fu) << 15);
 }
 static u32 enc_u(u32 match, u32 rd, u32 imm20) {
   return match | ((imm20 & 0xfffffu) << 12) | ((rd & 0x1fu) << 7);
 }
 static u32 enc_j(u32 match, u32 rd, i32 imm21) {
   return match | rv_imm_j(imm21) | ((rd & 0x1fu) << 7);
 }
 static u32 enc_r4(u32 match, u32 rd, u32 rs1, u32 rs2, u32 rs3, u32 rm) {
   return match | ((rs3 & 0x1fu) << 27) | ((rs2 & 0x1fu) << 20) |
          ((rs1 & 0x1fu) << 15) | ((rm & 0x7u) << 12) | ((rd & 0x1fu) << 7);
 }
 
 /* SLLI/SRLI/SRAI shift-imm. The shamt occupies bits 25:20 on rv64 (6-bit,
  * funct6 in match) but only bits 24:20 on rv32 (5-bit; bit 25 belongs to
  * funct7 and MUST stay 0, else the word reads as a different funct7). The
  * variant's shamt_bits drives the mask; an rv32 shamt >= 32 is rejected. */
 static u32 enc_ishift(AsmDriver* d, u32 match, u32 rd, u32 rs1, u32 shamt) {
   u32 shamt_bits = rv_asm_variant(d)->shamt_bits;
   u32 shamt_mask = (shamt_bits == 5u) ? 0x1fu : 0x3fu;
   if (shamt > shamt_mask)
     asm_driver_panic(d, "rv64 asm: shift amount out of range for target XLEN");
   return match | ((shamt & shamt_mask) << 20) | ((rs1 & 0x1fu) << 15) |
          ((rd & 0x1fu) << 7);
 }
 /* RV32 word shift-imm: shamt5 occupies bits 24:20 (funct7 already pinned). */
 static u32 enc_ishiftw(u32 match, u32 rd, u32 rs1, u32 shamt) {
   return match | ((shamt & 0x1fu) << 20) | ((rs1 & 0x1fu) << 15) |
          ((rd & 0x1fu) << 7);
 }
 /* AMO: aq/rl bits 26/25 — we accept them as optional .aq/.rl suffixes
  * on the mnemonic. For now mnemonics arrive bare. */
 static u32 enc_amo(u32 match, u32 aq, u32 rl, u32 rd, u32 rs1, u32 rs2) {
   return match | ((aq & 1u) << 26) | ((rl & 1u) << 25) | ((rs2 & 0x1fu) << 20) |
          ((rs1 & 0x1fu) << 15) | ((rd & 0x1fu) << 7);
 }
 
 static u32 c_reg3(AsmDriver* d, u32 r) {
   if (r < 8u || r > 15u)
     asm_driver_panic(d,
                      "rv64 asm: compressed register must be x8..x15/f8..f15");
   return r - 8u;
 }
 
 static u32 enc_c_ci(u32 match, u32 rd, i32 imm) {
   u32 u = (u32)imm & 0x3fu;
   return match | (((u >> 5) & 1u) << 12) | ((rd & 0x1fu) << 7) |
          ((u & 0x1fu) << 2);
 }
 
 static u32 enc_c_cr(u32 match, u32 rd_rs1, u32 rs2) {
   return match | ((rd_rs1 & 0x1fu) << 7) | ((rs2 & 0x1fu) << 2);
 }
 
 static u32 enc_c_addi16sp(u32 match, i32 imm) {
   u32 u = (u32)imm & 0x3ffu;
   return match | (((u >> 9) & 1u) << 12) | (((u >> 4) & 1u) << 6) |
          (((u >> 6) & 1u) << 5) | (((u >> 7) & 3u) << 3) |
          (((u >> 5) & 1u) << 2);
 }
 
 static u32 enc_c_addi4spn(u32 match, u32 rd3, u32 imm) {
   u32 enc = (((imm >> 4) & 3u) << 6) | (((imm >> 6) & 0xfu) << 2) |
             (((imm >> 2) & 1u) << 1) | ((imm >> 3) & 1u);
   return match | ((enc & 0xffu) << 5) | ((rd3 & 7u) << 2);
 }
 
 static u32 enc_c_lwld(u32 match, u32 rd3, u32 rs1_3, u32 off, int wide64) {
   if (wide64) {
     return match | (((off >> 3) & 7u) << 10) | ((rs1_3 & 7u) << 7) |
            (((off >> 6) & 3u) << 5) | ((rd3 & 7u) << 2);
   }
   return match | (((off >> 3) & 7u) << 10) | ((rs1_3 & 7u) << 7) |
          (((off >> 2) & 1u) << 6) | (((off >> 6) & 1u) << 5) |
          ((rd3 & 7u) << 2);
 }
 
 static u32 enc_c_swld(u32 match, u32 rs2_3, u32 rs1_3, u32 off, int wide64) {
   return enc_c_lwld(match, rs2_3, rs1_3, off, wide64);
 }
 
 static u32 enc_c_lwsp(u32 match, u32 rd, u32 off, int wide64) {
   if (wide64) {
     return match | (((off >> 5) & 1u) << 12) | ((rd & 0x1fu) << 7) |
            (((off >> 3) & 3u) << 5) | (((off >> 6) & 7u) << 2);
   }
   return match | (((off >> 5) & 1u) << 12) | ((rd & 0x1fu) << 7) |
          (((off >> 2) & 7u) << 4) | (((off >> 6) & 3u) << 2);
 }
 
 static u32 enc_c_swsp(u32 match, u32 rs2, u32 off, int wide64) {
   u32 imm6;
   if (wide64)
     imm6 = (((off >> 3) & 7u) << 3) | ((off >> 6) & 7u);
   else
     imm6 = (((off >> 2) & 0xfu) << 2) | ((off >> 6) & 3u);
   return match | ((imm6 & 0x3fu) << 7) | ((rs2 & 0x1fu) << 2);
 }
 
 static u32 enc_c_cb_imm(u32 match, u32 rs1_3, i32 imm) {
   u32 u = (u32)imm & 0x1ffu;
   return match | (((u >> 8) & 1u) << 12) | (((u >> 3) & 3u) << 10) |
          ((rs1_3 & 7u) << 7) | (((u >> 6) & 3u) << 5) | (((u >> 1) & 3u) << 3) |
          (((u >> 5) & 1u) << 2);
 }
 
 static u32 enc_c_cb_alu_imm(u32 match, u32 rd3, i32 imm) {
   u32 u = (u32)imm & 0x3fu;
   return match | (((u >> 5) & 1u) << 12) | ((rd3 & 7u) << 7) |
          ((u & 0x1fu) << 2);
 }
 
 static u32 enc_c_cj(u32 match, i32 imm) {
   u32 u = (u32)imm & 0xfffu;
   return match | (((u >> 11) & 1u) << 12) | (((u >> 4) & 1u) << 11) |
          (((u >> 8) & 3u) << 9) | (((u >> 10) & 1u) << 8) |
          (((u >> 6) & 1u) << 7) | (((u >> 7) & 1u) << 6) |
          (((u >> 1) & 7u) << 3) | (((u >> 5) & 1u) << 2);
 }
 
 /* Parse a branch/jump target operand. With a symbolic target (a label), emit
  * the relocation at the current position — which is exactly where the caller
  * is about to write this instruction word — and return 0 as the placeholder
  * immediate. With a bare constant, return it as the PC-relative byte
  * displacement (preserving the existing numeric-offset corpus behavior). */
 static i32 rv_reloc_target(AsmDriver* d, RelocKind kind) {
   ObjSymId sym = OBJ_SYM_NONE;
   i64 off = 0;
   asm_driver_parse_sym_expr(d, &sym, &off);
   if (sym != OBJ_SYM_NONE) {
     MCEmitter* mc = asm_driver_mc(d);
     mc_emit_reloc_at(mc, mc->section_id, mc_pos(mc), kind, sym, off, 0, 0);
     return 0;
   }
   return (i32)off;
 }
 
 /* Per-format parser — reads the operand list off the driver and returns
  * the encoded 32-bit word, given the matched descriptor. */
 static u32 assemble_one(AsmDriver* d, const Rv64InsnDesc* desc) {
   u32 m = desc->match;
   u32 rd = 0, rs1 = 0, rs2 = 0;
   i32 imm = 0;
   Rv64Mem mem;
 
   switch ((Rv64Format)desc->fmt) {
     case RV64_FMT_R:
       /* Two-operand aliases: snez/neg/negw — rd, rs (rs1=x0). */
       if (desc->flags & RV64_ASMFL_ALIAS) {
         rd = parse_xreg(d);
         expect_comma(d);
         rs2 = parse_xreg(d);
         return enc_r(m, rd, 0u, rs2);
       }
       rd = parse_xreg(d);
       expect_comma(d);
       rs1 = parse_xreg(d);
       expect_comma(d);
       rs2 = parse_xreg(d);
       return enc_r(m, rd, rs1, rs2);
 
     case RV64_FMT_R4: {
       u32 rs3;
       rd = parse_freg(d);
       expect_comma(d);
       rs1 = parse_freg(d);
       expect_comma(d);
       rs2 = parse_freg(d);
       expect_comma(d);
       rs3 = parse_freg(d);
       return enc_r4(m, rd, rs1, rs2, rs3, 0x7u);
     }
 
     case RV64_FMT_I:
       /* Aliases first. `li` is handled earlier by rv64_emit_pseudo (it may
        * need a multi-word expansion), so it never reaches here. */
       if (desc->flags & RV64_ASMFL_ALIAS) {
         if (slice_eq_cstr(desc->mnemonic, "mv")) {
           /* Standard two-operand `mv rd, rs` = `addi rd, rs, 0`. (A %pcrel_lo
            * low-half is emitted as the canonical `addi rd, rs, %pcrel_lo(L)`,
            * not a non-standard 3-operand `mv`, so it lands in the ADDI path
            * below — matching clang.) */
           rd = parse_xreg(d);
           expect_comma(d);
           rs1 = parse_xreg(d);
           return enc_i(m, rd, rs1, 0);
         }
         if (slice_eq_cstr(desc->mnemonic, "sext.w")) {
           rd = parse_xreg(d);
           expect_comma(d);
           rs1 = parse_xreg(d);
           return enc_i(m, rd, rs1, 0);
         }
         if (slice_eq_cstr(desc->mnemonic, "seqz") ||
             slice_eq_cstr(desc->mnemonic, "not")) {
           rd = parse_xreg(d);
           expect_comma(d);
           rs1 = parse_xreg(d);
           /* match already has imm12 + funct3 + op pinned. */
           return m | ((rs1 & 0x1fu) << 15) | ((rd & 0x1fu) << 7);
         }
       }
       rd = parse_xreg(d);
       expect_comma(d);
       rs1 = parse_xreg(d);
       expect_comma(d);
       /* `addi rd, rs1, %lo(sym)` / `%pcrel_lo(label)` → R_RV_LO12_I. */
       if (rv_emit_imm_mod_reloc(d, RV_MODPOS_LO_I)) return enc_i(m, rd, rs1, 0);
       imm = (i32)asm_driver_parse_const(d);
       return enc_i(m, rd, rs1, imm);
 
     case RV64_FMT_I_SHIFT:
       rd = parse_xreg(d);
       expect_comma(d);
       rs1 = parse_xreg(d);
       expect_comma(d);
       return enc_ishift(d, m, rd, rs1, (u32)asm_driver_parse_const(d));
 
     case RV64_FMT_I_SHIFTW:
       rd = parse_xreg(d);
       expect_comma(d);
       rs1 = parse_xreg(d);
       expect_comma(d);
       return enc_ishiftw(m, rd, rs1, (u32)asm_driver_parse_const(d));
 
     case RV64_FMT_U:
       rd = parse_xreg(d);
       expect_comma(d);
       /* `lui rd, %hi(sym)` → R_RV_HI20; `auipc rd, %pcrel_hi(sym)` →
        * R_RV_PCREL_HI20 (or %got_pcrel_hi → R_RV_GOT_HI20). */
       if (rv_emit_imm_mod_reloc(d, RV_MODPOS_HI20)) return enc_u(m, rd, 0);
       imm = (i32)asm_driver_parse_const(d);
       /* LUI/AUIPC immediate is the upper-20 value: the input is interpreted
        * as the literal 20-bit value (already shifted-out form). */
       return enc_u(m, rd, (u32)imm);
 
     case RV64_FMT_J:
       /* `j label` / `jal rd, label` accept a symbolic target (R_RV_JAL) or a
        * bare numeric displacement. */
       if ((desc->flags & RV64_ASMFL_ALIAS) &&
           slice_eq_cstr(desc->mnemonic, "j")) {
         return enc_j(m, 0u, rv_reloc_target(d, R_RV_JAL));
       }
       rd = parse_xreg(d);
       expect_comma(d);
       return enc_j(m, rd, rv_reloc_target(d, R_RV_JAL));
 
     case RV64_FMT_B:
       /* `beq rs1, rs2, label` (and beqz/bnez aliases) accept a symbolic target
        * (R_RV_BRANCH) or a bare numeric displacement. */
       if (desc->flags & RV64_ASMFL_ALIAS) {
         /* beqz / bnez: rs, off. */
         rs1 = parse_xreg(d);
         expect_comma(d);
         return enc_b(m, rs1, 0u, rv_reloc_target(d, R_RV_BRANCH));
       }
       rs1 = parse_xreg(d);
       expect_comma(d);
       rs2 = parse_xreg(d);
       expect_comma(d);
       return enc_b(m, rs1, rs2, rv_reloc_target(d, R_RV_BRANCH));
 
     case RV64_FMT_LOAD:
       rd = (desc->flags & RV64_ASMFL_FP) ? parse_freg(d) : parse_xreg(d);
       expect_comma(d);
       mem = parse_mem(d);
       rv_emit_mem_mod_reloc(d, &mem, /*is_store=*/0);
       return enc_i(m, rd, mem.base, mem.disp);
 
     case RV64_FMT_FP_LOAD:
       rd = parse_freg(d);
       expect_comma(d);
       mem = parse_mem(d);
       rv_emit_mem_mod_reloc(d, &mem, /*is_store=*/0);
       return enc_i(m, rd, mem.base, mem.disp);
 
     case RV64_FMT_STORE:
       rs2 = (desc->flags & RV64_ASMFL_FP) ? parse_freg(d) : parse_xreg(d);
       expect_comma(d);
       mem = parse_mem(d);
       rv_emit_mem_mod_reloc(d, &mem, /*is_store=*/1);
       return enc_s(m, rs2, mem.base, mem.disp);
 
     case RV64_FMT_FP_STORE:
       rs2 = parse_freg(d);
       expect_comma(d);
       mem = parse_mem(d);
       rv_emit_mem_mod_reloc(d, &mem, /*is_store=*/1);
       return enc_s(m, rs2, mem.base, mem.disp);
 
     case RV64_FMT_JALR:
       if ((desc->flags & RV64_ASMFL_ALIAS) &&
           slice_eq_cstr(desc->mnemonic, "jr")) {
         rs1 = parse_xreg(d);
         return enc_i(m, 0u, rs1, 0);
       }
       rd = parse_xreg(d);
       if (!asm_driver_eat_comma(d)) {
         if (slice_eq_cstr(desc->mnemonic, "jalr"))
           return enc_i(m, RV_RA, rd, 0);
         asm_driver_panic(d, "rv64 asm: expected ','");
       }
       /* Accept both `jalr rd, imm(rs1)` and `jalr rd, rs1, imm`. */
       {
         AsmTok t = asm_driver_peek(d);
         if (t.kind == ASM_TOK_IDENT) {
           /* register first → register form */
           rs1 = parse_xreg(d);
           if (asm_driver_eat_comma(d)) {
             imm = (i32)asm_driver_parse_const(d);
           } else {
             imm = 0;
           }
           return enc_i(m, rd, rs1, imm);
         }
       }
       mem = parse_mem(d);
       return enc_i(m, rd, mem.base, mem.disp);
 
     case RV64_FMT_FENCE: {
       u32 pred, succ;
       pred = parse_fence_mask(d);
       expect_comma(d);
       succ = parse_fence_mask(d);
       return m | (pred << 24) | (succ << 20);
     }
 
     case RV64_FMT_SYSTEM:
       /* No operands. nop/ret/ecall/ebreak. */
       return m;
 
     case RV64_FMT_FP_RM:
       rd = parse_freg(d);
       expect_comma(d);
       rs1 = parse_freg(d);
       expect_comma(d);
       rs2 = parse_freg(d);
       /* Use DYN(=7) rounding mode by default. */
       return enc_r(m | (0x7u << 12), rd, rs1, rs2);
 
     case RV64_FMT_FP_R:
       if (desc->flags & RV64_ASMFL_FP) {
         rd = parse_freg(d);
       } else {
         rd = parse_xreg(d);
       }
       expect_comma(d);
       rs1 = parse_freg(d);
       expect_comma(d);
       rs2 = parse_freg(d);
       return enc_r(m, rd, rs1, rs2);
 
     case RV64_FMT_FP_CVT:
       if (desc->flags & RV64_ASMFL_FP) {
         rd = parse_freg(d);
         expect_comma(d);
         /* Source: integer reg for fcvt.s.w etc (no FP flag would
          * indicate); but since we have ASMFL_FP set on dest, source may
          * be either. Disambiguate by mnemonic. */
         if (slice_has_prefix_cstr(desc->mnemonic, "fcvt.s.", 7) &&
             (desc->mnemonic.s[7] == 'w' || desc->mnemonic.s[7] == 'l')) {
           rs1 = parse_xreg(d);
         } else if (slice_has_prefix_cstr(desc->mnemonic, "fcvt.d.", 7) &&
                    (desc->mnemonic.s[7] == 'w' || desc->mnemonic.s[7] == 'l')) {
           rs1 = parse_xreg(d);
         } else if (slice_eq_cstr(desc->mnemonic, "fmv.w.x") ||
                    slice_eq_cstr(desc->mnemonic, "fmv.d.x")) {
           rs1 = parse_xreg(d);
         } else {
           rs1 = parse_freg(d);
         }
       } else {
         rd = parse_xreg(d);
         expect_comma(d);
         rs1 = parse_freg(d);
       }
       /* match encodes rs2 (type selector); OR in rd/rs1 and the rounding mode.
        * An explicit `, <rm>` suffix (cc -S emits it for non-default modes, and
        * clang/gas accept it) takes precedence; otherwise a bare conversion
        * mnemonic encodes the dynamic rounding mode (DYN=7), matching gas/clang
        * for hand-written assembly. (Codegen's C float->int truncation is RTZ,
        * but that path uses the rv_fcvt_* encoders directly and supplies its own
        * rm; the text assembler must follow the assembler convention.) fmv
        * bit-moves carry no rounding (rm=0). */
       {
         u32 funct7 = (m >> 25) & 0x7fu;
         u32 rm;
         if (asm_driver_eat_comma(d)) {
           rm = rv_parse_rm_name(d);
         } else {
           switch (funct7) {
             case 0x70: /* fmv.x.w */
             case 0x71: /* fmv.x.d */
             case 0x78: /* fmv.w.x */
             case 0x79: /* fmv.d.x */
               rm = 0x0u;
               break;
             default: /* fcvt families: DYN (explicit suffix overrides above) */
               rm = 0x7u;
               break;
           }
         }
         return m | (rm << 12) | ((rs1 & 0x1fu) << 15) | ((rd & 0x1fu) << 7);
       }
 
     case RV64_FMT_AMO:
       rd = parse_xreg(d);
       expect_comma(d);
       rs2 = parse_xreg(d);
       expect_comma(d);
       asm_driver_expect_punct(d, '(', "'(' in rv64 amo operand");
       rs1 = parse_xreg(d);
       asm_driver_expect_punct(d, ')', "')' in rv64 amo operand");
       return enc_amo(m, 0u, 0u, rd, rs1, rs2);
 
     case RV64_FMT_LR:
       rd = parse_xreg(d);
       expect_comma(d);
       asm_driver_expect_punct(d, '(', "'(' in rv64 lr operand");
       rs1 = parse_xreg(d);
       asm_driver_expect_punct(d, ')', "')' in rv64 lr operand");
       return enc_amo(m, 0u, 0u, rd, rs1, 0u);
 
     case RV64_FMT_CSR: {
       u32 csr;
       rd = parse_xreg(d);
       expect_comma(d);
       csr = parse_csr(d);
       expect_comma(d);
       rs1 = parse_xreg(d);
       return enc_i(m, rd, rs1, (i32)csr);
     }
 
     case RV64_FMT_CSRI: {
       u32 csr;
       rd = parse_xreg(d);
       expect_comma(d);
       csr = parse_csr(d);
       expect_comma(d);
       u32 uimm = (u32)asm_driver_parse_const(d) & 0x1fu;
       return enc_i(m, rd, uimm, (i32)csr);
     }
 
     case RV64_FMT_CSR_PSEUDO: {
       /* 2-operand CSR pseudos. The match word already pins funct3+opcode; we
        * supply x0 for the implicit rd or rs1 per the mnemonic. */
       u32 csr;
       if (slice_eq_cstr(desc->mnemonic, "csrr")) {
         /* csrr rd, csr = csrrs rd, csr, x0 */
         rd = parse_xreg(d);
         expect_comma(d);
         csr = parse_csr(d);
         return enc_i(m, rd, 0u, (i32)csr);
       }
       /* csrw/csrs/csrc csr, rs   and   csrwi/csrsi/csrci csr, uimm:
        * destination is x0, csr comes first. */
       csr = parse_csr(d);
       expect_comma(d);
       if (slice_eq_cstr(desc->mnemonic, "csrwi") ||
           slice_eq_cstr(desc->mnemonic, "csrsi") ||
           slice_eq_cstr(desc->mnemonic, "csrci")) {
         u32 uimm = (u32)asm_driver_parse_const(d) & 0x1fu;
         return enc_i(m, 0u, uimm, (i32)csr);
       }
       rs1 = parse_xreg(d);
       return enc_i(m, 0u, rs1, (i32)csr);
     }
 
     case RV64_FMT_CR:
       if (slice_eq_cstr(desc->mnemonic, "c.jr") ||
           slice_eq_cstr(desc->mnemonic, "c.jalr")) {
         rs1 = parse_xreg(d);
         return enc_c_cr(m, rs1, 0u);
       }
       rd = parse_xreg(d);
       expect_comma(d);
       rs2 = parse_xreg(d);
       return enc_c_cr(m, rd, rs2);
 
     case RV64_FMT_CI:
       if (slice_eq_cstr(desc->mnemonic, "c.lwsp") ||
           slice_eq_cstr(desc->mnemonic, "c.ldsp") ||
           slice_eq_cstr(desc->mnemonic, "c.fldsp")) {
         rd = slice_eq_cstr(desc->mnemonic, "c.fldsp") ? parse_freg(d)
                                                       : parse_xreg(d);
         expect_comma(d);
         mem = parse_mem(d);
         if (mem.base != RV_SP)
           asm_driver_panic(d, "rv64 asm: compressed stack load needs sp base");
         return enc_c_lwsp(m, rd, (u32)mem.disp,
                           !slice_eq_cstr(desc->mnemonic, "c.lwsp"));
       }
       rd = parse_xreg(d);
       expect_comma(d);
       imm = (i32)asm_driver_parse_const(d);
       if (slice_eq_cstr(desc->mnemonic, "c.lui") && ((u32)imm & 0xfffu) == 0)
         imm >>= 12;
       if (slice_eq_cstr(desc->mnemonic, "c.addi16sp")) {
         if (rd != RV_SP)
           asm_driver_panic(d, "rv64 asm: c.addi16sp needs sp destination");
         return enc_c_addi16sp(m, imm);
       }
       return enc_c_ci(m, rd, imm);
 
     case RV64_FMT_CSS:
       rs2 = (desc->flags & RV64_ASMFL_FP) ? parse_freg(d) : parse_xreg(d);
       expect_comma(d);
       mem = parse_mem(d);
       if (mem.base != RV_SP)
         asm_driver_panic(d, "rv64 asm: compressed stack store needs sp base");
       return enc_c_swsp(m, rs2, (u32)mem.disp,
                         !slice_eq_cstr(desc->mnemonic, "c.swsp"));
 
     case RV64_FMT_CIW:
       rd = parse_xreg(d);
       expect_comma(d);
       rs1 = parse_xreg(d);
       expect_comma(d);
       if (rs1 != RV_SP)
         asm_driver_panic(d, "rv64 asm: c.addi4spn needs sp source");
       imm = (i32)asm_driver_parse_const(d);
       return enc_c_addi4spn(m, c_reg3(d, rd), (u32)imm);
 
     case RV64_FMT_CL:
       rd = (desc->flags & RV64_ASMFL_FP) ? parse_freg(d) : parse_xreg(d);
       expect_comma(d);
       mem = parse_mem(d);
       return enc_c_lwld(m, c_reg3(d, rd), c_reg3(d, mem.base), (u32)mem.disp,
                         !slice_eq_cstr(desc->mnemonic, "c.lw"));
 
     case RV64_FMT_CS:
       rs2 = (desc->flags & RV64_ASMFL_FP) ? parse_freg(d) : parse_xreg(d);
       expect_comma(d);
       mem = parse_mem(d);
       return enc_c_swld(m, c_reg3(d, rs2), c_reg3(d, mem.base), (u32)mem.disp,
                         !slice_eq_cstr(desc->mnemonic, "c.sw"));
 
     case RV64_FMT_CA:
       rd = parse_xreg(d);
       expect_comma(d);
       rs2 = parse_xreg(d);
       return m | (c_reg3(d, rd) << 7) | (c_reg3(d, rs2) << 2);
 
     case RV64_FMT_CB:
       rs1 = parse_xreg(d);
       expect_comma(d);
       imm = (i32)asm_driver_parse_const(d);
       if (slice_eq_cstr(desc->mnemonic, "c.beqz") ||
           slice_eq_cstr(desc->mnemonic, "c.bnez")) {
         return enc_c_cb_imm(m, c_reg3(d, rs1), imm);
       }
       return enc_c_cb_alu_imm(m, c_reg3(d, rs1), imm);
 
     case RV64_FMT_CJ:
       imm = (i32)asm_driver_parse_const(d);
       return enc_c_cj(m, imm);
 
     case RV64_FMT_C_NONE:
       return m;
 
     default:
       asm_driver_panic(d, "rv64 asm: unsupported format");
   }
 }
 
 /* ============================================================
  * Multi-word pseudo-instruction expansion.
  *
  * call/tail/la/lla expand to a PC-relative AUIPC + (JALR | ADDI) pair;
  * `li` with a constant that does not fit a 12-bit signed immediate
  * expands to an LUI/ADDI(W)/SLLI chain (no relocations). Each 32-bit
  * word goes out through rv64_emit32 — the same path assemble_one's
  * single-word result uses — and relocations are attached via
  * mc_emit_reloc_at at the appropriate word offset. */
 
 /* 12-bit signed immediate range check for li short-circuit. */
 static bool rv_fits_i12(i64 v) { return v >= -2048 && v <= 2047; }
 
 /* Sign-extend the low 12 bits of v. */
 static i64 rv_sext12(i64 v) {
   return (i64)((((u64)v & 0xfffu) ^ 0x800u)) - 0x800;
 }
 
 /* Emit an AUIPC rd,0 + a R_RV_PCREL_HI20(sym) reloc, then create a local
  * `.LpcrelHi` anchor at the AUIPC offset and return that anchor symbol so
  * the paired low-half reloc can reference it. Mirrors native.c's
  * rv_emit_global_addr (the non-GOT branch). */
 static ObjSymId rv_emit_pcrel_hi(AsmDriver* d, u32 rd, ObjSymId sym,
                                  i64 addend) {
   MCEmitter* mc = asm_driver_mc(d);
   ObjBuilder* obj = asm_driver_ob(d);
   Compiler* c = asm_driver_compiler(d);
   u32 sec = mc->section_id;
   u32 ap = mc_pos(mc);
   rv64_emit32(mc, rv_auipc(rd, 0));
   mc_emit_reloc_at(mc, sec, ap, R_RV_PCREL_HI20, sym, addend, 0, 0);
   Sym an = pool_intern_slice(c->global, SLICE_LIT(".LpcrelHi"));
   return obj_symbol(obj, an, SB_LOCAL, SK_OBJ, sec, (u64)ap, 0);
 }
 
 /* call/tail: AUIPC <link>,0 + JALR <rd>,<link>,0 with one R_RV_CALL reloc
  * at the AUIPC. `link` is the register the AUIPC materializes into and the
  * JALR's base; `rd` is the JALR link-register (ra for call, zero for
  * tail). The linker patches both words from the single R_RV_CALL reloc. */
 static void rv_emit_call_pseudo(AsmDriver* d, u32 link, u32 rd) {
   MCEmitter* mc = asm_driver_mc(d);
   ObjSymId sym = OBJ_SYM_NONE;
   i64 off = 0;
   asm_driver_parse_sym_expr(d, &sym, &off);
   if (sym == OBJ_SYM_NONE)
     asm_driver_panic(d, "rv64 asm: call/tail target must be a symbol");
   u32 sec = mc->section_id;
   u32 ap = mc_pos(mc);
   rv64_emit32(mc, rv_auipc(link, 0));
   rv64_emit32(mc, rv_jalr(rd, link, 0));
   mc_emit_reloc_at(mc, sec, ap, R_RV_CALL, sym, off, 0, 0);
 }
 
 /* la/lla rd, sym: AUIPC rd,%pcrel_hi(sym) + ADDI rd,rd,%pcrel_lo(anchor).
  * kit's static Local-Exec model has no GOT, so `la` == `lla`. */
 static void rv_emit_la_pseudo(AsmDriver* d) {
   MCEmitter* mc = asm_driver_mc(d);
   u32 rd = parse_xreg(d);
   expect_comma(d);
   ObjSymId sym = OBJ_SYM_NONE;
   i64 off = 0;
   asm_driver_parse_sym_expr(d, &sym, &off);
   if (sym == OBJ_SYM_NONE)
     asm_driver_panic(d, "rv64 asm: la/lla target must be a symbol");
   ObjSymId anchor = rv_emit_pcrel_hi(d, rd, sym, off);
   u32 sec = mc->section_id;
   u32 lp = mc_pos(mc);
   rv64_emit32(mc, rv_addi(rd, rd, 0));
   mc_emit_reloc_at(mc, sec, lp, R_RV_PCREL_LO12_I, anchor, 0, 0, 0);
 }
 
 /* LUI immediate that sign-extends to a negative 32-bit value: bit 19 of
  * the 20-bit field is set, i.e. Hi20 >= 0x80000. */
 #define RV_LUI_HI20_SIGN 0x80000LL
 
 /* Materialize a constant into `rd` via the LLVM RISCVMatInt sequence: for
  * values fitting a signed 32-bit range, LUI + ADDI/ADDIW; otherwise a
  * recursive top-down hi20/lo12 split with SLLI shifts that absorb trailing
  * zeros. No relocations.
  *
  * On rv64, after an LUI the low-half add uses ADDIW only when the LUI value
  * is negative in 32-bit form (Hi20 >= RV_LUI_HI20_SIGN): there the add must
  * wrap in 32-bit arithmetic and re-sign-extend to land in range. When the
  * LUI value is non-negative in its low 32 bits, plain ADDI keeps the
  * 64-bit result correct (matching LLVM's generateInstSeqImpl).
  *
  * On rv32 there is no ADDIW and the GPR is 32 bits wide, so every constant
  * fits a LUI + ADDI pair (the add already wraps mod 2^32). The variant's
  * has_w_forms gates both the ADDIW use and the >32-bit recursion below. */
 static void rv_emit_li_value(MCEmitter* mc, const RiscvVariant* variant, u32 rd,
                              i64 val) {
   if (!variant->has_w_forms || (val >= -2147483648LL && val <= 2147483647LL)) {
     i64 hi20 = ((val + 0x800) >> 12) & 0xfffffLL;
     i64 lo12 = rv_sext12(val);
     if (hi20) rv64_emit32(mc, rv_lui(rd, (u32)hi20));
     if (lo12 || hi20 == 0) {
       u32 src = hi20 ? rd : (u32)RV_ZERO;
       if (variant->has_w_forms && hi20 >= RV_LUI_HI20_SIGN)
         rv64_emit32(mc, rv_addiw(rd, src, (i32)lo12));
       else
         rv64_emit32(mc, rv_addi(rd, src, (i32)lo12));
     }
     return;
   }
   /* >32-bit: split off the low 12 bits, recurse on the (shifted) high
    * part, then SLLI back and ADD the low bits. The subtraction is done in
    * unsigned space so it cannot signed-overflow at the int64 extremes
    * (e.g. val=INT64_MAX, lo12=-1); the result has its low 12 bits clear,
    * and the arithmetic right shift recovers the sign-extended high part. */
   i64 lo12 = rv_sext12(val);
   i64 hi = (i64)((u64)val - (u64)lo12) >> 12;
   u32 shift = 12;
   /* Absorb trailing zeros of the high part into the shift amount. */
   while ((hi & 1) == 0) {
     hi >>= 1;
     ++shift;
   }
   rv_emit_li_value(mc, variant, rd, hi);
   rv64_emit32(mc, rv_slli(rd, rd, shift));
   if (lo12) rv64_emit32(mc, rv_addi(rd, rd, (i32)lo12));
 }
 
 /* Dispatch a multi-word pseudo. Returns true if it consumed the operands
  * and emitted its expansion; false to fall through to the single-word
  * path. `li` is handled here only when its immediate exceeds the 12-bit
  * signed range the alias row encodes directly. */
 static bool rv64_emit_pseudo(AsmDriver* d, const Rv64InsnDesc* desc) {
   MCEmitter* mc = asm_driver_mc(d);
   if (desc->fmt == RV64_FMT_PSEUDO) {
     if (slice_eq_cstr(desc->mnemonic, "call")) {
       rv_emit_call_pseudo(d, RV_RA, RV_RA);
       return true;
     }
     if (slice_eq_cstr(desc->mnemonic, "tail")) {
       /* Standard RISC-V `tail` materializes the address into t1 (x6). kit
        * codegen uses t0 for its own tail-call temp, so a `cc -S`-fused
        * `tail sym` re-assembles to t1 not t0 — execution-equivalent (both are
        * caller-saved temps clobbered by the tail jump; cross-exec still
        * matches), only the byte image differs on tail-call cases. Keeping the
        * assembler's `tail` standard preserves clang/gas interop. */
       rv_emit_call_pseudo(d, RV_T1, RV_ZERO);
       return true;
     }
     /* la / lla — identical PC-relative expansion in kit. */
     rv_emit_la_pseudo(d);
     return true;
   }
   if ((desc->flags & RV64_ASMFL_ALIAS) && slice_eq_cstr(desc->mnemonic, "li")) {
     /* Peek the immediate without consuming the destination register: the
      * single-word alias path re-parses both. We commit to the multi-word
      * path only for out-of-range constants, leaving the existing 12-bit
      * fast path (and its golden behavior) untouched. */
     u32 rd = parse_xreg(d);
     expect_comma(d);
     i64 imm = asm_driver_parse_const(d);
     if (rv_fits_i12(imm)) {
       rv64_emit32(mc, rv_addi(rd, RV_ZERO, (i32)imm));
     } else {
       rv_emit_li_value(mc, rv_asm_variant(d), rd, imm);
     }
     return true;
   }
   return false;
 }
 
 static void rv64_arch_asm_insn(ArchAsm* base, AsmDriver* d, Sym mnemonic) {
   MCEmitter* mc = asm_driver_mc(d);
   const Rv64InsnDesc* desc;
   u8 av = rv_asm_variant(d)->xlen == 32u ? (u8)RV_AV_RV32 : (u8)RV_AV_RV64;
   (void)base;
   (void)asm_driver_cur_section(d);
   desc = rv64_asm_find(pool_slice(asm_driver_pool(d), mnemonic), av);
   if (!desc)
     asm_driver_panic(d, av == (u8)RV_AV_RV32
                             ? "rv32 asm: unsupported instruction"
                             : "rv64 asm: unsupported instruction");
   if (rv64_emit_pseudo(d, desc)) return;
   if (desc->flags & RV64_ASMFL_C16)
     rv64_emit16(mc, assemble_one(d, desc));
   else
     rv64_emit32(mc, assemble_one(d, desc));
 }
 
 static void rv64_arch_asm_destroy(ArchAsm* base) { (void)base; }
 
 /* ---- textual-assembly operand syntax (printer <-> parser) ----------------
  *
  * Inverse of the `.s` parsers above (rv_parse_mod_reloc / rv_reloc_target and
  * the call/la pseudo expanders): how a relocated rv64 operand is spelled in
  * `cc -S` so the same text re-assembles under kit-as. RISC-V uses the same
  * `%hi`/`%lo`/`%pcrel_hi`/`%pcrel_lo` operator syntax on every object format,
  * so `fmt` is unused. See ArchAsmOps and src/api/asm_emit.c. */
 static int rv64_reloc_operand(u16 kind, KitObjFmt fmt, ArchRelocOperand* out) {
   (void)fmt;
   out->prefix = "";
   out->suffix = "";
   out->addend_bias = 0;
   out->emit_anchor = 0;
   out->ref_anchor = 0;
   switch (kind) {
     case R_RV_PCREL_HI20:
       out->surg = ARCH_RELOC_SURG_TAIL;
       out->prefix = "%pcrel_hi(";
       out->suffix = ")";
       out->emit_anchor = 1; /* define a unique anchor label at this AUIPC */
       return 1;
     case R_RV_GOT_HI20:
       out->surg = ARCH_RELOC_SURG_TAIL;
       out->prefix = "%got_pcrel_hi(";
       out->suffix = ")";
       out->emit_anchor = 1;
       return 1;
     case R_RV_PCREL_LO12_I:
     case R_RV_PCREL_LO12_S:
       out->surg = ARCH_RELOC_SURG_RV_LO12;
       out->prefix = "%pcrel_lo(";
       out->suffix = ")";
       out->ref_anchor = 1; /* references the preceding AUIPC's anchor label */
       return 1;
     case R_RV_HI20:
       out->surg = ARCH_RELOC_SURG_TAIL;
       out->prefix = "%hi(";
       out->suffix = ")";
       return 1;
     case R_RV_LO12_I:
     case R_RV_LO12_S:
       out->surg = ARCH_RELOC_SURG_RV_LO12;
       out->prefix = "%lo(";
       out->suffix = ")";
       return 1;
     case R_RV_BRANCH:
     case R_RV_JAL:
       out->surg = ARCH_RELOC_SURG_TAIL;
       return 1;
     default:
       return 0; /* R_ABS*, R_RV_RVC_*, R_RV_RELAX, TLS, ... → keep numeric */
   }
 }
 
 /* Intra-section local branches whose target codegen resolved in place (no
  * relocation): the disassembler renders the target numerically, so cc -S
  * synthesizes a label there. `j`/`jal x0` are JAL aliases; the conditional
  * branches are B-type. `call`/`tail` are excluded — they carry R_RV_CALL. */
 static int rv64_is_local_branch(KitSlice m) {
   if (m.len == 1 && m.s[0] == 'j') return 1;
   if (m.len == 3 && memcmp(m.s, "jal", 3) == 0) return 1;
   if (m.len == 3 && memcmp(m.s, "beq", 3) == 0) return 1;
   if (m.len == 3 && memcmp(m.s, "bne", 3) == 0) return 1;
   if (m.len == 3 && memcmp(m.s, "blt", 3) == 0) return 1;
   if (m.len == 3 && memcmp(m.s, "bge", 3) == 0) return 1;
   if (m.len == 4 && memcmp(m.s, "bltu", 4) == 0) return 1;
   if (m.len == 4 && memcmp(m.s, "bgeu", 4) == 0) return 1;
   if (m.len == 4 && memcmp(m.s, "beqz", 4) == 0) return 1;
   if (m.len == 4 && memcmp(m.s, "bnez", 4) == 0) return 1;
   if (m.len == 4 && memcmp(m.s, "blez", 4) == 0) return 1;
   if (m.len == 4 && memcmp(m.s, "bgez", 4) == 0) return 1;
   if (m.len == 4 && memcmp(m.s, "bltz", 4) == 0) return 1;
   if (m.len == 4 && memcmp(m.s, "bgtz", 4) == 0) return 1;
   if (m.len == 6 && memcmp(m.s, "c.beqz", 6) == 0) return 1;
   if (m.len == 6 && memcmp(m.s, "c.bnez", 6) == 0) return 1;
   if (m.len == 3 && memcmp(m.s, "c.j", 3) == 0) return 1;
   return 0;
 }
 
 /* R_RV_CALL fuses an AUIPC+JALR pair into a single `call`/`tail sym` pseudo
  * (the canonical `.s` spelling the assembler re-expands to the same pair +
  * reloc). The reloc sits on the AUIPC; the JALR partner carries no reloc. A
  * tail call links into x0 (the JALR's rd is `zero`); a regular call links into
  * ra. We read that from the partner JALR's disassembled text. */
 static int rv64_reloc_call_pair(u16 kind, KitSlice pair_mnemonic,
                                 KitSlice pair_ops, const char** mnemonic_out) {
   if (kind != R_RV_CALL) return 0;
   /* The partner JALR links into ra (regular call) or x0 (tail). The
    * disassembler renders the x0-link, zero-immediate form as the `jr rs`
    * alias, and the ra form as `jalr ra, 0(ra)`. So a `jr` partner is always a
    * tail; a `jalr` partner is a tail iff its link register is `zero`. */
   if (pair_mnemonic.len == 2 && memcmp(pair_mnemonic.s, "jr", 2) == 0) {
     *mnemonic_out = "tail";
     return 1;
   }
   if (pair_mnemonic.len == 4 && memcmp(pair_mnemonic.s, "jalr", 4) == 0) {
     if (pair_ops.len >= 4 && memcmp(pair_ops.s, "zero", 4) == 0)
       *mnemonic_out = "tail";
     else
       *mnemonic_out = "call";
     return 1;
   }
   return 0;
 }
 
 const ArchAsmOps rv64_asm_ops = {
     .reloc_operand = rv64_reloc_operand,
     .is_local_branch = rv64_is_local_branch,
     .reloc_call_pair = rv64_reloc_call_pair,
 };
 
 ArchAsm* rv64_arch_asm_new(Compiler* c) {
   Rv64Asm* a = arena_new(c->tu, Rv64Asm);
   memset(a, 0, sizeof *a);
   a->base.insn = rv64_arch_asm_insn;
   a->base.destroy = rv64_arch_asm_destroy;
   a->c = c;
   return &a->base;
 }
 
 /* ============================================================
  * Inline-asm template walker (parallel to aa64 asm.c §"inline-asm
  * template walker"). The walker substitutes %N / %[name] / %% / %a%w%x
  * placeholders into a per-line StrBuf, then re-lexes each line through
  * rv64_arch_asm_insn for assembly. Statement separators recognised are
  * '\n' and ';' (outside parens / quoted strings).
  * ============================================================ */
 
 Rv64Asm* rv64_asm_open(Compiler* c) {
   Rv64Asm* a = arena_new(c->tu, Rv64Asm);
   memset(a, 0, sizeof *a);
   a->base.insn = rv64_arch_asm_insn;
   a->base.destroy = rv64_arch_asm_destroy;
   a->c = c;
   return a;
 }
 
 void rv64_asm_close(Rv64Asm* a) { (void)a; }
 
 void rv64_inline_bind(Rv64Asm* a, const AsmConstraint* outs, u32 nout,
                       Operand* out_ops, const AsmConstraint* ins, u32 nin,
                       const Operand* in_ops, const Sym* clobbers, u32 nclob) {
   a->outs = outs;
   a->out_ops = out_ops;
   a->ins = ins;
   a->in_ops = in_ops;
   a->clobbers = clobbers;
   a->nout = nout;
   a->nin = nin;
   a->nclob = nclob;
 }
 
 /* Per-line rendered buffer cap. Inline asm rarely emits more than a
  * handful of insns per block; one substituted line fits comfortably.
  * Truncation panics — the operator grammar should never grow a single
  * line beyond this without a deliberate reason. */
 #define RV64_INLINE_LINE_CAP 1024
 
 _Noreturn static void inline_panic(Rv64Asm* a, const char* msg) {
   SrcLoc loc = {0, 0, 0};
   compiler_panic(a->c, loc, "rv64 inline asm: %.*s",
                  SLICE_ARG(slice_from_cstr(msg)));
 }
 
 /* Render a 5-bit integer register number using its canonical psABI name. */
 static void render_xreg(StrBuf* sb, u32 reg) {
   const char* nm = rv64_register_name(reg & 0x1fu);
   if (!nm) {
     strbuf_putc(sb, 'x');
     if ((reg & 0x1fu) >= 10u)
       strbuf_putc(sb, (char)('0' + ((reg & 0x1fu) / 10u)));
     strbuf_putc(sb, (char)('0' + ((reg & 0x1fu) % 10u)));
     return;
   }
   strbuf_puts(sb, nm);
 }
 
 /* Render an FP register by its canonical psABI name (e.g., fa0). */
 static void render_freg(StrBuf* sb, u32 reg) {
   const char* nm = rv64_register_name(32u + (reg & 0x1fu));
   if (!nm) {
     strbuf_putc(sb, 'f');
     if ((reg & 0x1fu) >= 10u)
       strbuf_putc(sb, (char)('0' + ((reg & 0x1fu) / 10u)));
     strbuf_putc(sb, (char)('0' + ((reg & 0x1fu) % 10u)));
     return;
   }
   strbuf_puts(sb, nm);
 }
 
 /* Render a signed 64-bit integer. Inline asm immediates appear bare in
  * RISC-V (no '#' prefix), matching the standalone .s parser. */
 static void render_imm(StrBuf* sb, i64 v) { strbuf_put_i64(sb, v); }
 
 /* Render addressing form `disp(base)`. */
 static void render_indirect(Rv64Asm* a, StrBuf* sb, Reg base, i32 ofs) {
   (void)a;
   if (ofs != 0)
     strbuf_put_i64(sb, (i64)ofs);
   else
     strbuf_putc(sb, '0');
   strbuf_putc(sb, '(');
   render_xreg(sb, (u32)base);
   strbuf_putc(sb, ')');
 }
 
 /* Resolve operand index → render into sb. form:
  *   0 = default (per-kind),
  *   1 = %wN (width hint; on rv64 same as default xreg form),
  *   2 = %xN (force 64-bit reg form — identical to default for rv64),
  *   3 = %aN (memory addressing form).
  *   4 = %zN (RISC-V GCC: emits "zero" if operand is imm 0, else reg). */
 static void render_operand(Rv64Asm* a, StrBuf* sb, u32 idx, int form) {
   u32 ntot = a->nout + a->nin;
   if (idx >= ntot) inline_panic(a, "operand index out of range");
   const Operand* op =
       (idx < a->nout) ? &a->out_ops[idx] : &a->in_ops[idx - a->nout];
   switch (form) {
     case 1: /* %wN — accept any reg/imm; rv64 has no narrower spelling. */
     case 2: /* %xN — same. */
       if (op->kind == RV64_INLINE_OPK_REG) {
         if (op->pad[0] == RV64_INLINE_OPCLS_FP)
           render_freg(sb, (u32)op->v.local);
         else
           render_xreg(sb, (u32)op->v.local);
         return;
       }
       if (op->kind == OPK_IMM) {
         render_imm(sb, op->v.imm);
         return;
       }
       inline_panic(a, "%w/%x on unsupported operand kind");
     case 3: /* %aN — memory addressing form */
       if (op->kind != OPK_INDIRECT) inline_panic(a, "%a on non-memory operand");
       if (op->v.ind.index != CG_LOCAL_NONE)
         inline_panic(a,
                      "%a on indexed memory operand: rv64 inline asm "
                      "requires base+disp only");
       render_indirect(a, sb, (Reg)op->v.ind.base, op->v.ind.ofs);
       return;
     case 4: /* %zN — zero-or-reg */
       if (op->kind == OPK_IMM && op->v.imm == 0) {
         strbuf_puts(sb, "zero");
         return;
       }
       if (op->kind == RV64_INLINE_OPK_REG) {
         if (op->pad[0] == RV64_INLINE_OPCLS_FP)
           render_freg(sb, (u32)op->v.local);
         else
           render_xreg(sb, (u32)op->v.local);
         return;
       }
       inline_panic(a, "%z on unsupported operand kind");
     default:
       break;
   }
   switch (op->kind) {
     case RV64_INLINE_OPK_REG:
       if (op->pad[0] == RV64_INLINE_OPCLS_FP)
         render_freg(sb, (u32)op->v.local);
       else
         render_xreg(sb, (u32)op->v.local);
       return;
     case OPK_IMM:
       render_imm(sb, op->v.imm);
       return;
     case OPK_INDIRECT:
       if (op->v.ind.index != CG_LOCAL_NONE)
         inline_panic(a,
                      "indexed memory operand in inline asm: rv64 requires "
                      "base+disp only");
       render_indirect(a, sb, (Reg)op->v.ind.base, op->v.ind.ofs);
       return;
     default:
       inline_panic(a, "unsupported operand kind for %N");
   }
 }
 
 /* Resolve a `%[name]` operand by looking up `needle` against the
  * constraint.name fields on the combined outs+ins list. Returns the
  * combined index, or (u32)-1 on miss. */
 static u32 lookup_named(Rv64Asm* a, Sym needle) {
   for (u32 k = 0; k < a->nout; ++k) {
     if (a->outs[k].name == needle) return k;
   }
   for (u32 k = 0; k < a->nin; ++k) {
     if (a->ins[k].name == needle) return a->nout + k;
   }
   return (u32)-1;
 }
 
 /* Lex one line of substituted asm and dispatch via rv64_arch_asm_insn. */
 static void run_one_line(Rv64Asm* a, MCEmitter* mc, const char* text,
                          size_t len) {
   /* Skip blank lines. */
   size_t i;
   for (i = 0; i < len; ++i) {
     if (text[i] != ' ' && text[i] != '\t') break;
   }
   if (i == len) return;
 
   AsmLexer* lx = asm_lex_open_mem(a->c, "<inline-asm>", text, len);
   AsmDriver* d = asm_driver_open_inline(a->c, mc, lx);
 
   /* The first non-trivial token must be the mnemonic identifier. */
   AsmTok t = asm_driver_peek(d);
   while (t.kind == ASM_TOK_NEWLINE) {
     (void)asm_driver_next(d);
     t = asm_driver_peek(d);
   }
   if (t.kind == ASM_TOK_EOF) {
     asm_driver_close_inline(d);
     asm_lex_close(lx);
     return;
   }
   if (t.kind != ASM_TOK_IDENT)
     inline_panic(a, "expected mnemonic at start of inline asm line");
   (void)asm_driver_next(d);
   Sym mn = t.v.ident;
   /* Compose `fcvt.s.w` etc. — rv64 has dotted mnemonics; the standalone
    * lexer already strings them together as a single IDENT in most paths.
    * Mirror the aa64 composite handling for safety. */
   AsmTok dot = asm_driver_peek(d);
   while (asm_driver_tok_is_punct(dot, '.')) {
     (void)asm_driver_next(d);
     AsmTok rest = asm_driver_next(d);
     if (rest.kind != ASM_TOK_IDENT)
       inline_panic(a, "composite mnemonic: expected ident after '.'");
     Slice hsl = pool_slice(asm_driver_pool(d), mn);
     Slice rsl = pool_slice(asm_driver_pool(d), rest.v.ident);
     size_t hn = hsl.len, rn = rsl.len;
     char buf[64];
     if (hn + 1 + rn >= sizeof buf)
       inline_panic(a, "composite mnemonic too long");
     for (size_t k = 0; k < hn; ++k) buf[k] = hsl.s[k];
     buf[hn] = '.';
     for (size_t k = 0; k < rn; ++k) buf[hn + 1 + k] = rsl.s[k];
     mn = pool_intern_slice(asm_driver_pool(d),
                            (Slice){.s = buf, .len = hn + 1 + rn});
     dot = asm_driver_peek(d);
   }
   rv64_arch_asm_insn(&a->base, d, mn);
   asm_driver_close_inline(d);
   asm_lex_close(lx);
 }
 
 /* Substitute placeholders into one line's StrBuf, then dispatch. */
 static void render_and_run_line(Rv64Asm* a, MCEmitter* mc, StrBuf* sb,
                                 const char* start, const char* end) {
   strbuf_reset(sb);
   for (const char* p = start; p < end; ++p) {
     char c = *p;
     if (c != '%') {
       strbuf_putc(sb, c);
       continue;
     }
     /* Placeholder. */
     if (p + 1 >= end) inline_panic(a, "trailing '%' in template");
     char n = *(p + 1);
     if (n == '%') {
       strbuf_putc(sb, '%');
       ++p;
       continue;
     }
     if (n == '[') {
       const char* nbeg = p + 2;
       const char* nend = nbeg;
       while (nend < end && *nend != ']') ++nend;
       if (nend == end) inline_panic(a, "unterminated %[name]");
       size_t nlen = (size_t)(nend - nbeg);
       Sym needle =
           pool_intern_slice(a->c->global, (Slice){.s = nbeg, .len = nlen});
       u32 idx = lookup_named(a, needle);
       if (idx == (u32)-1)
         inline_panic(a, "%[name] does not match any constraint");
       p = nend; /* loop's ++p steps past the ']' */
       render_operand(a, sb, idx, 0);
       continue;
     }
     int form = 0; /* 0=default, 1=w, 2=x, 3=a, 4=z */
     if (n == 'w' || n == 'x' || n == 'a' || n == 'z') {
       form = (n == 'w') ? 1 : (n == 'x') ? 2 : (n == 'a') ? 3 : 4;
       ++p;
       if (p + 1 >= end) inline_panic(a, "trailing '%' modifier in template");
       n = *(p + 1);
     }
     if (n == '[') {
       const char* nbeg = p + 2;
       const char* nend = nbeg;
       while (nend < end && *nend != ']') ++nend;
       if (nend == end) inline_panic(a, "unterminated %[name]");
       size_t nlen = (size_t)(nend - nbeg);
       Sym needle =
           pool_intern_slice(a->c->global, (Slice){.s = nbeg, .len = nlen});
       u32 idx = lookup_named(a, needle);
       if (idx == (u32)-1)
         inline_panic(a, "%[name] does not match any constraint");
       p = nend;
       render_operand(a, sb, idx, form);
       continue;
     }
     if (n < '0' || n > '9') inline_panic(a, "expected digit after '%'");
     u32 idx = (u32)(n - '0');
     ++p;
     /* GCC syntax permits up to two digits (%0..%99). */
     if (p + 1 < end && *(p + 1) >= '0' && *(p + 1) <= '9') {
       idx = idx * 10 + (u32)(*(p + 1) - '0');
       ++p;
     }
     render_operand(a, sb, idx, form);
   }
   if (sb->truncated) inline_panic(a, "inline asm line buffer overflow");
   run_one_line(a, mc, strbuf_cstr(sb), strbuf_len(sb));
 }
 
 void rv64_asm_run_template(Rv64Asm* a, MCEmitter* mc, const char* tmpl) {
   if (!tmpl || !*tmpl) return;
 
   char buf[RV64_INLINE_LINE_CAP];
   StrBuf sb;
   strbuf_init(&sb, buf, sizeof buf);
 
   /* Walk tmpl, splitting on '\n' and ';'. Track paren depth and quote
    * state so that a literal ';' inside `( ... )` (memory operand) or a
    * quoted string is not mistaken for a statement separator. RISC-V uses
    * `disp(base)` for memory, hence we track parens. */
   const char* line_start = tmpl;
   int paren = 0;
   char quote = 0;
   for (const char* p = tmpl;; ++p) {
     char c = *p;
     if (c == '\0') {
       render_and_run_line(a, mc, &sb, line_start, p);
       break;
     }
     if (quote) {
       if (c == '\\' && *(p + 1)) {
         ++p;
         continue;
       }
       if (c == quote) quote = 0;
       continue;
     }
     if (c == '"' || c == '\'') {
       quote = c;
       continue;
     }
     if (c == '(') {
       ++paren;
       continue;
     }
     if (c == ')') {
       if (paren) --paren;
       continue;
     }
     if (paren == 0 && (c == '\n' || c == ';')) {
       render_and_run_line(a, mc, &sb, line_start, p);
       line_start = p + 1;
     }
   }
 }