kit

asm.c (110411B)

---

 /* AArch64 standalone .s instruction parser.
  *
  * Per-mnemonic dispatch: each entry in the mnemonic table names a
  * parse function that reads operand tokens through the asm-driver
  * surface and emits the encoded word via the inline encoders in
  * aa64_isa.h.  Encoders are the single source of truth for bit
  * layout — the disassembler shares them through aa64_*_unpack.
  *
  * Aliases (`mov`, `neg`, `cmp`, `mul`, ...) live in this table as
  * dedicated rows that pick the canonical form's encoder with the
  * alias-specific operand shape.  When a mnemonic admits multiple
  * forms (e.g. `mov` register-vs-immediate, `add` register-vs-
  * immediate), the parser branches on operand shape after reading
  * the first non-Rd operand. */
 
 #include "arch/aa64/asm.h"
 
 #include <string.h>
 
 #include "arch/aa64/isa.h"
 #include "arch/aa64/regs.h"
 #include "arch/arch.h"
 #include "asm/asm_helpers.h"
 #include "asm/asm_lex.h"
 #include "cg/type.h"
 #include "core/arena.h"
 #include "core/pool.h"
 #include "core/slice.h"
 #include "core/strbuf.h"
 #include "obj/obj.h"
 
 /* ---- public handle ---- */
 
 struct AA64Asm {
   ArchAsm base;
   Compiler* c;
 
   /* Inline-asm bound state (set by aa64_inline_bind, cleared otherwise).
    * Operand indexing per GCC convention: 0..nout-1 are outputs, then
    * nout..nout+nin-1 are inputs.  Templates address into this combined
    * list via %N / %wN / %xN / %aN.  out_ops is mutable (the binder fills
    * in result locations); in_ops + constraints + clobbers are read-only
    * borrows. */
   const AsmConstraint* outs;
   Operand* out_ops;
   const AsmConstraint* ins;
   const Operand* in_ops;
   const Sym* clobbers;
   u32 nout;
   u32 nin;
   u32 nclob;
 };
 
 static void aa64_arch_asm_insn(ArchAsm* base, AsmDriver* d, Sym mnemonic);
 static void aa64_arch_asm_destroy(ArchAsm* base);
 
 AA64Asm* aa64_asm_open(Compiler* c) {
   AA64Asm* a = arena_new(c->tu, AA64Asm);
   memset(a, 0, sizeof *a);
   a->base.insn = aa64_arch_asm_insn;
   a->base.destroy = aa64_arch_asm_destroy;
   a->c = c;
   return a;
 }
 
 void aa64_asm_close(AA64Asm* a) { (void)a; }
 
 ArchAsm* aa64_arch_asm_new(Compiler* c) { return &aa64_asm_open(c)->base; }
 
 static void aa64_arch_asm_insn(ArchAsm* base, AsmDriver* d, Sym mnemonic) {
   aa64_asm_insn((AA64Asm*)base, d, mnemonic);
 }
 
 static void aa64_arch_asm_destroy(ArchAsm* base) {
   aa64_asm_close((AA64Asm*)base);
 }
 
 void aa64_inline_bind(AA64Asm* a, const AsmConstraint* outs, u32 nout,
                       Operand* out_ops, const AsmConstraint* ins, u32 nin,
                       const Operand* in_ops, const Sym* clobbers, u32 nclob) {
   a->outs = outs;
   a->out_ops = out_ops;
   a->ins = ins;
   a->in_ops = in_ops;
   a->clobbers = clobbers;
   a->nout = nout;
   a->nin = nin;
   a->nclob = nclob;
 }
 
 /* ---- helpers ---- */
 
 static int tok_punct(AsmTok t, u32 p) { return asm_driver_tok_is_punct(t, p); }
 
 static int icase_eq(const char* a, size_t an, const char* b) {
   size_t i;
   for (i = 0; i < an; ++i) {
     char x = a[i], y = b[i];
     if (x >= 'A' && x <= 'Z') x = (char)(x + ('a' - 'A'));
     if (y >= 'A' && y <= 'Z') y = (char)(y + ('a' - 'A'));
     if (x != y || !y) return 0;
   }
   return b[an] == '\0';
 }
 
 /* Parse a register operand.  Returns the 5-bit encoded register number
  * via *reg_out and the form via *is64_out.  Recognized forms (case-
  * insensitive):
  *   w0..w30, wzr            → is64=0, reg=0..30 / 31
  *   x0..x30, xzr, lr (=x30) → is64=1, reg=0..30 / 31
  *   sp                      → is64=1, reg=31  (sp_means_sp set)
  *   wsp                     → is64=0, reg=31  (sp_means_sp set)
  * Aliases:
  *   fp = x29
  *   ip0 = x16, ip1 = x17  (PLT scratch — useful for hand-written PLTs) */
 typedef struct AA64Reg {
   u32 num;
   u8 is64;
   u8 is_sp;    /* 1 if the spelling was "sp" / "wsp" */
   u8 is_fp;    /* 1 for SIMD/FP register spellings accepted in FP forms */
   u8 fp_bytes; /* 8 for Dn, 16 for Qn */
 } AA64Reg;
 
 static int parse_reg_from_ident(AsmDriver* d, Sym ident, AA64Reg* out) {
   Slice sl = pool_slice(asm_driver_pool(d), ident);
   const char* p = sl.s;
   size_t n = sl.len;
   if (!p || !n) return 0;
   /* "sp" */
   if (icase_eq(p, n, "sp")) {
     out->num = 31;
     out->is64 = 1;
     out->is_sp = 1;
     out->is_fp = 0;
     return 1;
   }
   if (icase_eq(p, n, "wsp")) {
     out->num = 31;
     out->is64 = 0;
     out->is_sp = 1;
     out->is_fp = 0;
     return 1;
   }
   if (icase_eq(p, n, "lr")) {
     out->num = 30;
     out->is64 = 1;
     out->is_sp = 0;
     out->is_fp = 0;
     return 1;
   }
   if (icase_eq(p, n, "fp")) {
     out->num = 29;
     out->is64 = 1;
     out->is_sp = 0;
     out->is_fp = 0;
     return 1;
   }
   if (icase_eq(p, n, "ip0")) {
     out->num = 16;
     out->is64 = 1;
     out->is_sp = 0;
     out->is_fp = 0;
     return 1;
   }
   if (icase_eq(p, n, "ip1")) {
     out->num = 17;
     out->is64 = 1;
     out->is_sp = 0;
     out->is_fp = 0;
     return 1;
   }
   if (icase_eq(p, n, "xzr")) {
     out->num = 31;
     out->is64 = 1;
     out->is_sp = 0;
     out->is_fp = 0;
     return 1;
   }
   if (icase_eq(p, n, "wzr")) {
     out->num = 31;
     out->is64 = 0;
     out->is_sp = 0;
     out->is_fp = 0;
     return 1;
   }
   /* W/X<num> */
   if ((p[0] == 'w' || p[0] == 'W' || p[0] == 'x' || p[0] == 'X') && n >= 2) {
     u32 r = 0;
     size_t i;
     for (i = 1; i < n; ++i) {
       char c = p[i];
       if (c < '0' || c > '9') return 0;
       r = r * 10 + (u32)(c - '0');
       if (r > 31) return 0;
     }
     out->num = r;
     out->is64 = (p[0] == 'x' || p[0] == 'X') ? 1 : 0;
     out->is_sp = 0;
     out->is_fp = 0;
     return 1;
   }
   return 0;
 }
 
 static int parse_fp_pair_reg_from_ident(AsmDriver* d, Sym ident, AA64Reg* out) {
   Slice sl = pool_slice(asm_driver_pool(d), ident);
   const char* p = sl.s;
   size_t n = sl.len;
   if (!p || n < 2 || (p[0] != 'd' && p[0] != 'D' && p[0] != 'q' && p[0] != 'Q'))
     return 0;
   u32 r = 0;
   for (size_t i = 1; i < n; ++i) {
     char c = p[i];
     if (c < '0' || c > '9') return 0;
     r = r * 10 + (u32)(c - '0');
     if (r > 31) return 0;
   }
   out->num = r;
   out->is64 = 1;
   out->is_sp = 0;
   out->is_fp = 1;
   out->fp_bytes = (p[0] == 'q' || p[0] == 'Q') ? 16u : 8u;
   return 1;
 }
 
 /* Scalar SIMD/FP transfer register for ldr/str/ldur/stur: b/h/s/d/q with the
  * access width in fp_bytes (1/2/4/8/16). Unlike parse_fp_pair_reg_from_ident
  * (ldp/stp, d/q only) this accepts the sub-64-bit scalar widths a single-reg
  * FP load/store can carry. */
 static int parse_fp_scalar_reg_from_ident(AsmDriver* d, Sym ident,
                                           AA64Reg* out) {
   Slice sl = pool_slice(asm_driver_pool(d), ident);
   const char* p = sl.s;
   size_t n = sl.len;
   u8 bytes;
   u32 r = 0;
   size_t i;
   if (!p || n < 2) return 0;
   switch (p[0]) {
     case 'b':
     case 'B':
       bytes = 1;
       break;
     case 'h':
     case 'H':
       bytes = 2;
       break;
     case 's':
     case 'S':
       bytes = 4;
       break;
     case 'd':
     case 'D':
       bytes = 8;
       break;
     case 'q':
     case 'Q':
       bytes = 16;
       break;
     default:
       return 0;
   }
   for (i = 1; i < n; ++i) {
     char c = p[i];
     if (c < '0' || c > '9') return 0;
     r = r * 10 + (u32)(c - '0');
     if (r > 31) return 0;
   }
   out->num = r;
   out->is64 = 1;
   out->is_sp = 0;
   out->is_fp = 1;
   out->fp_bytes = bytes;
   return 1;
 }
 
 static AA64Reg parse_reg(AsmDriver* d) {
   AsmTok t = asm_driver_next(d);
   AA64Reg r;
   memset(&r, 0, sizeof r);
   if (t.kind != ASM_TOK_IDENT || !parse_reg_from_ident(d, t.v.ident, &r))
     asm_driver_panic(d, "asm: expected register");
   return r;
 }
 
 /* Non-consuming lookahead: is the next operand a register? Used to pick
  * between the register and immediate forms of dual-form mnemonics (e.g.
  * `and Rd,Rn,Rm` vs `and Rd,Rn,#imm`). */
 static int peek_is_reg(AsmDriver* d) {
   AsmTok t = asm_driver_peek(d);
   AA64Reg r;
   return t.kind == ASM_TOK_IDENT && parse_reg_from_ident(d, t.v.ident, &r);
 }
 
 /* ldr/str transfer register: GPR (Wt/Xt) or scalar SIMD/FP (Bt..Qt). */
 static AA64Reg parse_ldst_reg(AsmDriver* d) {
   AsmTok t = asm_driver_next(d);
   AA64Reg r;
   memset(&r, 0, sizeof r);
   if (t.kind != ASM_TOK_IDENT ||
       (!parse_reg_from_ident(d, t.v.ident, &r) &&
        !parse_fp_scalar_reg_from_ident(d, t.v.ident, &r)))
     asm_driver_panic(d, "asm: ldr/str: expected register");
   return r;
 }
 
 /* Resolve the (size, V, opc, scale) load/store encoding fields from the
  * transfer register and mnemonic flavor. GPR width comes from fixed_size (the
  * sized mnemonics ldrb/ldrsw/…) or the register; FP uses V=1 with size/opc
  * keyed on the scalar width (b/h/s/d = size 0/1/2/3, opc store=0/load=1; the
  * 128-bit q is size=0 opc=2/3). `scale` is the byte access width for the
  * scaled unsigned-imm12 form. */
 typedef struct {
   u32 size, V, opc, scale;
 } AA64LdStEnc;
 
 static AA64LdStEnc ldst_encoding(AsmDriver* d, AA64Reg rt, int is_load,
                                  int fixed_size, int sign_ext) {
   AA64LdStEnc e;
   if (rt.is_fp) {
     if (fixed_size >= 0 || sign_ext)
       asm_driver_panic(d, "asm: sized/signed ld/st takes a GPR, not an FP reg");
     e.V = 1;
     e.scale = rt.fp_bytes;
     e.size = (rt.fp_bytes == 1)   ? 0u
              : (rt.fp_bytes == 2) ? 1u
              : (rt.fp_bytes == 4) ? 2u
              : (rt.fp_bytes == 8) ? 3u
                                   : 0u; /* 16 (Q): size=0, opc carries width */
     e.opc = (rt.fp_bytes == 16) ? (is_load ? 3u : 2u) : (is_load ? 1u : 0u);
   } else {
     e.V = 0;
     e.size = (fixed_size >= 0) ? (u32)fixed_size : (rt.is64 ? 3u : 2u);
     e.scale = 1u << e.size;
     e.opc = !is_load    ? AA64_LDST_OPC_STR
             : !sign_ext ? AA64_LDST_OPC_LDR
             : rt.is64   ? 2u  /* LDRS*, 64-bit dst */
                         : 3u; /* LDRS*, 32-bit dst */
   }
   return e;
 }
 
 static AA64Reg parse_ldstp_reg(AsmDriver* d) {
   AsmTok t = asm_driver_next(d);
   AA64Reg r;
   memset(&r, 0, sizeof r);
   if (t.kind != ASM_TOK_IDENT ||
       (!parse_reg_from_ident(d, t.v.ident, &r) &&
        !parse_fp_pair_reg_from_ident(d, t.v.ident, &r))) {
     asm_driver_panic(d, "asm: expected register");
   }
   return r;
 }
 
 static void reject_sp_reg(AsmDriver* d, AA64Reg r, const char* what) {
   if (r.is_sp)
     asm_driver_panic(d, "asm: %.*s: SP register not allowed",
                      SLICE_ARG(slice_from_cstr(what)));
 }
 
 static void require_sp_spelling(AsmDriver* d, AA64Reg r, const char* what) {
   if (r.num == 31u && !r.is_sp)
     asm_driver_panic(d, "asm: %.*s: zero register not allowed in SP operand",
                      SLICE_ARG(slice_from_cstr(what)));
 }
 
 /* Parse "#imm" (with optional + / -) or a bare expression — GNU as is
  * lenient about the leading hash.  Returns an i64. */
 static i64 parse_imm_const(AsmDriver* d) {
   (void)asm_driver_eat_punct(d, '#');
   return asm_driver_parse_const(d);
 }
 
 /* Parse a possibly-symbolic operand prefixed by '#'. */
 static void parse_imm_sym(AsmDriver* d, ObjSymId* sym_out, i64* val_out) {
   (void)asm_driver_eat_punct(d, '#');
   asm_driver_parse_sym_expr(d, sym_out, val_out);
 }
 
 /* GNU-as relocation modifier on an aarch64 operand (`:lo12:`, `:got:`,
  * `:got_lo12:`).  AA64_RELMOD_NONE means no modifier was present. */
 typedef enum AA64RelMod {
   AA64_RELMOD_NONE = 0,
   AA64_RELMOD_PAGE, /* explicit adrp page reloc (Mach-O `@PAGE`); == bare adrp
                      */
   AA64_RELMOD_LO12,
   AA64_RELMOD_GOT,
   AA64_RELMOD_GOT_LO12,
 } AA64RelMod;
 
 /* True when the assembler's target object format is Mach-O, which spells
  * operand relocations as `@PAGE`/`@PAGEOFF` suffixes; ELF/COFF spell them as
  * `:lo12:`/`:got:` prefixes.  kit as parses the dialect of its target only
  * (no hybrid), mirroring what `cc -S` emits for that format. */
 static int target_is_macho(AsmDriver* d) {
   return asm_driver_compiler(d)->target.obj == KIT_OBJ_MACHO;
 }
 
 /* If the next token is ':', consume a `:name:` relocation modifier prefix and
  * return its kind.  A leading ':' is unambiguous at an operand position (a
  * label's ':' only appears at end-of-mnemonic).  Returns AA64_RELMOD_NONE and
  * leaves the stream untouched when there is no modifier. */
 static AA64RelMod parse_reloc_mod(AsmDriver* d) {
   if (!tok_punct(asm_driver_peek(d), ':')) return AA64_RELMOD_NONE;
   (void)asm_driver_next(d); /* eat ':' */
   AsmTok name = asm_driver_next(d);
   if (name.kind != ASM_TOK_IDENT)
     asm_driver_panic(d, "asm: expected relocation modifier name after ':'");
   Slice s = pool_slice(asm_driver_pool(d), name.v.ident);
   AA64RelMod mod;
   if (icase_eq(s.s, s.len, "lo12"))
     mod = AA64_RELMOD_LO12;
   else if (icase_eq(s.s, s.len, "got"))
     mod = AA64_RELMOD_GOT;
   else if (icase_eq(s.s, s.len, "got_lo12"))
     mod = AA64_RELMOD_GOT_LO12;
   else
     asm_driver_panic(d, "asm: unsupported relocation modifier");
   asm_driver_expect_punct(d, ':', "':' closing relocation modifier");
   return mod;
 }
 
 /* Mach-O operand relocation suffix: after a symbol(+addend), an optional
  * `@PAGE` / `@PAGEOFF` / `@GOTPAGE` / `@GOTPAGEOFF`.  Maps to the same
  * AA64RelMod the ELF `:mod:` prefix produces, so downstream encoding/reloc
  * emission is shared.  `@PAGE` is the explicit spelling of an adrp page reloc
  * (a bare adrp on ELF).  Returns AA64_RELMOD_NONE, stream untouched, when the
  * next token is not '@'. */
 static AA64RelMod parse_reloc_suffix(AsmDriver* d) {
   if (!tok_punct(asm_driver_peek(d), '@')) return AA64_RELMOD_NONE;
   (void)asm_driver_next(d); /* eat '@' */
   AsmTok name = asm_driver_next(d);
   if (name.kind != ASM_TOK_IDENT)
     asm_driver_panic(d, "asm: expected relocation suffix name after '@'");
   Slice s = pool_slice(asm_driver_pool(d), name.v.ident);
   if (icase_eq(s.s, s.len, "PAGE")) return AA64_RELMOD_PAGE;
   if (icase_eq(s.s, s.len, "PAGEOFF")) return AA64_RELMOD_LO12;
   if (icase_eq(s.s, s.len, "GOTPAGE")) return AA64_RELMOD_GOT;
   if (icase_eq(s.s, s.len, "GOTPAGEOFF")) return AA64_RELMOD_GOT_LO12;
   asm_driver_panic(d, "asm: unsupported relocation suffix");
 }
 
 /* The R_AARCH64_LDST{8,16,32,64}_ABS_LO12_NC reloc for an access log2-size. */
 static RelocKind aa64_ldst_lo12_reloc(AsmDriver* d, u32 size) {
   switch (size) {
     case 0:
       return R_AARCH64_LDST8_ABS_LO12_NC;
     case 1:
       return R_AARCH64_LDST16_ABS_LO12_NC;
     case 2:
       return R_AARCH64_LDST32_ABS_LO12_NC;
     case 3:
       return R_AARCH64_LDST64_ABS_LO12_NC;
     default:
       asm_driver_panic(d,
                        "asm: ldr/str: :lo12: not valid for this access size");
   }
 }
 
 /* Printer-side inverse of the operand reloc-modifier parsers above: how a
  * relocated aarch64 operand is spelled in `cc -S` text for the target object
  * format.  ELF uses a `:mod:` prefix; Mach-O uses an `@MOD` suffix — and even
  * a bare adrp page reloc needs an explicit `@PAGE` there.  Kept adjacent to
  * the `.s` parser (parse_reloc_mod / parse_reloc_suffix and their call sites)
  * so the emit and parse spellings stay in lockstep.  See ArchAsmOps. */
 static int aa64_reloc_operand(u16 kind, KitObjFmt fmt, ArchRelocOperand* out) {
   ArchRelocSurg surg;
   const char* elf;   /* `:mod:` prefix */
   const char* macho; /* `@MOD` suffix */
   switch (kind) {
     case R_AARCH64_CALL26:
     case R_AARCH64_JUMP26:
     case R_AARCH64_CONDBR19:
     case R_AARCH64_ADR_PREL_LO21:
       surg = ARCH_RELOC_SURG_TAIL, elf = "", macho = "";
       break;
     case R_AARCH64_ADR_PREL_PG_HI21:
       surg = ARCH_RELOC_SURG_TAIL, elf = "", macho = "@PAGE";
       break;
     case R_AARCH64_ADR_GOT_PAGE:
       surg = ARCH_RELOC_SURG_TAIL, elf = ":got:", macho = "@GOTPAGE";
       break;
     case R_AARCH64_ADD_ABS_LO12_NC:
       surg = ARCH_RELOC_SURG_TAIL, elf = ":lo12:", macho = "@PAGEOFF";
       break;
     case R_AARCH64_LDST8_ABS_LO12_NC:
     case R_AARCH64_LDST16_ABS_LO12_NC:
     case R_AARCH64_LDST32_ABS_LO12_NC:
     case R_AARCH64_LDST64_ABS_LO12_NC:
       surg = ARCH_RELOC_SURG_MEM, elf = ":lo12:", macho = "@PAGEOFF";
       break;
     case R_AARCH64_LD64_GOT_LO12_NC:
       surg = ARCH_RELOC_SURG_MEM, elf = ":got_lo12:", macho = "@GOTPAGEOFF";
       break;
     default:
       return 0; /* TLV and anything else: keep the numeric operand */
   }
   out->surg = surg;
   out->addend_bias = 0; /* aarch64 relocs store the symbol offset directly */
   if (fmt == KIT_OBJ_MACHO) {
     out->prefix = "";
     out->suffix = macho;
   } else {
     out->prefix = elf;
     out->suffix = "";
   }
   return 1;
 }
 
 /* Intra-section local branches whose target codegen resolved in place (no
  * relocation): b, b.<cc>, cbz/cbnz, tbz/tbnz, and adr (address-of-label, e.g.
  * `&&label`). Excludes bl (a call — always relocated), adrp (page-relative; its
  * lo12 partner carries the reloc), and register-form branches. Moved here from
  * the printer so branch-mnemonic knowledge is arch-local. */
 static int aa64_is_local_branch(KitSlice m) {
   if (m.len == 1 && m.s[0] == 'b') return 1;
   if (m.len >= 2 && m.s[0] == 'b' && m.s[1] == '.') return 1;
   if (m.len == 3 && memcmp(m.s, "cbz", 3) == 0) return 1;
   if (m.len == 4 && memcmp(m.s, "cbnz", 4) == 0) return 1;
   if (m.len == 3 && memcmp(m.s, "tbz", 3) == 0) return 1;
   if (m.len == 4 && memcmp(m.s, "tbnz", 4) == 0) return 1;
   return 0;
 }
 
 const ArchAsmOps aa64_asm_ops = {
     .reloc_operand = aa64_reloc_operand,
     .is_local_branch = aa64_is_local_branch,
 };
 
 static void emit32(AsmDriver* d, u32 word) {
   MCEmitter* mc = asm_driver_mc(d);
   (void)asm_driver_cur_section(d);
   u8 buf[4];
   buf[0] = (u8)(word & 0xff);
   buf[1] = (u8)((word >> 8) & 0xff);
   buf[2] = (u8)((word >> 16) & 0xff);
   buf[3] = (u8)((word >> 24) & 0xff);
   mc->emit_bytes(mc, buf, 4);
 }
 
 static int parse_cond_from_ident(AsmDriver* d, Sym ident, u32* out) {
   Slice sl = pool_slice(asm_driver_pool(d), ident);
   const char* s = sl.s;
   size_t n = sl.len;
   if (!s) return 0;
   if (icase_eq(s, n, "eq"))
     *out = 0;
   else if (icase_eq(s, n, "ne"))
     *out = 1;
   else if (icase_eq(s, n, "cs") || icase_eq(s, n, "hs"))
     *out = 2;
   else if (icase_eq(s, n, "cc") || icase_eq(s, n, "lo"))
     *out = 3;
   else if (icase_eq(s, n, "mi"))
     *out = 4;
   else if (icase_eq(s, n, "pl"))
     *out = 5;
   else if (icase_eq(s, n, "vs"))
     *out = 6;
   else if (icase_eq(s, n, "vc"))
     *out = 7;
   else if (icase_eq(s, n, "hi"))
     *out = 8;
   else if (icase_eq(s, n, "ls"))
     *out = 9;
   else if (icase_eq(s, n, "ge"))
     *out = 10;
   else if (icase_eq(s, n, "lt"))
     *out = 11;
   else if (icase_eq(s, n, "gt"))
     *out = 12;
   else if (icase_eq(s, n, "le"))
     *out = 13;
   else if (icase_eq(s, n, "al"))
     *out = 14;
   else
     return 0;
   return 1;
 }
 
 static u32 parse_cond(AsmDriver* d, const char* what) {
   AsmTok t = asm_driver_next(d);
   u32 cond = 0;
   if (t.kind != ASM_TOK_IDENT || !parse_cond_from_ident(d, t.v.ident, &cond))
     asm_driver_panic(d, "asm: %.*s: expected condition code",
                      SLICE_ARG(slice_from_cstr(what)));
   return cond;
 }
 
 static void expect_comma(AsmDriver* d, const char* what) {
   if (!asm_driver_eat_comma(d))
     asm_driver_panic(d, "asm: expected ',' (%.*s)",
                      SLICE_ARG(slice_from_cstr(what)));
 }
 
 /* ---- per-mnemonic parsers ---- */
 
 /* ret [Xn] — Xn defaults to x30. */
 static void p_ret(AsmDriver* d) {
   if (asm_driver_at_eol(d)) {
     emit32(d, aa64_ret(30));
     return;
   }
   AA64Reg r = parse_reg(d);
   if (!r.is64) asm_driver_panic(d, "asm: ret: 64-bit register expected");
   emit32(d, aa64_ret(r.num));
 }
 
 static void p_br(AsmDriver* d) {
   AA64Reg r = parse_reg(d);
   if (!r.is64) asm_driver_panic(d, "asm: br: 64-bit register expected");
   emit32(d, aa64_br(r.num));
 }
 
 static void p_blr(AsmDriver* d) {
   AA64Reg r = parse_reg(d);
   if (!r.is64) asm_driver_panic(d, "asm: blr: 64-bit register expected");
   emit32(d, aa64_blr(r.num));
 }
 
 static void p_nop(AsmDriver* d) {
   (void)d;
   emit32(d, aa64_nop());
 }
 
 /* Memory barriers (DMB / DSB / ISB / CLREX).
  *
  *   dmb <option>        ; option in {sy, ish, nsh, osh, ld, st, ishld,
  *                                    ishst, nshld, nshst, oshld, oshst}
  *   dmb #imm4           ; numeric form
  *   dsb <option> | #imm4
  *   isb [<option>]      ; option defaults to sy when omitted
  *   clrex [#imm4]       ; option defaults to sy (15) when omitted */
 static u32 parse_barrier_option(AsmDriver* d, int allow_dmb_ld_st) {
   if (asm_driver_at_eol(d)) return AA64_BARRIER_OPT_SY;
   AsmTok t = asm_driver_peek(d);
   if (t.kind == ASM_TOK_IDENT) {
     (void)asm_driver_next(d);
     Slice sl = pool_slice(asm_driver_pool(d), t.v.ident);
     const char* s = sl.s;
     size_t n = sl.len;
     if (icase_eq(s, n, "sy")) return AA64_BARRIER_OPT_SY;
     if (icase_eq(s, n, "ish")) return AA64_BARRIER_OPT_ISH;
     if (icase_eq(s, n, "ishld")) return AA64_BARRIER_OPT_ISHLD;
     if (icase_eq(s, n, "ishst")) return AA64_BARRIER_OPT_ISHST;
     if (icase_eq(s, n, "nsh")) return AA64_BARRIER_OPT_NSH;
     if (icase_eq(s, n, "nshld")) return AA64_BARRIER_OPT_NSHLD;
     if (icase_eq(s, n, "nshst")) return AA64_BARRIER_OPT_NSHST;
     if (icase_eq(s, n, "osh")) return AA64_BARRIER_OPT_OSH;
     if (icase_eq(s, n, "oshld")) return AA64_BARRIER_OPT_OSHLD;
     if (icase_eq(s, n, "oshst")) return AA64_BARRIER_OPT_OSHST;
     if (allow_dmb_ld_st) {
       if (icase_eq(s, n, "ld")) return AA64_BARRIER_OPT_LD;
       if (icase_eq(s, n, "st")) return AA64_BARRIER_OPT_ST;
     }
     asm_driver_panic(d, "asm: unknown barrier option");
   }
   /* Numeric form: '#imm4'. */
   i64 imm = parse_imm_const(d);
   if (imm < 0 || imm > 15) asm_driver_panic(d, "asm: barrier imm out of range");
   return (u32)imm;
 }
 
 static void p_dmb(AsmDriver* d) {
   u32 opt = parse_barrier_option(d, /*allow_dmb_ld_st=*/1);
   emit32(d, aa64_dmb(opt));
 }
 static void p_dsb(AsmDriver* d) {
   u32 opt = parse_barrier_option(d, /*allow_dmb_ld_st=*/0);
   emit32(d, aa64_dsb(opt));
 }
 static void p_isb(AsmDriver* d) {
   u32 opt = parse_barrier_option(d, /*allow_dmb_ld_st=*/0);
   emit32(d, aa64_isb(opt));
 }
 static void p_clrex(AsmDriver* d) {
   u32 opt = parse_barrier_option(d, /*allow_dmb_ld_st=*/0);
   emit32(d, aa64_clrex(opt));
 }
 
 /* System-register access (MRS/MSR register form).
  *
  *   mrs Xt, <sysreg>     ; read  system register into Xt
  *   msr <sysreg>, Xt     ; write Xt into the system register
  *
  * A system register is named (resolved against the shared isa.c name table
  * via aa64_sysreg_by_name) or given by the architectural generic spelling
  * S<op0>_<op1>_C<crn>_C<crm>_<op2> (e.g. s3_3_c13_c0_2 == tpidr_el0), which
  * covers any encodable register. The immediate PSTATE forms (msr daifset,
  * #imm / spsel, #imm) are not handled. */
 
 /* Consume the next token as a system-register name into the 5 fields. */
 static void parse_sysreg(AsmDriver* d, const char* what, u32* op0, u32* op1,
                          u32* crn, u32* crm, u32* op2) {
   AsmTok t = asm_driver_peek(d);
   if (t.kind != ASM_TOK_IDENT)
     asm_driver_panic(d, "asm: expected system register");
   (void)asm_driver_next(d);
   Slice sl = pool_slice(asm_driver_pool(d), t.v.ident);
   if (!aa64_sysreg_by_name(sl.s, sl.len, op0, op1, crn, crm, op2))
     asm_driver_panic(d, what);
 }
 
 static void p_mrs_(AsmDriver* d) {
   AA64Reg rt = parse_reg(d);
   if (!rt.is64 || rt.is_sp)
     asm_driver_panic(d, "asm: mrs: destination must be a 64-bit GPR");
   expect_comma(d, "mrs");
   u32 op0, op1, crn, crm, op2;
   parse_sysreg(d, "asm: mrs: unknown system register", &op0, &op1, &crn, &crm,
                &op2);
   emit32(d, aa64_sysreg_move(/*is_read=*/1, op0, op1, crn, crm, op2, rt.num));
 }
 
 static void p_msr_(AsmDriver* d) {
   u32 op0, op1, crn, crm, op2;
   parse_sysreg(d,
                "asm: msr: unknown system register (immediate PSTATE forms "
                "like daifset are unsupported)",
                &op0, &op1, &crn, &crm, &op2);
   expect_comma(d, "msr");
   AA64Reg rt = parse_reg(d);
   if (!rt.is64 || rt.is_sp)
     asm_driver_panic(d, "asm: msr: source must be a 64-bit GPR");
   emit32(d, aa64_sysreg_move(/*is_read=*/0, op0, op1, crn, crm, op2, rt.num));
 }
 
 /* mov:
  *   mov Rd, Rm        → ORR Rd, ZR, Rm
  *   mov Rd, #imm      → MOVZ (if imm fits in a single halfword unshifted)
  *                       MOVN (if ~imm fits)
  *                       otherwise: panic (multi-step expansion deferred). */
 static void p_mov(AsmDriver* d) {
   AA64Reg rd = parse_reg(d);
   expect_comma(d, "mov");
   AsmTok t = asm_driver_peek(d);
   if (t.kind == ASM_TOK_IDENT) {
     AA64Reg src;
     memset(&src, 0, sizeof src);
     if (parse_reg_from_ident(d, t.v.ident, &src)) {
       (void)asm_driver_next(d);
       if (src.is64 != rd.is64)
         asm_driver_panic(d, "asm: mov: register width mismatch");
       /* mov involving SP encodes as `ADD Rd, Rsp, #0` per AArch64;
        * approximate with that exact form. */
       if (rd.is_sp || src.is_sp) {
         require_sp_spelling(d, rd, "mov sp");
         require_sp_spelling(d, src, "mov sp");
         emit32(d, aa64_add_imm(rd.is64, rd.num, src.num, 0, 0));
         return;
       }
       emit32(d, aa64_mov_reg(rd.is64, rd.num, src.num));
       return;
     }
     /* fall through: identifier that is not a register → treat as
      * symbol/equate via expression below. */
   }
   /* Immediate. */
   i64 imm = parse_imm_const(d);
   if (rd.is_sp) asm_driver_panic(d, "asm: mov: cannot move imm into SP");
   u64 uv = (u64)imm;
   u64 mask = rd.is64 ? ~0ull : 0xffffffffull;
   uv &= mask;
   /* Try MOVZ with one of four halfwords. */
   for (u32 hw = 0; hw < (rd.is64 ? 4u : 2u); ++hw) {
     u64 shift = (u64)hw * 16;
     u64 hwmask = 0xffffull << shift;
     if ((uv & ~hwmask) == 0) {
       u32 v = (u32)((uv >> shift) & 0xffff);
       emit32(d, aa64_movz(rd.is64, rd.num, v, hw));
       return;
     }
   }
   /* Try MOVN with one halfword (encodes ~imm in that halfword). */
   u64 nv = (~uv) & mask;
   for (u32 hw = 0; hw < (rd.is64 ? 4u : 2u); ++hw) {
     u64 shift = (u64)hw * 16;
     u64 hwmask = 0xffffull << shift;
     if ((nv & ~hwmask) == 0) {
       u32 v = (u32)((nv >> shift) & 0xffff);
       emit32(d, aa64_movn(rd.is64, rd.num, v, hw));
       return;
     }
   }
   /* Try the ORR-bitmask alias (mov Rd,#imm → ORR Rd,ZR,#bitmask). */
   {
     u32 N = 0, immr = 0, imms = 0;
     if (aa64_logimm_encode(uv, rd.is64, &N, &immr, &imms)) {
       emit32(d, aa64_orr_imm(rd.is64, rd.num, AA64_ZR, N, immr, imms));
       return;
     }
   }
   asm_driver_panic(d, "asm: mov: immediate cannot be encoded in one insn");
 }
 
 /* mvn Rd, Rm */
 static void p_mvn(AsmDriver* d) {
   AA64Reg rd = parse_reg(d);
   expect_comma(d, "mvn");
   AA64Reg rm = parse_reg(d);
   if (rd.is64 != rm.is64) asm_driver_panic(d, "asm: mvn: width mismatch");
   emit32(d, aa64_mvn(rd.is64, rd.num, rm.num));
 }
 
 /* movz / movn / movk Rd, #imm[, lsl #shift] */
 static void p_movwide(AsmDriver* d, u32 opc) {
   AA64Reg rd = parse_reg(d);
   expect_comma(d, "movz/n/k");
   i64 imm = parse_imm_const(d);
   u32 hw = 0;
   if (asm_driver_eat_comma(d)) {
     /* lsl #N (N is 0/16/32/48). */
     AsmTok lid = asm_driver_next(d);
     if (lid.kind != ASM_TOK_IDENT) asm_driver_panic(d, "asm: expected 'lsl'");
     Slice lsl = pool_slice(asm_driver_pool(d), lid.v.ident);
     const char* lp = lsl.s;
     size_t ln = lsl.len;
     if (!lp || !icase_eq(lp, ln, "lsl"))
       asm_driver_panic(d, "asm: expected 'lsl'");
     i64 sh = parse_imm_const(d);
     if (sh % 16 != 0 || sh < 0 || sh > 48)
       asm_driver_panic(d, "asm: movz/n/k: bad lsl shift");
     hw = (u32)(sh / 16);
   }
   u32 word = ((rd.is64 & 1u) << 31) | ((opc & 3u) << 29) |
              AA64_MOVEWIDE_FAMILY_MATCH | ((hw & 3u) << 21) |
              (((u32)imm & 0xffffu) << 5) | (rd.num & 0x1fu);
   emit32(d, word);
 }
 
 /* svc / brk / hlt #imm */
 static void p_except(AsmDriver* d, u32 form) {
   i64 imm = parse_imm_const(d);
   switch (form) {
     case 0:
       emit32(d, aa64_svc((u32)imm));
       break;
     case 1:
       emit32(d, aa64_brk((u32)imm));
       break;
     case 2: {
       /* HLT */
       u32 word = AA64_EXCEPT_FAMILY_MATCH | ((u32)2 << 21) |
                  (((u32)imm & 0xffffu) << 5);
       emit32(d, word);
       break;
     }
     default:
       asm_driver_panic(d, "asm: bad exception form");
   }
 }
 
 /* Read optional `, lsl|lsr|asr|ror #imm` shift modifier. Returns 1 if
  * present. */
 static int parse_shift_mod(AsmDriver* d, u32* shift_out, u32* imm6_out) {
   AsmTok t = asm_driver_peek(d);
   if (t.kind != ASM_TOK_IDENT) return 0;
   Slice sl = pool_slice(asm_driver_pool(d), t.v.ident);
   const char* p = sl.s;
   size_t n = sl.len;
   u32 sh;
   if (icase_eq(p, n, "lsl"))
     sh = 0;
   else if (icase_eq(p, n, "lsr"))
     sh = 1;
   else if (icase_eq(p, n, "asr"))
     sh = 2;
   else if (icase_eq(p, n, "ror"))
     sh = 3;
   else
     return 0;
   (void)asm_driver_next(d);
   i64 imm = parse_imm_const(d);
   if (imm < 0 || imm > 63)
     asm_driver_panic(d, "asm: shift amount out of range");
   *shift_out = sh;
   *imm6_out = (u32)imm;
   return 1;
 }
 
 /* add / sub family.
  * Forms:
  *   add Rd, Rn, Rm[, lsl #s]   shifted-register
  *   add Rd, Rn, #imm           immediate
  *   add Rd, Rn, #imm, lsl #12  immediate w/ shift
  * S-suffixed (adds/subs) sets flags. */
 static void p_addsub(AsmDriver* d, int is_sub, int set_flags) {
   AA64Reg rd = parse_reg(d);
   expect_comma(d, "add/sub");
   AA64Reg rn = parse_reg(d);
   expect_comma(d, "add/sub");
   AsmTok t = asm_driver_peek(d);
   /* `add Rd, Rn, <sym lo12>` — ADD (immediate), zero imm12, plus an
    * R_AARCH64_ADD_ABS_LO12_NC relocation (the low-12 PIC/abs sequence). ELF
    * spells the modifier as a `:lo12:` prefix (leading ':'); Mach-O spells it
    * as a `sym@PAGEOFF` suffix, so the trigger there is a non-register IDENT
    * third operand (probe with parse_reg_from_ident so `add x0,x1,x2` stays the
    * register path). */
   int symbolic = 0;
   if (!is_sub && !set_flags) {
     if (target_is_macho(d)) {
       AA64Reg probe;
       memset(&probe, 0, sizeof probe);
       symbolic = (t.kind == ASM_TOK_IDENT &&
                   !parse_reg_from_ident(d, t.v.ident, &probe));
     } else {
       symbolic = tok_punct(t, ':');
     }
   }
   if (symbolic) {
     AA64RelMod mod;
     ObjSymId sym = OBJ_SYM_NONE;
     i64 off = 0;
     if (target_is_macho(d)) {
       parse_imm_sym(d, &sym, &off);
       mod = parse_reloc_suffix(d);
     } else {
       mod = parse_reloc_mod(d);
       parse_imm_sym(d, &sym, &off);
     }
     if (mod != AA64_RELMOD_LO12)
       asm_driver_panic(d,
                        "asm: add: only :lo12: (ELF) / @PAGEOFF (Mach-O) is "
                        "valid here");
     if (rd.is64 != rn.is64)
       asm_driver_panic(d, "asm: add lo12: width mismatch");
     u32 word = aa64_addsubimm_pack((AA64AddSubImm){.sf = rd.is64,
                                                    .op = 0,
                                                    .S = 0,
                                                    .sh = 0,
                                                    .imm12 = 0,
                                                    .Rn = rn.num,
                                                    .Rd = rd.num});
     emit32(d, word);
     MCEmitter* mc = asm_driver_mc(d);
     mc->emit_reloc_at(mc, asm_driver_cur_section(d), mc->pos(mc) - 4,
                       R_AARCH64_ADD_ABS_LO12_NC, sym, off, 1, 0);
     return;
   }
   if (tok_punct(t, '#') || t.kind == ASM_TOK_NUM || tok_punct(t, '-') ||
       tok_punct(t, '+')) {
     /* immediate form */
     if (rd.is64 != rn.is64)
       asm_driver_panic(d, "asm: add/sub imm: width mismatch");
     require_sp_spelling(d, rn, "add/sub imm");
     if (set_flags) {
       reject_sp_reg(d, rd, "add/sub imm");
     } else {
       require_sp_spelling(d, rd, "add/sub imm");
     }
     i64 imm = parse_imm_const(d);
     u32 sh = 0;
     if (asm_driver_eat_comma(d)) {
       AsmTok lid = asm_driver_next(d);
       if (lid.kind != ASM_TOK_IDENT)
         asm_driver_panic(d, "asm: expected 'lsl #12'");
       Slice lsl = pool_slice(asm_driver_pool(d), lid.v.ident);
       const char* lp = lsl.s;
       size_t ln = lsl.len;
       if (!lp || !icase_eq(lp, ln, "lsl"))
         asm_driver_panic(d, "asm: expected 'lsl'");
       i64 s = parse_imm_const(d);
       if (s == 12)
         sh = 1;
       else if (s == 0)
         sh = 0;
       else
         asm_driver_panic(d, "asm: add/sub imm: lsl must be 0 or 12");
     }
     if (imm < 0 || imm > 0xfff)
       asm_driver_panic(d, "asm: add/sub imm out of range");
     u32 word = aa64_addsubimm_pack((AA64AddSubImm){.sf = rd.is64,
                                                    .op = (u32)is_sub,
                                                    .S = (u32)set_flags,
                                                    .sh = sh,
                                                    .imm12 = (u32)imm,
                                                    .Rn = rn.num,
                                                    .Rd = rd.num});
     emit32(d, word);
     return;
   }
   /* register form */
   AA64Reg rm = parse_reg(d);
   reject_sp_reg(d, rd, "add/sub reg");
   reject_sp_reg(d, rn, "add/sub reg");
   reject_sp_reg(d, rm, "add/sub reg");
   if (rd.is64 != rm.is64 || rd.is64 != rn.is64)
     asm_driver_panic(d, "asm: add/sub reg: width mismatch");
   u32 shift = 0, imm6 = 0;
   if (asm_driver_eat_comma(d)) {
     if (!parse_shift_mod(d, &shift, &imm6))
       asm_driver_panic(d, "asm: add/sub reg: expected shift modifier");
   }
   u32 word = aa64_addsubsr_pack((AA64AddSubSR){.sf = rd.is64,
                                                .op = (u32)is_sub,
                                                .S = (u32)set_flags,
                                                .shift = shift,
                                                .Rm = rm.num,
                                                .imm6 = imm6,
                                                .Rn = rn.num,
                                                .Rd = rd.num});
   emit32(d, word);
 }
 
 /* cmp Rn, Rm | cmp Rn, #imm  → SUBS ZR, Rn, ... */
 static void p_cmp(AsmDriver* d, int is_neg /* cmn flips op */) {
   AA64Reg rn = parse_reg(d);
   expect_comma(d, "cmp");
   AsmTok t = asm_driver_peek(d);
   if (tok_punct(t, '#') || t.kind == ASM_TOK_NUM || tok_punct(t, '-') ||
       tok_punct(t, '+')) {
     require_sp_spelling(d, rn, "cmp imm");
     i64 imm = parse_imm_const(d);
     u32 sh = 0;
     if (asm_driver_eat_comma(d)) {
       AsmTok lid = asm_driver_next(d);
       Slice lsl = (lid.kind == ASM_TOK_IDENT)
                       ? pool_slice(asm_driver_pool(d), lid.v.ident)
                       : SLICE_NULL;
       const char* lp = lsl.s;
       size_t ln = lsl.len;
       if (!lp || !icase_eq(lp, ln, "lsl"))
         asm_driver_panic(d, "asm: cmp imm: expected 'lsl'");
       i64 s = parse_imm_const(d);
       if (s == 12)
         sh = 1;
       else if (s != 0)
         asm_driver_panic(d, "asm: cmp imm: lsl must be 0 or 12");
     }
     if (imm < 0 || imm > 0xfff)
       asm_driver_panic(d, "asm: cmp imm out of range");
     u32 word = aa64_addsubimm_pack((AA64AddSubImm){.sf = rn.is64,
                                                    .op = (u32)(!is_neg),
                                                    .S = 1,
                                                    .sh = sh,
                                                    .imm12 = (u32)imm,
                                                    .Rn = rn.num,
                                                    .Rd = AA64_ZR});
     emit32(d, word);
     return;
   }
   AA64Reg rm = parse_reg(d);
   reject_sp_reg(d, rn, "cmp reg");
   reject_sp_reg(d, rm, "cmp reg");
   if (rm.is64 != rn.is64) asm_driver_panic(d, "asm: cmp: width mismatch");
   u32 shift = 0, imm6 = 0;
   if (asm_driver_eat_comma(d)) parse_shift_mod(d, &shift, &imm6);
   u32 word = aa64_addsubsr_pack((AA64AddSubSR){.sf = rn.is64,
                                                .op = (u32)(!is_neg),
                                                .S = 1,
                                                .shift = shift,
                                                .Rm = rm.num,
                                                .imm6 = imm6,
                                                .Rn = rn.num,
                                                .Rd = AA64_ZR});
   emit32(d, word);
 }
 
 static void p_condsel(AsmDriver* d, u32 op, u32 op2, const char* what) {
   AA64Reg rd = parse_reg(d);
   expect_comma(d, what);
   AA64Reg rn = parse_reg(d);
   expect_comma(d, what);
   AA64Reg rm = parse_reg(d);
   expect_comma(d, what);
   u32 cond = parse_cond(d, what);
   if (rd.is_sp || rn.is_sp || rm.is_sp)
     asm_driver_panic(d, "asm: %.*s: SP register not allowed",
                      SLICE_ARG(slice_from_cstr(what)));
   if (rd.is64 != rn.is64 || rd.is64 != rm.is64)
     asm_driver_panic(d, "asm: %.*s: width mismatch",
                      SLICE_ARG(slice_from_cstr(what)));
   u32 word = aa64_condsel_pack((AA64CondSel){.sf = (u32)rd.is64,
                                              .op = op,
                                              .S = 0,
                                              .Rm = rm.num,
                                              .cond = cond,
                                              .op2 = op2,
                                              .Rn = rn.num,
                                              .Rd = rd.num});
   emit32(d, word);
 }
 
 static void p_cset_like(AsmDriver* d, u32 op, u32 op2, const char* what) {
   AA64Reg rd = parse_reg(d);
   expect_comma(d, what);
   u32 cond = parse_cond(d, what);
   if (rd.is_sp)
     asm_driver_panic(d, "asm: %.*s: SP register not allowed",
                      SLICE_ARG(slice_from_cstr(what)));
   u32 word = aa64_condsel_pack((AA64CondSel){.sf = (u32)rd.is64,
                                              .op = op,
                                              .S = 0,
                                              .Rm = AA64_ZR,
                                              .cond = cond ^ 1u,
                                              .op2 = op2,
                                              .Rn = AA64_ZR,
                                              .Rd = rd.num});
   emit32(d, word);
 }
 
 /* neg / negs Rd, Rm  → SUB / SUBS Rd, ZR, Rm */
 static void p_neg(AsmDriver* d, int set_flags) {
   AA64Reg rd = parse_reg(d);
   expect_comma(d, "neg");
   AA64Reg rm = parse_reg(d);
   reject_sp_reg(d, rd, "neg");
   reject_sp_reg(d, rm, "neg");
   if (rd.is64 != rm.is64) asm_driver_panic(d, "asm: neg: width mismatch");
   u32 shift = 0, imm6 = 0;
   if (asm_driver_eat_comma(d)) parse_shift_mod(d, &shift, &imm6);
   u32 word = aa64_addsubsr_pack((AA64AddSubSR){.sf = rd.is64,
                                                .op = 1,
                                                .S = (u32)set_flags,
                                                .shift = shift,
                                                .Rm = rm.num,
                                                .imm6 = imm6,
                                                .Rn = AA64_ZR,
                                                .Rd = rd.num});
   emit32(d, word);
 }
 
 /* Logical family: shifted-register `<op> Rd,Rn,Rm{,shift}` or, for the
  * non-negated AND/ORR/EOR/ANDS, the bitmask-immediate `<op> Rd,Rn,#imm`.
  * N is the SR-form negate bit (BIC/ORN/EON/BICS); those have no immediate
  * form, so an immediate third operand is only valid when N==0. */
 static void p_log_sr(AsmDriver* d, u32 opc, u32 N) {
   AA64Reg rd = parse_reg(d);
   expect_comma(d, "logical");
   AA64Reg rn = parse_reg(d);
   expect_comma(d, "logical");
   if (!peek_is_reg(d)) {
     /* Bitmask-immediate form. AND/ORR/EOR use the SP-capable destination;
      * ANDS uses ZR. Rn is always a GPR (caller's parse_reg already enforced
      * GP for the two register operands). */
     if (N) asm_driver_panic(d, "asm: logical: immediate form has no negation");
     if (rd.is64 != rn.is64) asm_driver_panic(d, "asm: logical: width mismatch");
     u64 imm = (u64)parse_imm_const(d);
     u32 bN = 0, immr = 0, imms = 0;
     if (!aa64_logimm_encode(imm, rd.is64, &bN, &immr, &imms))
       asm_driver_panic(d, "asm: logical: immediate is not a valid bitmask");
     emit32(d, aa64_logimm_pack((AA64LogImm){.sf = rd.is64,
                                             .opc = opc,
                                             .N = bN,
                                             .immr = immr,
                                             .imms = imms,
                                             .Rn = rn.num,
                                             .Rd = rd.num}));
     return;
   }
   AA64Reg rm = parse_reg(d);
   if (rd.is64 != rn.is64 || rd.is64 != rm.is64)
     asm_driver_panic(d, "asm: logical: width mismatch");
   u32 shift = 0, imm6 = 0;
   if (asm_driver_eat_comma(d)) parse_shift_mod(d, &shift, &imm6);
   u32 word = aa64_logsr_pack((AA64LogSR){.sf = rd.is64,
                                          .opc = opc,
                                          .shift = shift,
                                          .N = N,
                                          .Rm = rm.num,
                                          .imm6 = imm6,
                                          .Rn = rn.num,
                                          .Rd = rd.num});
   emit32(d, word);
 }
 
 /* Data-processing 3-source: madd/msub Rd, Rn, Rm, Ra. */
 static void p_dp3(AsmDriver* d, u32 o0) {
   AA64Reg rd = parse_reg(d);
   expect_comma(d, "dp3");
   AA64Reg rn = parse_reg(d);
   expect_comma(d, "dp3");
   AA64Reg rm = parse_reg(d);
   expect_comma(d, "dp3");
   AA64Reg ra = parse_reg(d);
   if (rd.is64 != rn.is64 || rd.is64 != rm.is64 || rd.is64 != ra.is64)
     asm_driver_panic(d, "asm: dp3: width mismatch");
   u32 word = aa64_dp3_pack((AA64DP3){.sf = rd.is64,
                                      .op31 = 0,
                                      .o0 = o0,
                                      .Rm = rm.num,
                                      .Ra = ra.num,
                                      .Rn = rn.num,
                                      .Rd = rd.num});
   emit32(d, word);
 }
 
 /* mul Rd, Rn, Rm  → MADD Rd, Rn, Rm, ZR */
 static void p_mul(AsmDriver* d, u32 o0) {
   AA64Reg rd = parse_reg(d);
   expect_comma(d, "mul");
   AA64Reg rn = parse_reg(d);
   expect_comma(d, "mul");
   AA64Reg rm = parse_reg(d);
   if (rd.is64 != rn.is64 || rd.is64 != rm.is64)
     asm_driver_panic(d, "asm: mul: width mismatch");
   u32 word = aa64_dp3_pack((AA64DP3){.sf = rd.is64,
                                      .op31 = 0,
                                      .o0 = o0,
                                      .Rm = rm.num,
                                      .Ra = AA64_ZR,
                                      .Rn = rn.num,
                                      .Rd = rd.num});
   emit32(d, word);
 }
 
 /* DP2: udiv/sdiv/lslv/lsrv/asrv/rorv Rd, Rn, Rm. */
 static void p_dp2(AsmDriver* d, u32 opcode) {
   AA64Reg rd = parse_reg(d);
   expect_comma(d, "dp2");
   AA64Reg rn = parse_reg(d);
   expect_comma(d, "dp2");
   AA64Reg rm = parse_reg(d);
   if (rd.is64 != rn.is64 || rd.is64 != rm.is64)
     asm_driver_panic(d, "asm: dp2: width mismatch");
   u32 word = aa64_dp2_pack((AA64DP2){.sf = rd.is64,
                                      .opcode = opcode,
                                      .Rm = rm.num,
                                      .Rn = rn.num,
                                      .Rd = rd.num});
   emit32(d, word);
 }
 
 /* Shift aliases: `<op> Rd, Rn, (Rm | #imm)`.
  *   register form  → LSLV/LSRV/ASRV (DP2 variable shift)
  *   immediate form → UBFM (lsl/lsr) / SBFM (asr) bitfield alias
  * `kind` indexes the three shifts: 0=lsl 1=lsr 2=asr. The immediate aliases
  * are exactly what the disassembler prints for these UBFM/SBFM encodings, so
  * `cc -S | as` round-trips. (ROR's immediate form is EXTR, which the
  * disassembler doesn't decode, so it is left out — `rorv` covers the register
  * rotate.) */
 static void p_shift(AsmDriver* d, u32 kind) {
   static const u32 dp2op[3] = {AA64_DP2_LSLV_OP, AA64_DP2_LSRV_OP,
                                AA64_DP2_ASRV_OP};
   AA64Reg rd = parse_reg(d);
   expect_comma(d, "shift");
   AA64Reg rn = parse_reg(d);
   if (rd.is64 != rn.is64) asm_driver_panic(d, "asm: shift: width mismatch");
   expect_comma(d, "shift");
   if (peek_is_reg(d)) {
     AA64Reg rm = parse_reg(d);
     if (rd.is64 != rm.is64) asm_driver_panic(d, "asm: shift: width mismatch");
     emit32(d, aa64_dp2_pack((AA64DP2){.sf = rd.is64,
                                       .opcode = dp2op[kind],
                                       .Rm = rm.num,
                                       .Rn = rn.num,
                                       .Rd = rd.num}));
     return;
   }
   i64 sv = parse_imm_const(d);
   u32 width = rd.is64 ? 64u : 32u;
   if (sv < 0 || (u64)sv >= width)
     asm_driver_panic(d, "asm: shift: amount out of range");
   u32 shift = (u32)sv, immr = 0, imms = 0;
   if (kind == 2) { /* asr → SBFM */
     aa64_asr_imm_fields(shift, rd.is64, &immr, &imms);
     emit32(d, aa64_bitfield(rd.is64, 0u, immr, imms, rd.num, rn.num));
   } else { /* lsl/lsr → UBFM */
     if (kind == 0)
       aa64_lsl_imm_fields(shift, rd.is64, &immr, &imms);
     else
       aa64_lsr_imm_fields(shift, rd.is64, &immr, &imms);
     emit32(d, aa64_bitfield(rd.is64, 2u, immr, imms, rd.num, rn.num));
   }
 }
 
 /* Branch immediate / conditional / compare-and-branch. */
 
 static void emit_branch_imm(AsmDriver* d, u32 op_bl, ObjSymId target,
                             i64 addend, i64 const_disp) {
   MCEmitter* mc = asm_driver_mc(d);
   /* Emit a B/BL with imm26 = 0; record a CALL26/JUMP26 reloc against
    * either the symbol or the constant displacement. */
   u32 word = aa64_brimm_pack((AA64BrImm){.op = op_bl, .imm26 = 0});
   emit32(d, word);
   u32 ofs = mc->pos(mc) - 4;
   RelocKind k = op_bl ? R_AARCH64_CALL26 : R_AARCH64_JUMP26;
   if (target != OBJ_SYM_NONE) {
     mc->emit_reloc_at(mc, asm_driver_cur_section(d), ofs, k, target, addend, 1,
                       0);
   } else {
     /* Pure constant displacement is rare in real .s; reject it now.
      * The recommended form is to use a label and let the assembler
      * compute the displacement. */
     (void)const_disp;
     asm_driver_panic(d, "asm: branch with pure constant disp not supported");
   }
 }
 
 static void p_b(AsmDriver* d, u32 op_bl) {
   ObjSymId sym = OBJ_SYM_NONE;
   i64 off = 0;
   /* GNU as accepts `b sym`, `bl sym+8`, etc. */
   parse_imm_sym(d, &sym, &off);
   if (sym == OBJ_SYM_NONE)
     asm_driver_panic(d, "asm: b/bl: symbolic target required");
   emit_branch_imm(d, op_bl, sym, off, 0);
 }
 
 static void p_b_cond(AsmDriver* d, u32 cond) {
   ObjSymId sym = OBJ_SYM_NONE;
   i64 off = 0;
   parse_imm_sym(d, &sym, &off);
   if (sym == OBJ_SYM_NONE)
     asm_driver_panic(d, "asm: b.cond: symbolic target required");
   /* Emit the instruction with imm19=0 + R_AARCH64_CONDBR19 reloc. */
   u32 word = aa64_brcond_pack((AA64BrCond){.imm19 = 0, .cond = cond});
   emit32(d, word);
   MCEmitter* mc = asm_driver_mc(d);
   u32 ofs = mc->pos(mc) - 4;
   mc->emit_reloc_at(mc, asm_driver_cur_section(d), ofs, R_AARCH64_CONDBR19, sym,
                     off, 1, 0);
 }
 
 static void p_cbz(AsmDriver* d, u32 op) {
   AA64Reg rt = parse_reg(d);
   expect_comma(d, "cbz");
   ObjSymId sym = OBJ_SYM_NONE;
   i64 off = 0;
   parse_imm_sym(d, &sym, &off);
   if (sym == OBJ_SYM_NONE)
     asm_driver_panic(d, "asm: cbz: symbolic target required");
   u32 word =
       aa64_cb_pack((AA64CB){.sf = rt.is64, .op = op, .imm19 = 0, .Rt = rt.num});
   emit32(d, word);
   MCEmitter* mc = asm_driver_mc(d);
   u32 ofs = mc->pos(mc) - 4;
   mc->emit_reloc_at(mc, asm_driver_cur_section(d), ofs, R_AARCH64_CONDBR19, sym,
                     off, 1, 0);
 }
 
 /* Memory-operand parser.  Recognized shapes:
  *   [Xn]                                base only
  *   [Xn, #imm]                          base + immediate offset
  *   [Xn, #imm]!                         pre-index (writeback)
  *   [Xn], #imm                          post-index (writeback)
  *   [Xn, Xm]                            register offset (LSL #0)
  *   [Xn, Xm, LSL #s]                    register offset, scaled
  *   [Xn, Wm, {U,S}XTW {#s}]             32-bit index, extended
  *   [Xn, Xm, {U,S}XTX {#s}] / SXTX      64-bit index, extended
  *
  * imm is the literal byte offset (no scaling).  When has_index is set,
  * `index` is the index register, `option` its 3-bit extend code, and
  * shift_present records whether an explicit `#s` was written (with the
  * amount in `shift`).  pre_index / post_index flag the writeback forms. */
 typedef struct AA64Mem {
   AA64Reg base;
   AA64Reg index;
   i64 imm; /* byte offset (literal as written) */
   u32 option;
   u32 shift;
   AA64RelMod reloc_mod; /* :lo12: / :got_lo12: on the offset, or NONE */
   ObjSymId reloc_sym;   /* symbol when reloc_mod != NONE */
   i64 reloc_off;        /* addend when reloc_mod != NONE */
   u8 pre_index;
   u8 post_index;
   u8 has_offset;
   u8 has_index;
   u8 shift_present;
   u8 pad[3];
 } AA64Mem;
 
 /* Parse the optional extend/shift modifier of a register-offset memory
  * operand: `LSL #s`, `UXTW {#s}`, `SXTW {#s}`, `UXTX {#s}`, `SXTX {#s}`.
  * The index register width (32 vs 64) must agree with the extend kind.
  * Fills m->option / m->shift / m->shift_present. */
 static void parse_mem_extend(AsmDriver* d, AA64Mem* m) {
   AsmTok t = asm_driver_next(d);
   if (t.kind != ASM_TOK_IDENT)
     asm_driver_panic(d, "asm: ldr/str: expected extend (lsl/sxtw/uxtw/...)");
   Slice sl = pool_slice(asm_driver_pool(d), t.v.ident);
   const char* p = sl.s;
   size_t n = sl.len;
   int need64 = 0; /* index must be 64-bit */
   if (icase_eq(p, n, "lsl") || icase_eq(p, n, "uxtx")) {
     m->option = AA64_LDST_OPTION_LSL;
     need64 = 1;
   } else if (icase_eq(p, n, "sxtx")) {
     m->option = AA64_LDST_OPTION_SXTX;
     need64 = 1;
   } else if (icase_eq(p, n, "uxtw")) {
     m->option = AA64_LDST_OPTION_UXTW;
     need64 = 0;
   } else if (icase_eq(p, n, "sxtw")) {
     m->option = AA64_LDST_OPTION_SXTW;
     need64 = 0;
   } else {
     asm_driver_panic(d, "asm: ldr/str: unknown index extend");
   }
   if (need64 && !m->index.is64)
     asm_driver_panic(d, "asm: ldr/str: index must be 64-bit for this extend");
   if (!need64 && m->index.is64)
     asm_driver_panic(d, "asm: ldr/str: index must be 32-bit for sxtw/uxtw");
   /* LSL requires an explicit shift; the extends accept an optional one. */
   AsmTok nt = asm_driver_peek(d);
   if (tok_punct(nt, '#') || nt.kind == ASM_TOK_NUM) {
     i64 s = parse_imm_const(d);
     if (s < 0) asm_driver_panic(d, "asm: ldr/str: negative index shift");
     m->shift = (u32)s;
     m->shift_present = 1;
   } else if (m->option == AA64_LDST_OPTION_LSL) {
     asm_driver_panic(d, "asm: ldr/str: lsl requires a shift amount");
   }
 }
 
 static AA64Mem parse_mem(AsmDriver* d) {
   AA64Mem m;
   memset(&m, 0, sizeof m);
   if (!asm_driver_eat_punct(d, '[')) asm_driver_panic(d, "asm: expected '['");
   m.base = parse_reg(d);
   if (!m.base.is64)
     asm_driver_panic(d, "asm: ldr/str: base register must be 64-bit");
   require_sp_spelling(d, m.base, "ldr/str base");
   if (asm_driver_eat_comma(d)) {
     /* A relocation offset (ELF `:lo12:sym`/`:got_lo12:sym` prefix, or Mach-O
      * `sym@PAGEOFF`/`sym@GOTPAGEOFF` suffix), a register index, or a plain
      * `#imm`/expression. */
     AsmTok t = asm_driver_peek(d);
     AA64Reg idx;
     memset(&idx, 0, sizeof idx);
     if (!target_is_macho(d) && tok_punct(t, ':')) {
       m.reloc_mod = parse_reloc_mod(d);
       parse_imm_sym(d, &m.reloc_sym, &m.reloc_off);
       m.has_offset = 1; /* imm field stays 0; reloc supplies the low bits */
     } else if (t.kind == ASM_TOK_IDENT &&
                parse_reg_from_ident(d, t.v.ident, &idx)) {
       (void)asm_driver_next(d);
       reject_sp_reg(d, idx, "ldr/str index");
       m.index = idx;
       m.has_index = 1;
       m.option = idx.is64 ? AA64_LDST_OPTION_LSL : AA64_LDST_OPTION_UXTW;
       if (asm_driver_eat_comma(d)) parse_mem_extend(d, &m);
     } else if (target_is_macho(d) && t.kind == ASM_TOK_IDENT) {
       /* Mach-O: `[Xn, sym@PAGEOFF]` / `[Xn, sym@GOTPAGEOFF]`. */
       parse_imm_sym(d, &m.reloc_sym, &m.reloc_off);
       m.reloc_mod = parse_reloc_suffix(d);
       if (m.reloc_mod != AA64_RELMOD_LO12 &&
           m.reloc_mod != AA64_RELMOD_GOT_LO12)
         asm_driver_panic(
             d, "asm: ldr/str: symbol offset needs @PAGEOFF/@GOTPAGEOFF");
       m.has_offset = 1;
     } else {
       m.imm = parse_imm_const(d);
       m.has_offset = 1;
     }
   }
   if (!asm_driver_eat_punct(d, ']')) asm_driver_panic(d, "asm: expected ']'");
   if (asm_driver_eat_punct(d, '!')) {
     if (m.has_index)
       asm_driver_panic(d, "asm: ldr/str: writeback not allowed with index");
     m.pre_index = 1;
   } else if (asm_driver_eat_comma(d)) {
     /* Post-index: `[Xn], #imm`. */
     if (m.has_index || m.has_offset)
       asm_driver_panic(d, "asm: ldr/str: malformed post-index operand");
     m.imm = parse_imm_const(d);
     m.has_offset = 1;
     m.post_index = 1;
   }
   return m;
 }
 
 /* ldr/str Rt, [Xn, #imm] — chooses scaled or unscaled form based on
  * alignment of imm. */
 /* Core load/store. `fixed_size` is the access log2-size (0=byte..3=dword) for
  * ldrb/ldrh/ldrsw etc., or -1 to derive it from the register width (ldr/str).
  * `sign_ext` selects the signed-load opc (10 = sign-extend to 64-bit, 11 = to
  * 32-bit), keyed on the destination register width. */
 static void p_ldst_core(AsmDriver* d, int is_load, int fixed_size,
                         int sign_ext) {
   AA64Reg rt = parse_ldst_reg(d);
   reject_sp_reg(d, rt, "ldr/str");
   expect_comma(d, "ldr/str");
   AA64Mem m = parse_mem(d);
   AA64LdStEnc e = ldst_encoding(d, rt, is_load, fixed_size, sign_ext);
   u32 size = e.size, opc = e.opc, V = e.V;
   if (m.reloc_mod != AA64_RELMOD_NONE) {
     /* [Xn, :lo12:sym] / [Xn, :got_lo12:sym] — unsigned-imm12 form with a zero
      * immediate; the relocation supplies the low 12 bits.  :got_lo12: only
      * applies to a 64-bit `ldr` (the GOT entry is an 8-byte pointer); llvm-mc
      * rejects it on stores, signed loads, and sub-word loads. */
     if (m.reloc_mod == AA64_RELMOD_GOT_LO12 &&
         !(V == 0 && size == 3 && opc == AA64_LDST_OPC_LDR))
       asm_driver_panic(d, "asm: :got_lo12: only valid on a 64-bit ldr");
     u32 word = aa64_ldst_uimm_pack((AA64LdStUimm){.size = size,
                                                   .V = V,
                                                   .opc = opc,
                                                   .imm12 = 0,
                                                   .Rn = m.base.num,
                                                   .Rt = rt.num});
     emit32(d, word);
     RelocKind k = (m.reloc_mod == AA64_RELMOD_GOT_LO12)
                       ? R_AARCH64_LD64_GOT_LO12_NC
                       : aa64_ldst_lo12_reloc(d, size);
     MCEmitter* mc = asm_driver_mc(d);
     mc->emit_reloc_at(mc, asm_driver_cur_section(d), mc->pos(mc) - 4, k,
                       m.reloc_sym, m.reloc_off, 1, 0);
     return;
   }
   if (m.has_index) {
     /* Register-offset form.  The S bit (scale by access size) is set when an
      * explicit shift equal to the access log2-size is written.  An explicit
      * `#0` is legal and stays unscaled (S=0); for byte access #0 == size so it
      * sets S — matching llvm-mc, where the only legal amounts are 0 or size. */
     u32 S = 0;
     if (m.shift_present) {
       if (m.shift == size)
         S = 1;
       else if (m.shift != 0)
         asm_driver_panic(d,
                          "asm: ldr/str: index shift must be 0 or access size");
     }
     u32 word = aa64_ldst_regoff_pack((AA64LdStRegOff){.size = size,
                                                       .V = V,
                                                       .opc = opc,
                                                       .Rm = m.index.num,
                                                       .option = m.option,
                                                       .S = S,
                                                       .Rn = m.base.num,
                                                       .Rt = rt.num});
     emit32(d, word);
     return;
   }
   if (m.pre_index || m.post_index) {
     /* Immediate writeback (unscaled signed imm9). */
     if (m.imm < -256 || m.imm > 255)
       asm_driver_panic(d, "asm: ldr/str: writeback imm9 out of range");
     u32 imm9 = (u32)((u64)m.imm & 0x1ffu);
     u32 idx = m.pre_index ? AA64_LDST_IDX_PRE : AA64_LDST_IDX_POST;
     u32 word = aa64_ldst_wback_pack((AA64LdStWBack){.size = size,
                                                     .V = V,
                                                     .opc = opc,
                                                     .imm9 = imm9,
                                                     .idx = idx,
                                                     .Rn = m.base.num,
                                                     .Rt = rt.num});
     emit32(d, word);
     return;
   }
   {
     /* Try scaled unsigned-imm12 first. */
     u32 scale = e.scale;
     if (m.imm >= 0 && (i64)((u64)m.imm % scale) == 0 &&
         (u64)m.imm / scale <= 0xfff) {
       u32 imm12 = (u32)((u64)m.imm / scale);
       u32 word = aa64_ldst_uimm_pack((AA64LdStUimm){.size = size,
                                                     .V = V,
                                                     .opc = opc,
                                                     .imm12 = imm12,
                                                     .Rn = m.base.num,
                                                     .Rt = rt.num});
       emit32(d, word);
       return;
     }
     /* Fall back to unscaled signed-imm9 (LDUR/STUR). */
     if (m.imm >= -256 && m.imm <= 255) {
       u32 imm9 = (u32)((u64)m.imm & 0x1ffu);
       u32 word = aa64_ldst_simm9_pack((AA64LdStSimm9){.size = size,
                                                       .V = V,
                                                       .opc = opc,
                                                       .imm9 = imm9,
                                                       .Rn = m.base.num,
                                                       .Rt = rt.num});
       emit32(d, word);
       return;
     }
     asm_driver_panic(d, "asm: ldr/str: immediate out of range");
   }
 }
 
 /* ldr/str: access width follows the register (Wt=word, Xt=dword). */
 static void p_ldr_str(AsmDriver* d, int is_load) {
   p_ldst_core(d, is_load, /*fixed_size=*/-1, /*sign_ext=*/0);
 }
 /* Byte/half + signed sub-word loads/stores (fixed access width). */
 static void p_ldrb(AsmDriver* d) { p_ldst_core(d, 1, 0, 0); }
 static void p_strb(AsmDriver* d) { p_ldst_core(d, 0, 0, 0); }
 static void p_ldrh(AsmDriver* d) { p_ldst_core(d, 1, 1, 0); }
 static void p_strh(AsmDriver* d) { p_ldst_core(d, 0, 1, 0); }
 static void p_ldrsb(AsmDriver* d) { p_ldst_core(d, 1, 0, 1); }
 static void p_ldrsh(AsmDriver* d) { p_ldst_core(d, 1, 1, 1); }
 static void p_ldrsw(AsmDriver* d) { p_ldst_core(d, 1, 2, 1); }
 
 /* ldur/stur — unscaled signed-imm9. `fixed_size` is the access log2-size
  * (0=byte..3=dword) for sturb/ldurb/sturh/ldurh/ldursw etc., or -1 to derive
  * it from the register width (stur/ldur). `sign_ext` selects the signed-load
  * opc (ldursb/ldursh/ldursw), keyed on the destination register width — the
  * unscaled mirror of p_ldst_core. */
 static void p_ldur_stur(AsmDriver* d, int is_load, int fixed_size,
                         int sign_ext) {
   AA64Reg rt = parse_ldst_reg(d);
   reject_sp_reg(d, rt, "ldur/stur");
   expect_comma(d, "ldur/stur");
   AA64Mem m = parse_mem(d);
   AA64LdStEnc e = ldst_encoding(d, rt, is_load, fixed_size, sign_ext);
   if (m.imm < -256 || m.imm > 255)
     asm_driver_panic(d, "asm: ldur/stur: imm9 out of range");
   u32 imm9 = (u32)((u64)m.imm & 0x1ffu);
   u32 word = aa64_ldst_simm9_pack((AA64LdStSimm9){.size = e.size,
                                                   .V = e.V,
                                                   .opc = e.opc,
                                                   .imm9 = imm9,
                                                   .Rn = m.base.num,
                                                   .Rt = rt.num});
   emit32(d, word);
 }
 
 /* ldp / stp Rt, Rt2, [Xn, #imm] or [Xn, #imm]! */
 static void p_ldp_stp(AsmDriver* d, int is_load) {
   AA64Reg rt = parse_ldstp_reg(d);
   expect_comma(d, "ldp/stp");
   AA64Reg rt2 = parse_ldstp_reg(d);
   expect_comma(d, "ldp/stp");
   reject_sp_reg(d, rt, "ldp/stp");
   reject_sp_reg(d, rt2, "ldp/stp");
   if (rt.is64 != rt2.is64 || rt.is_fp != rt2.is_fp ||
       rt.fp_bytes != rt2.fp_bytes)
     asm_driver_panic(d, "asm: ldp/stp: width mismatch");
   AA64Mem m = parse_mem(d);
   u32 scale = rt.is_fp ? (u32)rt.fp_bytes : (rt.is64 ? 8u : 4u);
   if ((i64)((u64)m.imm % scale) != 0)
     asm_driver_panic(d, "asm: ldp/stp: imm not scale-aligned");
   i64 imm7 = m.imm / (i64)scale;
   if (imm7 < -64 || imm7 > 63)
     asm_driver_panic(d, "asm: ldp/stp: imm7 out of range");
   AA64LdStPPre f = {
       .opc = rt.is_fp ? (rt.fp_bytes == 16u ? 2u : 1u) : (rt.is64 ? 2u : 0u),
       .V = rt.is_fp ? 1u : 0u,
       .L = is_load ? 1u : 0u,
       .imm7 = (u32)imm7 & 0x7fu,
       .Rt2 = rt2.num,
       .Rn = m.base.num,
       .Rt = rt.num};
   if (m.pre_index)
     emit32(d, aa64_ldstp_pre_pack(f));
   else if (m.post_index)
     emit32(d, aa64_ldstp_post_pack(f));
   else
     emit32(d, aa64_ldstp_soff_pack(f));
 }
 
 /* adr / adrp Rd, sym */
 static void p_adr(AsmDriver* d, int is_adrp) {
   AA64Reg rd = parse_reg(d);
   expect_comma(d, "adr");
   /* adrp page reloc on a symbol: ELF spells a bare symbol (`:got:` selects the
    * GOT page); Mach-O spells `sym@PAGE` / `sym@GOTPAGE`.  adr takes a bare
    * symbol on both.  cc -S emits the form matching the target format. */
   AA64RelMod mod = AA64_RELMOD_NONE;
   ObjSymId sym = OBJ_SYM_NONE;
   i64 off = 0;
   if (target_is_macho(d)) {
     parse_imm_sym(d, &sym, &off);
     mod = parse_reloc_suffix(d);
   } else {
     mod = parse_reloc_mod(d);
     parse_imm_sym(d, &sym, &off);
   }
   if (!is_adrp) {
     if (mod != AA64_RELMOD_NONE)
       asm_driver_panic(d, "asm: adr: no relocation modifier valid here");
   } else if (mod != AA64_RELMOD_NONE && mod != AA64_RELMOD_PAGE &&
              mod != AA64_RELMOD_GOT) {
     asm_driver_panic(d,
                      "asm: adrp: only @PAGE/@GOTPAGE (Mach-O) or :got: "
                      "(ELF) valid here");
   }
   if (sym == OBJ_SYM_NONE)
     asm_driver_panic(d, "asm: adr/adrp: symbol required");
   AA64PCRelAdr f = {.op = is_adrp ? AA64_ADR_OP_ADRP : AA64_ADR_OP_ADR,
                     .immlo = 0,
                     .immhi = 0,
                     .Rd = rd.num};
   emit32(d, aa64_pcrel_adr_pack(f));
   MCEmitter* mc = asm_driver_mc(d);
   u32 ofs = mc->pos(mc) - 4;
   RelocKind k = !is_adrp                 ? R_AARCH64_ADR_PREL_LO21
                 : mod == AA64_RELMOD_GOT ? R_AARCH64_ADR_GOT_PAGE
                                          : R_AARCH64_ADR_PREL_PG_HI21;
   mc->emit_reloc_at(mc, asm_driver_cur_section(d), ofs, k, sym, off, 1, 0);
 }
 
 /* ---- atomics / exclusive ----
  *
  * Every form here addresses a bare base register `[Xn]` (no offset, no
  * index, no writeback).  parse_mem already rejects malformed shapes; we
  * additionally reject any offset/index so `ldxr w0,[x1,#4]` is an error,
  * matching llvm/gas. */
 static AA64Mem parse_mem_bare(AsmDriver* d, const char* what) {
   AA64Mem m = parse_mem(d);
   if (m.has_offset || m.has_index || m.pre_index || m.post_index)
     asm_driver_panic(d, "asm: %.*s: expected bare [Xn] address",
                      SLICE_ARG(slice_from_cstr(what)));
   return m;
 }
 
 /* Map an access log2-size (0..3) onto the GPR width the operand register
  * must have: byte/half/word use Wt (32-bit), dword uses Xt (64-bit). */
 static void require_gpr_width(AsmDriver* d, AA64Reg r, u32 size,
                               const char* what) {
   reject_sp_reg(d, r, what);
   u32 want64 = (size == 3u) ? 1u : 0u;
   if ((u32)r.is64 != want64)
     asm_driver_panic(d, "asm: %.*s: register width mismatch",
                      SLICE_ARG(slice_from_cstr(what)));
 }
 
 /* Load-exclusive / load-acquire: `<op> Wt|Xt, [Xn]`.
  *   o2/o0 select the family member (see aa64_ldstex_pack).  size is the
  *   access log2-size; Rs/Rt2 are fixed to 11111. */
 static void p_ldex(AsmDriver* d, u32 size, u32 o2, u32 o0, const char* what) {
   AA64Reg rt = parse_reg(d);
   require_gpr_width(d, rt, size, what);
   expect_comma(d, what);
   AA64Mem m = parse_mem_bare(d, what);
   emit32(d, aa64_ldstex_pack((AA64LdStEx){.size = size,
                                           .o2 = o2,
                                           .L = 1u,
                                           .o1 = 0u,
                                           .Rs = AA64_ZR,
                                           .o0 = o0,
                                           .Rt2 = AA64_ZR,
                                           .Rn = m.base.num,
                                           .Rt = rt.num}));
 }
 
 /* Store-release without status: `stlr Wt|Xt, [Xn]` (o2=1, L=0, o0=1). */
 static void p_stlr(AsmDriver* d, u32 size, const char* what) {
   AA64Reg rt = parse_reg(d);
   require_gpr_width(d, rt, size, what);
   expect_comma(d, what);
   AA64Mem m = parse_mem_bare(d, what);
   emit32(d, aa64_ldstex_pack((AA64LdStEx){.size = size,
                                           .o2 = 1u,
                                           .L = 0u,
                                           .o1 = 0u,
                                           .Rs = AA64_ZR,
                                           .o0 = 1u,
                                           .Rt2 = AA64_ZR,
                                           .Rn = m.base.num,
                                           .Rt = rt.num}));
 }
 
 /* Store-exclusive with status: `<op> Ws, Wt|Xt, [Xn]` (L=0).  Ws (the
  * 32-bit status result) must be a W register and distinct from Rt/Rn. */
 /* Store-exclusive constraint (ARM ARM): the status register Ws must differ
  * from the stored value Rt and from the base Rn, else the result is
  * UNPREDICTABLE.  The base is exempt when it is SP (reg #31 names SP, not the
  * WZR the status reg would be).  CAS/LSE atomics do NOT share this rule. */
 static void reject_stex_alias(AsmDriver* d, AA64Reg rs, AA64Reg rt, AA64Mem m,
                               const char* what) {
   if (rs.num == rt.num)
     asm_driver_panic(d, "asm: %.*s: status reg cannot be the value reg",
                      SLICE_ARG(slice_from_cstr(what)));
   if (!m.base.is_sp && rs.num == m.base.num)
     asm_driver_panic(d, "asm: %.*s: status reg cannot be the base reg",
                      SLICE_ARG(slice_from_cstr(what)));
 }
 
 static void p_stex(AsmDriver* d, u32 size, u32 o0, const char* what) {
   AA64Reg rs = parse_reg(d);
   reject_sp_reg(d, rs, what);
   if (rs.is64)
     asm_driver_panic(d, "asm: %.*s: status reg must be 32-bit",
                      SLICE_ARG(slice_from_cstr(what)));
   expect_comma(d, what);
   AA64Reg rt = parse_reg(d);
   require_gpr_width(d, rt, size, what);
   expect_comma(d, what);
   AA64Mem m = parse_mem_bare(d, what);
   reject_stex_alias(d, rs, rt, m, what);
   emit32(d, aa64_ldstex_pack((AA64LdStEx){.size = size,
                                           .o2 = 0u,
                                           .L = 0u,
                                           .o1 = 0u,
                                           .Rs = rs.num,
                                           .o0 = o0,
                                           .Rt2 = AA64_ZR,
                                           .Rn = m.base.num,
                                           .Rt = rt.num}));
 }
 
 /* Compare-and-swap: `<op> Ws, Wt, [Xn]` / `<op> Xs, Xt, [Xn]`.  Rs and Rt
  * share the operand width selected by `size` (word or dword). */
 static void p_cas(AsmDriver* d, u32 size, u32 L, u32 o0, const char* what) {
   AA64Reg rs = parse_reg(d);
   require_gpr_width(d, rs, size, what);
   expect_comma(d, what);
   AA64Reg rt = parse_reg(d);
   require_gpr_width(d, rt, size, what);
   expect_comma(d, what);
   AA64Mem m = parse_mem_bare(d, what);
   emit32(d, aa64_cas_pack((AA64Cas){.size = size,
                                     .L = L,
                                     .Rs = rs.num,
                                     .o0 = o0,
                                     .Rn = m.base.num,
                                     .Rt = rt.num}));
 }
 
 /* LSE atomic memory op: `<op> Ws, Wt, [Xn]` / `<op> Xs, Xt, [Xn]`.
  *   o3=1 selects SWP; otherwise opc names LDADD/LDCLR/LDEOR/LDSET. */
 static void p_lse(AsmDriver* d, u32 size, u32 A, u32 R, u32 o3, u32 opc,
                   const char* what) {
   AA64Reg rs = parse_reg(d);
   require_gpr_width(d, rs, size, what);
   expect_comma(d, what);
   AA64Reg rt = parse_reg(d);
   require_gpr_width(d, rt, size, what);
   expect_comma(d, what);
   AA64Mem m = parse_mem_bare(d, what);
   emit32(d, aa64_lse_atomic_pack((AA64LseAtomic){.size = size,
                                                  .A = A,
                                                  .R = R,
                                                  .Rs = rs.num,
                                                  .o3 = o3,
                                                  .opc = opc,
                                                  .Rn = m.base.num,
                                                  .Rt = rt.num}));
 }
 
 /* ---- mnemonic dispatch table ---- */
 
 typedef void (*P_Fn)(AsmDriver*);
 
 typedef struct AA64Mn {
   const char* name;
   P_Fn fn;
   u32 arg; /* per-fn discriminator (alias parameter) */
 } AA64Mn;
 
 /* Wrapper functions for the discriminator-taking parsers, since the
  * table holds a uniform P_Fn pointer.  Each wraps a single (fn, arg)
  * tuple. */
 static void p_addsub_add(AsmDriver* d) { p_addsub(d, /*is_sub=*/0, 0); }
 static void p_addsub_adds(AsmDriver* d) { p_addsub(d, 0, 1); }
 static void p_addsub_sub(AsmDriver* d) { p_addsub(d, 1, 0); }
 static void p_addsub_subs(AsmDriver* d) { p_addsub(d, 1, 1); }
 static void p_cmp_w(AsmDriver* d) { p_cmp(d, 0); }
 static void p_cmn_w(AsmDriver* d) { p_cmp(d, 1); }
 static void p_csel_(AsmDriver* d) { p_condsel(d, 0, 0, "csel"); }
 static void p_csinc_(AsmDriver* d) { p_condsel(d, 0, 1, "csinc"); }
 static void p_csinv_(AsmDriver* d) { p_condsel(d, 1, 0, "csinv"); }
 static void p_csneg_(AsmDriver* d) { p_condsel(d, 1, 1, "csneg"); }
 static void p_cset_(AsmDriver* d) { p_cset_like(d, 0, 1, "cset"); }
 static void p_csetm_(AsmDriver* d) { p_cset_like(d, 1, 0, "csetm"); }
 static void p_neg_w(AsmDriver* d) { p_neg(d, 0); }
 static void p_negs_w(AsmDriver* d) { p_neg(d, 1); }
 static void p_and_w(AsmDriver* d) { p_log_sr(d, AA64_LOG_AND_OPC, 0); }
 static void p_bic_w(AsmDriver* d) { p_log_sr(d, AA64_LOG_AND_OPC, 1); }
 static void p_orr_w(AsmDriver* d) { p_log_sr(d, AA64_LOG_ORR_OPC, 0); }
 static void p_orn_w(AsmDriver* d) { p_log_sr(d, AA64_LOG_ORR_OPC, 1); }
 static void p_eor_w(AsmDriver* d) { p_log_sr(d, AA64_LOG_EOR_OPC, 0); }
 static void p_eon_w(AsmDriver* d) { p_log_sr(d, AA64_LOG_EOR_OPC, 1); }
 static void p_ands_w(AsmDriver* d) { p_log_sr(d, AA64_LOG_ANDS_OPC, 0); }
 static void p_bics_w(AsmDriver* d) { p_log_sr(d, AA64_LOG_ANDS_OPC, 1); }
 static void p_madd(AsmDriver* d) { p_dp3(d, 0); }
 static void p_msub(AsmDriver* d) { p_dp3(d, 1); }
 static void p_mul_w(AsmDriver* d) { p_mul(d, 0); }
 static void p_mneg_w(AsmDriver* d) { p_mul(d, 1); }
 static void p_udiv_w(AsmDriver* d) { p_dp2(d, AA64_DP2_UDIV_OP); }
 static void p_sdiv_w(AsmDriver* d) { p_dp2(d, AA64_DP2_SDIV_OP); }
 static void p_lslv_w(AsmDriver* d) { p_dp2(d, AA64_DP2_LSLV_OP); }
 static void p_lsrv_w(AsmDriver* d) { p_dp2(d, AA64_DP2_LSRV_OP); }
 static void p_asrv_w(AsmDriver* d) { p_dp2(d, AA64_DP2_ASRV_OP); }
 static void p_rorv_w(AsmDriver* d) { p_dp2(d, AA64_DP2_RORV_OP); }
 static void p_lsl_(AsmDriver* d) { p_shift(d, 0); }
 static void p_lsr_(AsmDriver* d) { p_shift(d, 1); }
 static void p_asr_(AsmDriver* d) { p_shift(d, 2); }
 static void p_b_(AsmDriver* d) { p_b(d, 0); }
 static void p_bl_(AsmDriver* d) { p_b(d, 1); }
 static void p_cbz_(AsmDriver* d) { p_cbz(d, 0); }
 static void p_cbnz_(AsmDriver* d) { p_cbz(d, 1); }
 static void p_movz_(AsmDriver* d) { p_movwide(d, AA64_MOVZ_OPC); }
 static void p_movn_(AsmDriver* d) { p_movwide(d, AA64_MOVN_OPC); }
 static void p_movk_(AsmDriver* d) { p_movwide(d, AA64_MOVK_OPC); }
 static void p_svc_(AsmDriver* d) { p_except(d, 0); }
 static void p_brk_(AsmDriver* d) { p_except(d, 1); }
 static void p_hlt_(AsmDriver* d) { p_except(d, 2); }
 static void p_ldr_(AsmDriver* d) { p_ldr_str(d, 1); }
 static void p_str_(AsmDriver* d) { p_ldr_str(d, 0); }
 static void p_ldur_(AsmDriver* d) { p_ldur_stur(d, 1, -1, 0); }
 static void p_stur_(AsmDriver* d) { p_ldur_stur(d, 0, -1, 0); }
 static void p_ldurb(AsmDriver* d) { p_ldur_stur(d, 1, 0, 0); }
 static void p_sturb(AsmDriver* d) { p_ldur_stur(d, 0, 0, 0); }
 static void p_ldurh(AsmDriver* d) { p_ldur_stur(d, 1, 1, 0); }
 static void p_sturh(AsmDriver* d) { p_ldur_stur(d, 0, 1, 0); }
 static void p_ldursb(AsmDriver* d) { p_ldur_stur(d, 1, 0, 1); }
 static void p_ldursh(AsmDriver* d) { p_ldur_stur(d, 1, 1, 1); }
 static void p_ldursw(AsmDriver* d) { p_ldur_stur(d, 1, 2, 1); }
 static void p_ldp_(AsmDriver* d) { p_ldp_stp(d, 1); }
 static void p_stp_(AsmDriver* d) { p_ldp_stp(d, 0); }
 static void p_adr_(AsmDriver* d) { p_adr(d, 0); }
 static void p_adrp_(AsmDriver* d) { p_adr(d, 1); }
 
 /* b.cond family.  cond codes follow the standard ARMv8 numbering. */
 static void p_b_eq(AsmDriver* d) { p_b_cond(d, 0); }
 static void p_b_ne(AsmDriver* d) { p_b_cond(d, 1); }
 static void p_b_cs(AsmDriver* d) { p_b_cond(d, 2); }
 static void p_b_hs(AsmDriver* d) { p_b_cond(d, 2); }
 static void p_b_cc(AsmDriver* d) { p_b_cond(d, 3); }
 static void p_b_lo(AsmDriver* d) { p_b_cond(d, 3); }
 static void p_b_mi(AsmDriver* d) { p_b_cond(d, 4); }
 static void p_b_pl(AsmDriver* d) { p_b_cond(d, 5); }
 static void p_b_vs(AsmDriver* d) { p_b_cond(d, 6); }
 static void p_b_vc(AsmDriver* d) { p_b_cond(d, 7); }
 static void p_b_hi(AsmDriver* d) { p_b_cond(d, 8); }
 static void p_b_ls(AsmDriver* d) { p_b_cond(d, 9); }
 static void p_b_ge(AsmDriver* d) { p_b_cond(d, 10); }
 static void p_b_lt(AsmDriver* d) { p_b_cond(d, 11); }
 static void p_b_gt(AsmDriver* d) { p_b_cond(d, 12); }
 static void p_b_le(AsmDriver* d) { p_b_cond(d, 13); }
 static void p_b_al(AsmDriver* d) { p_b_cond(d, 14); }
 
 /* ---- Scalar floating-point ----
  * Sn/Dn/Hn are the single/double/half views of the FP register file; the
  * 2-bit ftype (0=s,1=d,3=h) drives both the encoding and the operand text. */
 static int parse_fp_scalar_from_ident(AsmDriver* d, Sym ident, u32* num,
                                       u32* ftype) {
   Slice sl = pool_slice(asm_driver_pool(d), ident);
   const char* p = sl.s;
   size_t n = sl.len;
   u32 ft, r = 0;
   size_t i;
   if (!p || n < 2) return 0;
   if (p[0] == 's' || p[0] == 'S')
     ft = 0u;
   else if (p[0] == 'd' || p[0] == 'D')
     ft = 1u;
   else if (p[0] == 'h' || p[0] == 'H')
     ft = 3u;
   else
     return 0;
   for (i = 1; i < n; ++i) {
     char c = p[i];
     if (c < '0' || c > '9') return 0;
     r = r * 10u + (u32)(c - '0');
     if (r > 31u) return 0;
   }
   *num = r;
   *ftype = ft;
   return 1;
 }
 
 static void parse_fp_scalar(AsmDriver* d, u32* num, u32* ftype) {
   AsmTok t = asm_driver_next(d);
   if (t.kind != ASM_TOK_IDENT ||
       !parse_fp_scalar_from_ident(d, t.v.ident, num, ftype))
     asm_driver_panic(d, "asm: expected FP register (Sn/Dn/Hn)");
 }
 
 /* A register operand that may be either a GPR or a scalar FP register — used
  * by fmov, whose three forms differ only by operand class. */
 typedef struct FpOrGpr {
   int is_fp;
   u32 num;
   u32 ftype; /* when is_fp */
   int is64;  /* when !is_fp */
 } FpOrGpr;
 
 static FpOrGpr parse_fp_or_gpr(AsmDriver* d) {
   AsmTok t = asm_driver_next(d);
   FpOrGpr r;
   AA64Reg g;
   memset(&r, 0, sizeof r);
   if (t.kind == ASM_TOK_IDENT &&
       parse_fp_scalar_from_ident(d, t.v.ident, &r.num, &r.ftype)) {
     r.is_fp = 1;
     return r;
   }
   memset(&g, 0, sizeof g);
   if (t.kind == ASM_TOK_IDENT && parse_reg_from_ident(d, t.v.ident, &g)) {
     r.is_fp = 0;
     r.num = g.num;
     r.is64 = (int)g.is64;
     return r;
   }
   asm_driver_panic(d, "asm: fmov: expected register");
   return r; /* unreachable */
 }
 
 static void p_fp_dp2(AsmDriver* d, u32 op) {
   u32 rd, rn, rm, ftd, ftn, ftm;
   parse_fp_scalar(d, &rd, &ftd);
   expect_comma(d, "fp");
   parse_fp_scalar(d, &rn, &ftn);
   expect_comma(d, "fp");
   parse_fp_scalar(d, &rm, &ftm);
   if (ftd != ftn || ftd != ftm)
     asm_driver_panic(d, "asm: fp: operand type mismatch");
   emit32(d, aa64_fp_dp2(ftd, op, rd, rn, rm));
 }
 static void p_fp_dp1(AsmDriver* d, u32 op) {
   u32 rd, rn, ftd, ftn;
   parse_fp_scalar(d, &rd, &ftd);
   expect_comma(d, "fp");
   parse_fp_scalar(d, &rn, &ftn);
   if (ftd != ftn) asm_driver_panic(d, "asm: fp: operand type mismatch");
   emit32(d, aa64_fp_dp1(ftd, op, rd, rn));
 }
 static void p_fadd(AsmDriver* d) { p_fp_dp2(d, AA64_FP_DP2_FADD); }
 static void p_fsub(AsmDriver* d) { p_fp_dp2(d, AA64_FP_DP2_FSUB); }
 static void p_fmul(AsmDriver* d) { p_fp_dp2(d, AA64_FP_DP2_FMUL); }
 static void p_fdiv(AsmDriver* d) { p_fp_dp2(d, AA64_FP_DP2_FDIV); }
 static void p_fmax(AsmDriver* d) { p_fp_dp2(d, AA64_FP_DP2_FMAX); }
 static void p_fmin(AsmDriver* d) { p_fp_dp2(d, AA64_FP_DP2_FMIN); }
 static void p_fnmul(AsmDriver* d) { p_fp_dp2(d, AA64_FP_DP2_FNMUL); }
 static void p_fneg(AsmDriver* d) { p_fp_dp1(d, AA64_FP_DP1_FNEG); }
 static void p_fabs(AsmDriver* d) { p_fp_dp1(d, AA64_FP_DP1_FABS); }
 static void p_fsqrt(AsmDriver* d) { p_fp_dp1(d, AA64_FP_DP1_FSQRT); }
 
 static void p_fcmp(AsmDriver* d) {
   u32 rn, rm, ftn, ftm;
   parse_fp_scalar(d, &rn, &ftn);
   expect_comma(d, "fcmp");
   parse_fp_scalar(d, &rm, &ftm);
   if (ftn != ftm) asm_driver_panic(d, "asm: fcmp: operand type mismatch");
   emit32(d, aa64_fcmp_reg(ftn, rn, rm));
 }
 static void p_fcvt(AsmDriver* d) {
   u32 rd, rn, ftd, ftn;
   parse_fp_scalar(d, &rd, &ftd);
   expect_comma(d, "fcvt");
   parse_fp_scalar(d, &rn, &ftn);
   emit32(d, aa64_fcvt_prec(ftn /*src*/, ftd /*dst*/, rd, rn));
 }
 /* scvtf/ucvtf: FP dst, GPR src. */
 static void p_cvtf(AsmDriver* d, u32 opcode) {
   u32 fd, ft;
   AA64Reg rn;
   parse_fp_scalar(d, &fd, &ft);
   expect_comma(d, "cvtf");
   rn = parse_reg(d);
   emit32(d, aa64_fp_int_cvt((u32)rn.is64, ft, opcode, fd, rn.num));
 }
 /* fcvtzs/fcvtzu: GPR dst, FP src. */
 static void p_fcvtz(AsmDriver* d, u32 opcode) {
   AA64Reg rd;
   u32 fn, ft;
   rd = parse_reg(d);
   expect_comma(d, "fcvtz");
   parse_fp_scalar(d, &fn, &ft);
   emit32(d, aa64_fp_int_cvt((u32)rd.is64, ft, opcode, rd.num, fn));
 }
 static void p_scvtf(AsmDriver* d) { p_cvtf(d, AA64_FP_ICVT_SCVTF); }
 static void p_ucvtf(AsmDriver* d) { p_cvtf(d, AA64_FP_ICVT_UCVTF); }
 static void p_fcvtzs(AsmDriver* d) { p_fcvtz(d, AA64_FP_ICVT_FCVTZS); }
 static void p_fcvtzu(AsmDriver* d) { p_fcvtz(d, AA64_FP_ICVT_FCVTZU); }
 
 /* Data-processing (1 source): clz/rbit/rev16, and rev (whose opcode2 is the
  * width: 2 for 32-bit, 3 for 64-bit). */
 static void p_dp1_op(AsmDriver* d, u32 opcode2) {
   AA64Reg rd = parse_reg(d);
   AA64Reg rn;
   expect_comma(d, "dp1");
   rn = parse_reg(d);
   if (rd.is64 != rn.is64) asm_driver_panic(d, "asm: dp1: width mismatch");
   emit32(d, aa64_dp1(rd.is64, opcode2, rd.num, rn.num));
 }
 static void p_clz(AsmDriver* d) { p_dp1_op(d, AA64_DP1_CLZ); }
 static void p_rbit(AsmDriver* d) { p_dp1_op(d, AA64_DP1_RBIT); }
 static void p_rev16(AsmDriver* d) { p_dp1_op(d, AA64_DP1_REV16); }
 static void p_rev(AsmDriver* d) {
   AA64Reg rd = parse_reg(d);
   AA64Reg rn;
   expect_comma(d, "rev");
   rn = parse_reg(d);
   if (rd.is64 != rn.is64) asm_driver_panic(d, "asm: rev: width mismatch");
   emit32(d, aa64_dp1(rd.is64, rd.is64 ? AA64_DP1_REV64 : AA64_DP1_REV32, rd.num,
                      rn.num));
 }
 
 /* Bitfield move (opc: 0=sbfm, 1=bfm, 2=ubfm): Rd, Rn, #immr, #imms. */
 static void p_bitfield(AsmDriver* d, u32 opc) {
   AA64Reg rd = parse_reg(d);
   AA64Reg rn;
   i64 immr, imms;
   expect_comma(d, "bitfield");
   rn = parse_reg(d);
   expect_comma(d, "bitfield");
   immr = parse_imm_const(d);
   expect_comma(d, "bitfield");
   imms = parse_imm_const(d);
   if (rd.is64 != rn.is64) asm_driver_panic(d, "asm: bitfield: width mismatch");
   emit32(d, aa64_bitfield(rd.is64, opc, (u32)immr, (u32)imms, rd.num, rn.num));
 }
 static void p_sbfm(AsmDriver* d) { p_bitfield(d, 0u); }
 static void p_bfm(AsmDriver* d) { p_bitfield(d, 1u); }
 static void p_ubfm(AsmDriver* d) { p_bitfield(d, 2u); }
 
 static void p_bfx(AsmDriver* d, u32 opc, const char* what) {
   AA64Reg rd = parse_reg(d);
   AA64Reg rn;
   i64 lsb, width;
   u32 reg_width;
   expect_comma(d, what);
   rn = parse_reg(d);
   reject_sp_reg(d, rd, what);
   reject_sp_reg(d, rn, what);
   if (rd.is64 != rn.is64)
     asm_driver_panic(d, "asm: %.*s: width mismatch",
                      SLICE_ARG(slice_from_cstr(what)));
   expect_comma(d, what);
   lsb = parse_imm_const(d);
   expect_comma(d, what);
   width = parse_imm_const(d);
   reg_width = rd.is64 ? 64u : 32u;
   if (lsb < 0 || width <= 0 || (u64)lsb >= reg_width ||
       (u64)width > (u64)reg_width - (u64)lsb) {
     asm_driver_panic(d, "asm: %.*s: bit range out of bounds",
                      SLICE_ARG(slice_from_cstr(what)));
   }
   emit32(d, aa64_bitfield(rd.is64, opc, (u32)lsb, (u32)(lsb + width - 1),
                           rd.num, rn.num));
 }
 
 static void p_sbfx(AsmDriver* d) { p_bfx(d, 0u, "sbfx"); }
 static void p_ubfx(AsmDriver* d) { p_bfx(d, 2u, "ubfx"); }
 
 static void p_sxt(AsmDriver* d, u32 bits, const char* what) {
   AA64Reg rd = parse_reg(d);
   AA64Reg rn;
   expect_comma(d, what);
   rn = parse_reg(d);
   reject_sp_reg(d, rd, what);
   reject_sp_reg(d, rn, what);
   if (rn.is64)
     asm_driver_panic(d, "asm: %.*s: source must be a W register",
                      SLICE_ARG(slice_from_cstr(what)));
   if (bits == 32u && !rd.is64)
     asm_driver_panic(d, "asm: sxtw: destination must be an X register");
   emit32(d, aa64_bitfield(rd.is64, 0u, 0u, bits - 1u, rd.num, rn.num));
 }
 
 static void p_uxt(AsmDriver* d, u32 bits, const char* what) {
   AA64Reg rd = parse_reg(d);
   AA64Reg rn;
   u32 sf;
   expect_comma(d, what);
   rn = parse_reg(d);
   reject_sp_reg(d, rd, what);
   reject_sp_reg(d, rn, what);
   if (rn.is64)
     asm_driver_panic(d, "asm: %.*s: source must be a W register",
                      SLICE_ARG(slice_from_cstr(what)));
   if (bits == 32u && !rd.is64)
     asm_driver_panic(d, "asm: uxtw: destination must be an X register");
   sf = bits == 32u ? 1u : 0u;
   emit32(d, aa64_bitfield(sf, 2u, 0u, bits - 1u, rd.num, rn.num));
 }
 
 static void p_sxtb(AsmDriver* d) { p_sxt(d, 8u, "sxtb"); }
 static void p_sxth(AsmDriver* d) { p_sxt(d, 16u, "sxth"); }
 static void p_sxtw(AsmDriver* d) { p_sxt(d, 32u, "sxtw"); }
 static void p_uxtb(AsmDriver* d) { p_uxt(d, 8u, "uxtb"); }
 static void p_uxth(AsmDriver* d) { p_uxt(d, 16u, "uxth"); }
 static void p_uxtw(AsmDriver* d) { p_uxt(d, 32u, "uxtw"); }
 
 /* fmov: Vd,Vn (FP reg move) | Rd,Vn (fp->gpr) | Vd,Rn (gpr->fp). */
 static void p_fmov(AsmDriver* d) {
   FpOrGpr a = parse_fp_or_gpr(d);
   FpOrGpr b;
   expect_comma(d, "fmov");
   b = parse_fp_or_gpr(d);
   if (a.is_fp && b.is_fp) {
     if (a.ftype != b.ftype)
       asm_driver_panic(d, "asm: fmov: operand type mismatch");
     emit32(d, aa64_fp_dp1(a.ftype, AA64_FP_DP1_FMOV, a.num, b.num));
   } else if (!a.is_fp && b.is_fp) {
     emit32(d, aa64_fp_int_cvt((u32)a.is64, b.ftype, AA64_FP_ICVT_FMOV_TO_GPR,
                               a.num, b.num));
   } else if (a.is_fp && !b.is_fp) {
     emit32(d, aa64_fp_int_cvt((u32)b.is64, a.ftype, AA64_FP_ICVT_FMOV_TO_FP,
                               a.num, b.num));
   } else {
     asm_driver_panic(d, "asm: fmov: gpr,gpr form not supported (use mov)");
   }
 }
 
 /* ---- atomics / exclusive wrappers ----
  *
  * Access log2-sizes: byte=0, half=1, word=2, dword=3.  The w/x variants
  * share a mnemonic stem (e.g. `ldxr`) and pick the size from the operand
  * register width — the encoders key on the explicit size, so a width-
  * sensing wrapper peeks the operand register before dispatching. */
 #define AA64_ATOMIC_SIZE_B 0u
 #define AA64_ATOMIC_SIZE_H 1u
 #define AA64_ATOMIC_SIZE_W 2u
 #define AA64_ATOMIC_SIZE_X 3u
 
 /* Load-exclusive family: o2,o0 select ldxr/ldaxr/ldar. */
 #define DEF_LDEX(fn, sz, o2, o0, name) \
   static void fn(AsmDriver* d) { p_ldex(d, sz, o2, o0, name); }
 /* ldxr / ldxrb / ldxrh: o2=0 o0=0.  The non-b/h stem derives size from
  * the register width, so we route it through a width-sensing wrapper. */
 static void p_ldxr_wx(AsmDriver* d) {
   /* Peek the destination register to choose word vs dword size. */
   AsmTok t = asm_driver_peek(d);
   AA64Reg r;
   memset(&r, 0, sizeof r);
   if (t.kind != ASM_TOK_IDENT || !parse_reg_from_ident(d, t.v.ident, &r))
     asm_driver_panic(d, "asm: ldxr: expected register");
   p_ldex(d, r.is64 ? AA64_ATOMIC_SIZE_X : AA64_ATOMIC_SIZE_W, 0u, 0u, "ldxr");
 }
 DEF_LDEX(p_ldxrb, AA64_ATOMIC_SIZE_B, 0u, 0u, "ldxrb")
 DEF_LDEX(p_ldxrh, AA64_ATOMIC_SIZE_H, 0u, 0u, "ldxrh")
 static void p_ldaxr_wx(AsmDriver* d) {
   AsmTok t = asm_driver_peek(d);
   AA64Reg r;
   memset(&r, 0, sizeof r);
   if (t.kind != ASM_TOK_IDENT || !parse_reg_from_ident(d, t.v.ident, &r))
     asm_driver_panic(d, "asm: ldaxr: expected register");
   p_ldex(d, r.is64 ? AA64_ATOMIC_SIZE_X : AA64_ATOMIC_SIZE_W, 0u, 1u, "ldaxr");
 }
 DEF_LDEX(p_ldaxrb, AA64_ATOMIC_SIZE_B, 0u, 1u, "ldaxrb")
 DEF_LDEX(p_ldaxrh, AA64_ATOMIC_SIZE_H, 0u, 1u, "ldaxrh")
 static void p_ldar_wx(AsmDriver* d) {
   AsmTok t = asm_driver_peek(d);
   AA64Reg r;
   memset(&r, 0, sizeof r);
   if (t.kind != ASM_TOK_IDENT || !parse_reg_from_ident(d, t.v.ident, &r))
     asm_driver_panic(d, "asm: ldar: expected register");
   p_ldex(d, r.is64 ? AA64_ATOMIC_SIZE_X : AA64_ATOMIC_SIZE_W, 1u, 1u, "ldar");
 }
 DEF_LDEX(p_ldarb, AA64_ATOMIC_SIZE_B, 1u, 1u, "ldarb")
 DEF_LDEX(p_ldarh, AA64_ATOMIC_SIZE_H, 1u, 1u, "ldarh")
 
 /* stlr (no status): width-driven for the non-b/h stem. */
 static void p_stlr_wx(AsmDriver* d) {
   AsmTok t = asm_driver_peek(d);
   AA64Reg r;
   memset(&r, 0, sizeof r);
   if (t.kind != ASM_TOK_IDENT || !parse_reg_from_ident(d, t.v.ident, &r))
     asm_driver_panic(d, "asm: stlr: expected register");
   p_stlr(d, r.is64 ? AA64_ATOMIC_SIZE_X : AA64_ATOMIC_SIZE_W, "stlr");
 }
 static void p_stlrb_(AsmDriver* d) { p_stlr(d, AA64_ATOMIC_SIZE_B, "stlrb"); }
 static void p_stlrh_(AsmDriver* d) { p_stlr(d, AA64_ATOMIC_SIZE_H, "stlrh"); }
 
 /* Store-exclusive family: o0 selects stxr vs stlxr.  Status reg is always
  * 32-bit; the stored value reg drives the size for the non-b/h stem. */
 static void p_stxr_wx(AsmDriver* d) {
   AA64Reg rs = parse_reg(d);
   reject_sp_reg(d, rs, "stxr");
   if (rs.is64) asm_driver_panic(d, "asm: stxr: status reg must be 32-bit");
   expect_comma(d, "stxr");
   AsmTok t = asm_driver_peek(d);
   AA64Reg rt;
   memset(&rt, 0, sizeof rt);
   if (t.kind != ASM_TOK_IDENT || !parse_reg_from_ident(d, t.v.ident, &rt))
     asm_driver_panic(d, "asm: stxr: expected value register");
   u32 size = rt.is64 ? AA64_ATOMIC_SIZE_X : AA64_ATOMIC_SIZE_W;
   rt = parse_reg(d);
   require_gpr_width(d, rt, size, "stxr");
   expect_comma(d, "stxr");
   AA64Mem m = parse_mem_bare(d, "stxr");
   reject_stex_alias(d, rs, rt, m, "stxr");
   emit32(d, aa64_ldstex_pack((AA64LdStEx){.size = size,
                                           .o2 = 0u,
                                           .L = 0u,
                                           .o1 = 0u,
                                           .Rs = rs.num,
                                           .o0 = 0u,
                                           .Rt2 = AA64_ZR,
                                           .Rn = m.base.num,
                                           .Rt = rt.num}));
 }
 static void p_stlxr_wx(AsmDriver* d) {
   AA64Reg rs = parse_reg(d);
   reject_sp_reg(d, rs, "stlxr");
   if (rs.is64) asm_driver_panic(d, "asm: stlxr: status reg must be 32-bit");
   expect_comma(d, "stlxr");
   AsmTok t = asm_driver_peek(d);
   AA64Reg rt;
   memset(&rt, 0, sizeof rt);
   if (t.kind != ASM_TOK_IDENT || !parse_reg_from_ident(d, t.v.ident, &rt))
     asm_driver_panic(d, "asm: stlxr: expected value register");
   u32 size = rt.is64 ? AA64_ATOMIC_SIZE_X : AA64_ATOMIC_SIZE_W;
   rt = parse_reg(d);
   require_gpr_width(d, rt, size, "stlxr");
   expect_comma(d, "stlxr");
   AA64Mem m = parse_mem_bare(d, "stlxr");
   reject_stex_alias(d, rs, rt, m, "stlxr");
   emit32(d, aa64_ldstex_pack((AA64LdStEx){.size = size,
                                           .o2 = 0u,
                                           .L = 0u,
                                           .o1 = 0u,
                                           .Rs = rs.num,
                                           .o0 = 1u,
                                           .Rt2 = AA64_ZR,
                                           .Rn = m.base.num,
                                           .Rt = rt.num}));
 }
 static void p_stxrb_(AsmDriver* d) {
   p_stex(d, AA64_ATOMIC_SIZE_B, 0u, "stxrb");
 }
 static void p_stxrh_(AsmDriver* d) {
   p_stex(d, AA64_ATOMIC_SIZE_H, 0u, "stxrh");
 }
 static void p_stlxrb_(AsmDriver* d) {
   p_stex(d, AA64_ATOMIC_SIZE_B, 1u, "stlxrb");
 }
 static void p_stlxrh_(AsmDriver* d) {
   p_stex(d, AA64_ATOMIC_SIZE_H, 1u, "stlxrh");
 }
 
 /* CAS family: width-driven for the non-b/h stems (Rs/Rt are same width). */
 #define DEF_CAS(fn, L, o0, name)                                             \
   static void fn##_wx(AsmDriver* d) {                                        \
     AsmTok t = asm_driver_peek(d);                                           \
     AA64Reg r;                                                               \
     memset(&r, 0, sizeof r);                                                 \
     if (t.kind != ASM_TOK_IDENT || !parse_reg_from_ident(d, t.v.ident, &r))  \
       asm_driver_panic(d, "asm: " name ": expected register");               \
     p_cas(d, r.is64 ? AA64_ATOMIC_SIZE_X : AA64_ATOMIC_SIZE_W, L, o0, name); \
   }                                                                          \
   static void fn##b(AsmDriver* d) {                                          \
     p_cas(d, AA64_ATOMIC_SIZE_B, L, o0, name "b");                           \
   }                                                                          \
   static void fn##h(AsmDriver* d) {                                          \
     p_cas(d, AA64_ATOMIC_SIZE_H, L, o0, name "h");                           \
   }
 DEF_CAS(p_cas, 0u, 0u, "cas")
 DEF_CAS(p_casa, 1u, 0u, "casa")
 DEF_CAS(p_casl, 0u, 1u, "casl")
 DEF_CAS(p_casal, 1u, 1u, "casal")
 
 /* LSE atomic family: A/R from the suffix, o3/opc from the stem.  Each
  * mnemonic generates a width-driven stem plus b/h wrappers. */
 #define DEF_LSE(fn, A, R, o3, opc, name)                                      \
   static void fn##_wx(AsmDriver* d) {                                         \
     AsmTok t = asm_driver_peek(d);                                            \
     AA64Reg r;                                                                \
     memset(&r, 0, sizeof r);                                                  \
     if (t.kind != ASM_TOK_IDENT || !parse_reg_from_ident(d, t.v.ident, &r))   \
       asm_driver_panic(d, "asm: " name ": expected register");                \
     p_lse(d, r.is64 ? AA64_ATOMIC_SIZE_X : AA64_ATOMIC_SIZE_W, A, R, o3, opc, \
           name);                                                              \
   }                                                                           \
   static void fn##b(AsmDriver* d) {                                           \
     p_lse(d, AA64_ATOMIC_SIZE_B, A, R, o3, opc, name "b");                    \
   }                                                                           \
   static void fn##h(AsmDriver* d) {                                           \
     p_lse(d, AA64_ATOMIC_SIZE_H, A, R, o3, opc, name "h");                    \
   }
 /* SWP (o3=1, opc=000). */
 DEF_LSE(p_swp, 0u, 0u, 1u, AA64_LSE_OPC_SWP, "swp")
 DEF_LSE(p_swpa, 1u, 0u, 1u, AA64_LSE_OPC_SWP, "swpa")
 DEF_LSE(p_swpl, 0u, 1u, 1u, AA64_LSE_OPC_SWP, "swpl")
 DEF_LSE(p_swpal, 1u, 1u, 1u, AA64_LSE_OPC_SWP, "swpal")
 /* LDADD. */
 DEF_LSE(p_ldadd, 0u, 0u, 0u, AA64_LSE_OPC_LDADD, "ldadd")
 DEF_LSE(p_ldadda, 1u, 0u, 0u, AA64_LSE_OPC_LDADD, "ldadda")
 DEF_LSE(p_ldaddl, 0u, 1u, 0u, AA64_LSE_OPC_LDADD, "ldaddl")
 DEF_LSE(p_ldaddal, 1u, 1u, 0u, AA64_LSE_OPC_LDADD, "ldaddal")
 /* LDCLR. */
 DEF_LSE(p_ldclr, 0u, 0u, 0u, AA64_LSE_OPC_LDCLR, "ldclr")
 DEF_LSE(p_ldclra, 1u, 0u, 0u, AA64_LSE_OPC_LDCLR, "ldclra")
 DEF_LSE(p_ldclrl, 0u, 1u, 0u, AA64_LSE_OPC_LDCLR, "ldclrl")
 DEF_LSE(p_ldclral, 1u, 1u, 0u, AA64_LSE_OPC_LDCLR, "ldclral")
 /* LDEOR. */
 DEF_LSE(p_ldeor, 0u, 0u, 0u, AA64_LSE_OPC_LDEOR, "ldeor")
 DEF_LSE(p_ldeora, 1u, 0u, 0u, AA64_LSE_OPC_LDEOR, "ldeora")
 DEF_LSE(p_ldeorl, 0u, 1u, 0u, AA64_LSE_OPC_LDEOR, "ldeorl")
 DEF_LSE(p_ldeoral, 1u, 1u, 0u, AA64_LSE_OPC_LDEOR, "ldeoral")
 /* LDSET. */
 DEF_LSE(p_ldset, 0u, 0u, 0u, AA64_LSE_OPC_LDSET, "ldset")
 DEF_LSE(p_ldseta, 1u, 0u, 0u, AA64_LSE_OPC_LDSET, "ldseta")
 DEF_LSE(p_ldsetl, 0u, 1u, 0u, AA64_LSE_OPC_LDSET, "ldsetl")
 DEF_LSE(p_ldsetal, 1u, 1u, 0u, AA64_LSE_OPC_LDSET, "ldsetal")
 
 static const AA64Mn kTable[] = {
     {"fadd", p_fadd, 0},
     {"fsub", p_fsub, 0},
     {"fmul", p_fmul, 0},
     {"fdiv", p_fdiv, 0},
     {"fmax", p_fmax, 0},
     {"fmin", p_fmin, 0},
     {"fnmul", p_fnmul, 0},
     {"fneg", p_fneg, 0},
     {"fabs", p_fabs, 0},
     {"fsqrt", p_fsqrt, 0},
     {"fmov", p_fmov, 0},
     {"fcmp", p_fcmp, 0},
     {"fcvt", p_fcvt, 0},
     {"scvtf", p_scvtf, 0},
     {"ucvtf", p_ucvtf, 0},
     {"fcvtzs", p_fcvtzs, 0},
     {"fcvtzu", p_fcvtzu, 0},
     {"clz", p_clz, 0},
     {"rbit", p_rbit, 0},
     {"rev", p_rev, 0},
     {"rev16", p_rev16, 0},
     {"sbfm", p_sbfm, 0},
     {"ubfm", p_ubfm, 0},
     {"bfm", p_bfm, 0},
     {"sbfx", p_sbfx, 0},
     {"ubfx", p_ubfx, 0},
     {"sxtb", p_sxtb, 0},
     {"sxth", p_sxth, 0},
     {"sxtw", p_sxtw, 0},
     {"uxtb", p_uxtb, 0},
     {"uxth", p_uxth, 0},
     {"uxtw", p_uxtw, 0},
     {"nop", p_nop, 0},
     {"dmb", p_dmb, 0},
     {"dsb", p_dsb, 0},
     {"isb", p_isb, 0},
     {"clrex", p_clrex, 0},
     {"ret", p_ret, 0},
     {"br", p_br, 0},
     {"blr", p_blr, 0},
     {"mov", p_mov, 0},
     {"mvn", p_mvn, 0},
     {"movz", p_movz_, 0},
     {"movn", p_movn_, 0},
     {"movk", p_movk_, 0},
     {"add", p_addsub_add, 0},
     {"adds", p_addsub_adds, 0},
     {"sub", p_addsub_sub, 0},
     {"subs", p_addsub_subs, 0},
     {"cmp", p_cmp_w, 0},
     {"cmn", p_cmn_w, 0},
     {"csel", p_csel_, 0},
     {"csinc", p_csinc_, 0},
     {"csinv", p_csinv_, 0},
     {"csneg", p_csneg_, 0},
     {"cset", p_cset_, 0},
     {"csetm", p_csetm_, 0},
     {"neg", p_neg_w, 0},
     {"negs", p_negs_w, 0},
     {"and", p_and_w, 0},
     {"bic", p_bic_w, 0},
     {"orr", p_orr_w, 0},
     {"orn", p_orn_w, 0},
     {"eor", p_eor_w, 0},
     {"eon", p_eon_w, 0},
     {"ands", p_ands_w, 0},
     {"bics", p_bics_w, 0},
     {"madd", p_madd, 0},
     {"msub", p_msub, 0},
     {"mul", p_mul_w, 0},
     {"mneg", p_mneg_w, 0},
     {"udiv", p_udiv_w, 0},
     {"sdiv", p_sdiv_w, 0},
     {"lslv", p_lslv_w, 0},
     {"lsrv", p_lsrv_w, 0},
     {"asrv", p_asrv_w, 0},
     {"rorv", p_rorv_w, 0},
     {"lsl", p_lsl_, 0},
     {"lsr", p_lsr_, 0},
     {"asr", p_asr_, 0},
     {"b", p_b_, 0},
     {"bl", p_bl_, 0},
     {"cbz", p_cbz_, 0},
     {"cbnz", p_cbnz_, 0},
     {"svc", p_svc_, 0},
     {"brk", p_brk_, 0},
     {"hlt", p_hlt_, 0},
     {"mrs", p_mrs_, 0},
     {"msr", p_msr_, 0},
     {"ldr", p_ldr_, 0},
     {"str", p_str_, 0},
     {"ldrb", p_ldrb, 0},
     {"strb", p_strb, 0},
     {"ldrh", p_ldrh, 0},
     {"strh", p_strh, 0},
     {"ldrsb", p_ldrsb, 0},
     {"ldrsh", p_ldrsh, 0},
     {"ldrsw", p_ldrsw, 0},
     {"ldur", p_ldur_, 0},
     {"stur", p_stur_, 0},
     {"ldurb", p_ldurb, 0},
     {"sturb", p_sturb, 0},
     {"ldurh", p_ldurh, 0},
     {"sturh", p_sturh, 0},
     {"ldursb", p_ldursb, 0},
     {"ldursh", p_ldursh, 0},
     {"ldursw", p_ldursw, 0},
     {"ldp", p_ldp_, 0},
     {"stp", p_stp_, 0},
     {"adr", p_adr_, 0},
     {"adrp", p_adrp_, 0},
     /* ---- atomics / exclusive ---- */
     {"ldxr", p_ldxr_wx, 0},
     {"ldxrb", p_ldxrb, 0},
     {"ldxrh", p_ldxrh, 0},
     {"ldaxr", p_ldaxr_wx, 0},
     {"ldaxrb", p_ldaxrb, 0},
     {"ldaxrh", p_ldaxrh, 0},
     {"ldar", p_ldar_wx, 0},
     {"ldarb", p_ldarb, 0},
     {"ldarh", p_ldarh, 0},
     {"stxr", p_stxr_wx, 0},
     {"stxrb", p_stxrb_, 0},
     {"stxrh", p_stxrh_, 0},
     {"stlxr", p_stlxr_wx, 0},
     {"stlxrb", p_stlxrb_, 0},
     {"stlxrh", p_stlxrh_, 0},
     {"stlr", p_stlr_wx, 0},
     {"stlrb", p_stlrb_, 0},
     {"stlrh", p_stlrh_, 0},
     {"cas", p_cas_wx, 0},
     {"casb", p_casb, 0},
     {"cash", p_cash, 0},
     {"casa", p_casa_wx, 0},
     {"casab", p_casab, 0},
     {"casah", p_casah, 0},
     {"casl", p_casl_wx, 0},
     {"caslb", p_caslb, 0},
     {"caslh", p_caslh, 0},
     {"casal", p_casal_wx, 0},
     {"casalb", p_casalb, 0},
     {"casalh", p_casalh, 0},
     {"swp", p_swp_wx, 0},
     {"swpb", p_swpb, 0},
     {"swph", p_swph, 0},
     {"swpa", p_swpa_wx, 0},
     {"swpab", p_swpab, 0},
     {"swpah", p_swpah, 0},
     {"swpl", p_swpl_wx, 0},
     {"swplb", p_swplb, 0},
     {"swplh", p_swplh, 0},
     {"swpal", p_swpal_wx, 0},
     {"swpalb", p_swpalb, 0},
     {"swpalh", p_swpalh, 0},
     {"ldadd", p_ldadd_wx, 0},
     {"ldaddb", p_ldaddb, 0},
     {"ldaddh", p_ldaddh, 0},
     {"ldadda", p_ldadda_wx, 0},
     {"ldaddab", p_ldaddab, 0},
     {"ldaddah", p_ldaddah, 0},
     {"ldaddl", p_ldaddl_wx, 0},
     {"ldaddlb", p_ldaddlb, 0},
     {"ldaddlh", p_ldaddlh, 0},
     {"ldaddal", p_ldaddal_wx, 0},
     {"ldaddalb", p_ldaddalb, 0},
     {"ldaddalh", p_ldaddalh, 0},
     {"ldclr", p_ldclr_wx, 0},
     {"ldclrb", p_ldclrb, 0},
     {"ldclrh", p_ldclrh, 0},
     {"ldclra", p_ldclra_wx, 0},
     {"ldclrab", p_ldclrab, 0},
     {"ldclrah", p_ldclrah, 0},
     {"ldclrl", p_ldclrl_wx, 0},
     {"ldclrlb", p_ldclrlb, 0},
     {"ldclrlh", p_ldclrlh, 0},
     {"ldclral", p_ldclral_wx, 0},
     {"ldclralb", p_ldclralb, 0},
     {"ldclralh", p_ldclralh, 0},
     {"ldeor", p_ldeor_wx, 0},
     {"ldeorb", p_ldeorb, 0},
     {"ldeorh", p_ldeorh, 0},
     {"ldeora", p_ldeora_wx, 0},
     {"ldeorab", p_ldeorab, 0},
     {"ldeorah", p_ldeorah, 0},
     {"ldeorl", p_ldeorl_wx, 0},
     {"ldeorlb", p_ldeorlb, 0},
     {"ldeorlh", p_ldeorlh, 0},
     {"ldeoral", p_ldeoral_wx, 0},
     {"ldeoralb", p_ldeoralb, 0},
     {"ldeoralh", p_ldeoralh, 0},
     {"ldset", p_ldset_wx, 0},
     {"ldsetb", p_ldsetb, 0},
     {"ldseth", p_ldseth, 0},
     {"ldseta", p_ldseta_wx, 0},
     {"ldsetab", p_ldsetab, 0},
     {"ldsetah", p_ldsetah, 0},
     {"ldsetl", p_ldsetl_wx, 0},
     {"ldsetlb", p_ldsetlb, 0},
     {"ldsetlh", p_ldsetlh, 0},
     {"ldsetal", p_ldsetal_wx, 0},
     {"ldsetalb", p_ldsetalb, 0},
     {"ldsetalh", p_ldsetalh, 0},
     {"b.eq", p_b_eq, 0},
     {"b.ne", p_b_ne, 0},
     {"b.cs", p_b_cs, 0},
     {"b.hs", p_b_hs, 0},
     {"b.cc", p_b_cc, 0},
     {"b.lo", p_b_lo, 0},
     {"b.mi", p_b_mi, 0},
     {"b.pl", p_b_pl, 0},
     {"b.vs", p_b_vs, 0},
     {"b.vc", p_b_vc, 0},
     {"b.hi", p_b_hi, 0},
     {"b.ls", p_b_ls, 0},
     {"b.ge", p_b_ge, 0},
     {"b.lt", p_b_lt, 0},
     {"b.gt", p_b_gt, 0},
     {"b.le", p_b_le, 0},
     {"b.al", p_b_al, 0},
     {NULL, NULL, 0},
 };
 
 void aa64_asm_insn(AA64Asm* a, AsmDriver* d, Sym mnemonic) {
   (void)a;
   Slice msl = pool_slice(asm_driver_pool(d), mnemonic);
   const char* mp = msl.s;
   size_t mn = msl.len;
   for (const AA64Mn* row = kTable; row->name; ++row) {
     if (icase_eq(mp, mn, row->name)) {
       row->fn(d);
       return;
     }
   }
   asm_driver_panic(d, "asm: unknown mnemonic");
 }
 
 /* ---- inline-asm template walker (Phase 4b Track C) ---- */
 
 /* Per-call rendered-line buffer.  GCC's inline asm rarely emits more
  * than a handful of instructions per block; one line of substituted
  * text fits comfortably inside this. Truncation panics — the operator
  * grammar should never grow a single line beyond this without a
  * deliberate reason. */
 #define AA64_INLINE_LINE_CAP 1024
 
 _Noreturn static void inline_panic(AA64Asm* a, const char* msg);
 
 /* Render a 5-bit register number into the StrBuf using the requested
  * width form.  is64 picks x-form vs w-form; SP / ZR encode as
  * register #31 and we render them as wzr/xzr or wsp/sp depending on
  * caller intent — for inline-asm v1 the bound operand always names a
  * GP register, never SP, so we emit wzr/xzr for #31. */
 static void render_reg(StrBuf* sb, u32 reg, int is64) {
   if (reg == 31u) {
     strbuf_puts(sb, is64 ? "xzr" : "wzr");
     return;
   }
   strbuf_putc(sb, is64 ? 'x' : 'w');
   if (reg >= 10u) strbuf_putc(sb, (char)('0' + (reg / 10u)));
   strbuf_putc(sb, (char)('0' + (reg % 10u)));
 }
 
 static void render_fp_reg(StrBuf* sb, u32 reg, u32 nbytes) {
   strbuf_putc(sb, nbytes <= 4u ? 's' : 'd');
   if (reg >= 10u) strbuf_putc(sb, (char)('0' + (reg / 10u)));
   strbuf_putc(sb, (char)('0' + (reg % 10u)));
 }
 
 static u32 inline_op_size(AA64Asm* a, const Operand* op) {
   if (!op->type) return 8u;
   u64 n = cg_type_size(a->c, op->type);
   if (!n) return 8u;
   if (n > 16u) inline_panic(a, "inline asm operand is too large");
   return (u32)n;
 }
 
 static int inline_op_is_ptr(AA64Asm* a, const Operand* op) {
   return op->type && cg_type_is_ptr(a->c, op->type);
 }
 
 /* Render a signed 64-bit integer prefixed with '#'. */
 static void render_imm(StrBuf* sb, i64 v) {
   strbuf_putc(sb, '#');
   strbuf_put_i64(sb, v);
 }
 
 /* Render an addressing form `[xN, #ofs]` for OPK_INDIRECT. */
 static void render_indirect(StrBuf* sb, Reg base, i32 ofs) {
   strbuf_putc(sb, '[');
   render_reg(sb, (u32)base, /*is64=*/1);
   if (ofs != 0) {
     strbuf_puts(sb, ", ");
     render_imm(sb, (i64)ofs);
   }
   strbuf_putc(sb, ']');
 }
 
 _Noreturn static void inline_panic(AA64Asm* a, const char* msg) {
   SrcLoc loc = {0, 0, 0};
   compiler_panic(a->c, loc, "inline asm: %.*s",
                  SLICE_ARG(slice_from_cstr(msg)));
 }
 
 /* Resolve operand index N → (kind=0 forced default, 1=force-w, 2=force-x,
  * 3=address form `%aN`).  Renders into sb. */
 static void render_operand(AA64Asm* a, StrBuf* sb, u32 idx, int form) {
   u32 ntot = a->nout + a->nin;
   if (idx >= ntot) inline_panic(a, "operand index out of range");
   const Operand* op =
       (idx < a->nout) ? &a->out_ops[idx] : &a->in_ops[idx - a->nout];
   switch (form) {
     case 1: /* %wN — force 32-bit register form */
       if (op->kind != AA64_INLINE_OPK_REG ||
           op->pad[0] != AA64_INLINE_OPCLS_INT)
         inline_panic(a, "%w on non-integer-register operand");
       render_reg(sb, (u32)op->v.local, 0);
       return;
     case 2: /* %xN — force 64-bit register form */
       if (op->kind != AA64_INLINE_OPK_REG ||
           op->pad[0] != AA64_INLINE_OPCLS_INT)
         inline_panic(a, "%x on non-integer-register operand");
       render_reg(sb, (u32)op->v.local, 1);
       return;
     case 3: /* %aN — memory addressing form */
       if (op->kind != OPK_INDIRECT) inline_panic(a, "%a on non-memory operand");
       /* Inline asm consumes a plain pointer-shaped address; the cg
        * contract guarantees no EA index here. */
       if (op->v.ind.index != REG_NONE)
         inline_panic(a, "%a operand has unexpected EA index");
       render_indirect(sb, op->v.ind.base, op->v.ind.ofs);
       return;
     default:
       break;
   }
   /* Default rendering by operand kind. */
   switch (op->kind) {
     case AA64_INLINE_OPK_REG:
       if (op->pad[0] == AA64_INLINE_OPCLS_FP) {
         render_fp_reg(sb, (u32)op->v.local, inline_op_size(a, op));
       } else {
         render_reg(sb, (u32)op->v.local,
                    inline_op_is_ptr(a, op) || inline_op_size(a, op) > 4u);
       }
       return;
     case OPK_IMM:
       render_imm(sb, op->v.imm);
       return;
     case OPK_INDIRECT:
       if (op->v.ind.index != REG_NONE)
         inline_panic(a, "inline-asm operand has unexpected EA index");
       render_indirect(sb, op->v.ind.base, op->v.ind.ofs);
       return;
     default:
       inline_panic(a, "unsupported operand kind for %N");
   }
 }
 
 /* Lex one line of substituted asm and dispatch via aa64_asm_insn. */
 static void run_one_line(AA64Asm* a, MCEmitter* mc, const char* text,
                          size_t len) {
   /* Skip blank lines. */
   size_t i;
   for (i = 0; i < len; ++i) {
     if (text[i] != ' ' && text[i] != '\t') break;
   }
   if (i == len) return;
 
   AsmLexer* lx = asm_lex_open_mem(a->c, "<inline-asm>", text, len);
   AsmDriver* d = asm_driver_open_inline(a->c, mc, lx);
 
   /* The first non-trivial token must be the mnemonic identifier (or a
    * `.directive`, but inline asm doesn't normally use directives — leave
    * that path unsupported until needed). */
   AsmTok t = asm_driver_peek(d);
   while (t.kind == ASM_TOK_NEWLINE || t.kind == ASM_TOK_HASH) {
     (void)asm_driver_next(d);
     if (t.kind == ASM_TOK_HASH) {
       /* Skip cpp linemarker rest of line. */
       while (!asm_driver_at_eol(d)) (void)asm_driver_next(d);
     }
     t = asm_driver_peek(d);
   }
   if (t.kind == ASM_TOK_EOF) {
     asm_driver_close_inline(d);
     asm_lex_close(lx);
     return;
   }
   if (t.kind != ASM_TOK_IDENT)
     inline_panic(a, "expected mnemonic at start of inline asm line");
   (void)asm_driver_next(d);
   Sym mn = t.v.ident;
   /* Compose `b.eq` etc. — same trick as the standalone driver. */
   AsmTok dot = asm_driver_peek(d);
   if (asm_driver_tok_is_punct(dot, '.')) {
     (void)asm_driver_next(d);
     AsmTok rest = asm_driver_next(d);
     if (rest.kind != ASM_TOK_IDENT)
       inline_panic(a, "composite mnemonic: expected ident after '.'");
     Slice hsl = pool_slice(asm_driver_pool(d), mn);
     Slice rsl = pool_slice(asm_driver_pool(d), rest.v.ident);
     size_t hn = hsl.len, rn = rsl.len;
     const char* hp = hsl.s;
     const char* rp = rsl.s;
     char buf[64];
     if (hn + 1 + rn >= sizeof buf)
       inline_panic(a, "composite mnemonic too long");
     for (size_t k = 0; k < hn; ++k) buf[k] = hp[k];
     buf[hn] = '.';
     for (size_t k = 0; k < rn; ++k) buf[hn + 1 + k] = rp[k];
     mn = pool_intern_slice(asm_driver_pool(d),
                            (Slice){.s = buf, .len = hn + 1 + rn});
   }
   aa64_asm_insn(a, d, mn);
   asm_driver_close_inline(d);
   asm_lex_close(lx);
 }
 
 /* Substitute placeholders into one line's StrBuf, then dispatch.
  *
  * The input range is [start, end) inside `tmpl`.  Updates `*line_idx`
  * is not used — the caller resets the StrBuf between lines. */
 static void render_and_run_line(AA64Asm* a, MCEmitter* mc, StrBuf* sb,
                                 const char* start, const char* end) {
   strbuf_reset(sb);
   for (const char* p = start; p < end; ++p) {
     char c = *p;
     if (c != '%') {
       strbuf_putc(sb, c);
       continue;
     }
     /* Placeholder. */
     if (p + 1 >= end) inline_panic(a, "trailing '%' in template");
     char n = *(p + 1);
     if (n == '%') {
       strbuf_putc(sb, '%');
       ++p;
       continue;
     }
     if (n == '[') {
       /* %[name] — scan to the closing ']' and resolve against
        * AsmConstraint.name on the combined outs+ins list. Match by
        * comparing the named-bracket contents against the interned name
        * Sym stored on each constraint. */
       const char* nbeg = p + 2;
       const char* nend = nbeg;
       while (nend < end && *nend != ']') ++nend;
       if (nend == end) inline_panic(a, "unterminated %[name]");
       size_t nlen = (size_t)(nend - nbeg);
       Sym needle =
           pool_intern_slice(a->c->global, (Slice){.s = nbeg, .len = nlen});
       u32 idx = (u32)-1;
       for (u32 k = 0; k < a->nout; ++k) {
         if (a->outs[k].name == needle) {
           idx = k;
           break;
         }
       }
       if (idx == (u32)-1) {
         for (u32 k = 0; k < a->nin; ++k) {
           if (a->ins[k].name == needle) {
             idx = a->nout + k;
             break;
           }
         }
       }
       if (idx == (u32)-1)
         inline_panic(a, "%[name] does not match any constraint");
       p = nend; /* loop's ++p steps past the ']' */
       render_operand(a, sb, idx, 0);
       continue;
     }
     int form = 0; /* 0=default, 1=w, 2=x, 3=a */
     if (n == 'w' || n == 'x' || n == 'a') {
       form = (n == 'w') ? 1 : (n == 'x') ? 2 : 3;
       ++p;
       if (p + 1 >= end) inline_panic(a, "trailing '%' modifier in template");
       n = *(p + 1);
     }
     if (n == '[') {
       /* %w[name] / %x[name] / %a[name] — width modifier + symbolic
        * operand. Resolves the same way as %[name] but renders with the
        * declared form. */
       const char* nbeg = p + 2;
       const char* nend = nbeg;
       while (nend < end && *nend != ']') ++nend;
       if (nend == end) inline_panic(a, "unterminated %[name]");
       size_t nlen = (size_t)(nend - nbeg);
       Sym needle =
           pool_intern_slice(a->c->global, (Slice){.s = nbeg, .len = nlen});
       u32 idx = (u32)-1;
       for (u32 k = 0; k < a->nout; ++k) {
         if (a->outs[k].name == needle) {
           idx = k;
           break;
         }
       }
       if (idx == (u32)-1) {
         for (u32 k = 0; k < a->nin; ++k) {
           if (a->ins[k].name == needle) {
             idx = a->nout + k;
             break;
           }
         }
       }
       if (idx == (u32)-1)
         inline_panic(a, "%[name] does not match any constraint");
       p = nend; /* loop's ++p steps past the ']' */
       render_operand(a, sb, idx, form);
       continue;
     }
     if (n < '0' || n > '9') inline_panic(a, "expected digit after '%'");
     u32 idx = (u32)(n - '0');
     ++p;
     /* GCC syntax permits up to two digits (%0..%99). */
     if (p + 1 < end && *(p + 1) >= '0' && *(p + 1) <= '9') {
       idx = idx * 10 + (u32)(*(p + 1) - '0');
       ++p;
     }
     render_operand(a, sb, idx, form);
   }
   if (sb->truncated) inline_panic(a, "inline asm line buffer overflow");
   run_one_line(a, mc, strbuf_cstr(sb), strbuf_len(sb));
 }
 
 void aa64_asm_run_template(AA64Asm* a, MCEmitter* mc, const char* tmpl) {
   if (!tmpl || !*tmpl) return;
 
   char buf[AA64_INLINE_LINE_CAP];
   StrBuf sb;
   strbuf_init(&sb, buf, sizeof buf);
 
   /* Walk tmpl, splitting on '\n' and ';' line terminators.  Track bracket
    * depth and quote state so that a literal ';' inside `[ ... ]` or a
    * quoted string is not mistaken for a statement separator. */
   const char* line_start = tmpl;
   int bracket = 0;
   char quote = 0;
   for (const char* p = tmpl;; ++p) {
     char c = *p;
     if (c == '\0') {
       render_and_run_line(a, mc, &sb, line_start, p);
       break;
     }
     if (quote) {
       if (c == '\\' && *(p + 1)) {
         ++p;
         continue;
       }
       if (c == quote) quote = 0;
       continue;
     }
     if (c == '"' || c == '\'') {
       quote = c;
       continue;
     }
     if (c == '[') {
       ++bracket;
       continue;
     }
     if (c == ']') {
       if (bracket) --bracket;
       continue;
     }
     if (bracket == 0 && (c == '\n' || c == ';')) {
       render_and_run_line(a, mc, &sb, line_start, p);
       line_start = p + 1;
     }
   }
 }