kit

asm.c (67902B)

---

 #include "arch/x64/asm.h"
 
 #include <string.h>
 
 #include "arch/x64/emit.h"
 #include "arch/x64/regs.h"
 #include "asm/asm_helpers.h"
 #include "core/arena.h"
 #include "core/pool.h"
 #include "core/slice.h"
 #include "core/strbuf.h"
 
 struct X64Asm {
   ArchAsm base;
   Compiler* c;
 
   const AsmConstraint* outs;
   Operand* out_ops;
   const AsmConstraint* ins;
   const Operand* in_ops;
   const Sym* clobbers;
   u32 nout;
   u32 nin;
   u32 nclob;
 };
 
 typedef enum X64AsmOperandKind {
   X64_ASM_OP_REG,
   X64_ASM_OP_XMM,
   X64_ASM_OP_IMM,
   X64_ASM_OP_MEM,
   X64_ASM_OP_IND_REG,
 } X64AsmOperandKind;
 
 typedef struct X64AsmOperand {
   u8 kind;
   u8 width;
   u8 reg;
   u8 base;
   u8 high8;
   u8 seg;
   u8 no_base;      /* MEM: segment-prefixed absolute, no base register */
   u8 index;        /* MEM SIB: index register (valid when has_index) */
   u8 scale;        /* MEM SIB: log2 of scale ∈ {0,1,2,3} → 1/2/4/8 */
   u8 has_index;    /* MEM: SIB index present */
   u8 rip_relative; /* MEM: bare (%rip)/disp(%rip) form */
   u8 has_reloc;    /* MEM: symbolic disp carries a relocation */
   u8 pad[3];
   i64 imm;
   i32 disp;
   RelocKind reloc_kind; /* MEM: reloc on the disp32 (PC32 / REX_GOTPCRELX) */
   ObjSymId reloc_sym;   /* MEM: relocated symbol */
   i64 reloc_off;        /* MEM: user addend on the symbol */
 } X64AsmOperand;
 
 static int x64_reg_from_name(AsmDriver* d, Sym s, u32* reg_out, u32* width_out,
                              u32* high8_out) {
   Slice sl = pool_slice(asm_driver_pool(d), s);
   char buf[16];
   u32 reg;
   u32 width = 8;
   Slice q;
   if (!sl.s || sl.len < 2 || sl.len >= sizeof buf) return 0;
   memcpy(buf, sl.s, sl.len);
   buf[sl.len] = '\0';
   q = slice_from_cstr(buf);
   if (slice_eq_cstr(q, "ah") || slice_eq_cstr(q, "ch") ||
       slice_eq_cstr(q, "dh") || slice_eq_cstr(q, "bh")) {
     static const u32 high_map[4] = {4u, 5u, 6u, 7u};
     const char* names = "acdb";
     for (u32 i = 0; i < 4u; ++i) {
       if (buf[0] == names[i]) {
         if (reg_out) *reg_out = high_map[i];
         if (width_out) *width_out = 1;
         if (high8_out) *high8_out = 1;
         return 1;
       }
     }
   }
   if (x64_register_hw_index(buf, &reg) != 0) return 0;
   if (reg > 15u) return 0;
   if (slice_eq_cstr(q, "al") || slice_eq_cstr(q, "cl") ||
       slice_eq_cstr(q, "dl") || slice_eq_cstr(q, "bl") ||
       slice_eq_cstr(q, "spl") || slice_eq_cstr(q, "bpl") ||
       slice_eq_cstr(q, "sil") || slice_eq_cstr(q, "dil") ||
       buf[sl.len - 1] == 'b') {
     width = 1;
   } else if (slice_eq_cstr(q, "ax") || slice_eq_cstr(q, "cx") ||
              slice_eq_cstr(q, "dx") || slice_eq_cstr(q, "bx") ||
              slice_eq_cstr(q, "sp") || slice_eq_cstr(q, "bp") ||
              slice_eq_cstr(q, "si") || slice_eq_cstr(q, "di") ||
              buf[sl.len - 1] == 'w') {
     width = 2;
   } else if (buf[sl.len - 1] == 'd' || buf[0] == 'e') {
     width = 4;
   }
   if (reg_out) *reg_out = reg;
   if (width_out) *width_out = width;
   if (high8_out) *high8_out = 0;
   return 1;
 }
 
 static int x64_xmm_from_name(AsmDriver* d, Sym s, u32* reg_out) {
   Slice sl = pool_slice(asm_driver_pool(d), s);
   const char* p = sl.s;
   size_t n = sl.len;
   u32 reg = 0;
   if (!p || n < 4 || n > 5) return 0;
   if (p[0] != 'x' || p[1] != 'm' || p[2] != 'm') return 0;
   for (size_t i = 3; i < n; ++i) {
     if (p[i] < '0' || p[i] > '9') return 0;
     reg = reg * 10u + (u32)(p[i] - '0');
   }
   if (reg > 15u) return 0;
   if (reg_out) *reg_out = reg;
   return 1;
 }
 
 static int x64_segment_prefix_from_name(AsmDriver* d, Sym s, u8* prefix_out) {
   Slice sl = pool_slice(asm_driver_pool(d), s);
   const char* p = sl.s;
   size_t n = sl.len;
   if (!p || n != 2) return 0;
   if (p[0] == 'f' && p[1] == 's') {
     if (prefix_out) *prefix_out = 0x64;
     return 1;
   }
   if (p[0] == 'g' && p[1] == 's') {
     if (prefix_out) *prefix_out = 0x65;
     return 1;
   }
   return 0;
 }
 
 static void expect_comma(AsmDriver* d);
 
 static u32 parse_reg(AsmDriver* d, u32* width_out, u32* high8_out) {
   AsmTok t;
   u32 reg;
   if (!asm_driver_eat_punct(d, '%'))
     asm_driver_panic(d, "x64 asm: expected register");
   t = asm_driver_next(d);
   if (t.kind != ASM_TOK_IDENT ||
       !x64_reg_from_name(d, t.v.ident, &reg, width_out, high8_out)) {
     asm_driver_panic(d, "x64 asm: bad register");
   }
   return reg;
 }
 
 /* True if the symbol names the instruction pointer ("rip"). */
 static int x64_ident_is_rip(AsmDriver* d, Sym s) {
   Slice sl = pool_slice(asm_driver_pool(d), s);
   return sl.s && sl.len == 3 && sl.s[0] == 'r' && sl.s[1] == 'i' &&
          sl.s[2] == 'p';
 }
 
 /* Convert a SIB scale literal (1/2/4/8) to its log2 (0/1/2/3). */
 static u32 x64_scale_to_log2(AsmDriver* d, i64 scale) {
   switch (scale) {
     case 1:
       return 0u;
     case 2:
       return 1u;
     case 4:
       return 2u;
     case 8:
       return 3u;
     default:
       asm_driver_panic(d, "x64 asm: memory scale must be 1, 2, 4, or 8");
   }
 }
 
 /* Parse the body of a memory operand once the leading '(' has been
  * consumed: '%base[,%index,scale])', '%rip)', or ',%index,scale)'.
  * Fills base/index/scale/has_index/rip_relative on `op` and eats the
  * closing ')'. */
 static void parse_mem_paren_body(AsmDriver* d, X64AsmOperand* op) {
   AsmTok t = asm_driver_peek(d);
   if (asm_driver_tok_is_punct(t, '%')) {
     /* Peek the register name to detect the RIP-relative form. */
     AsmTok ident;
     (void)asm_driver_next(d);
     ident = asm_driver_next(d);
     if (ident.kind != ASM_TOK_IDENT)
       asm_driver_panic(d, "x64 asm: bad register");
     if (x64_ident_is_rip(d, ident.v.ident)) {
       op->rip_relative = 1;
       asm_driver_expect_punct(d, ')', "')' in x64 memory operand");
       return;
     }
     {
       u32 reg = 0;
       if (!x64_reg_from_name(d, ident.v.ident, &reg, NULL, NULL))
         asm_driver_panic(d, "x64 asm: bad register");
       op->base = (u8)reg;
     }
     /* Optional ',%index,scale'. */
     if (asm_driver_eat_comma(d)) {
       op->index = (u8)parse_reg(d, NULL, NULL);
       op->has_index = 1;
       expect_comma(d);
       op->scale = (u8)x64_scale_to_log2(d, asm_driver_parse_const(d));
     }
     asm_driver_expect_punct(d, ')', "')' in x64 memory operand");
     return;
   }
   /* Index-only form: '(,%index,scale)' — base omitted. */
   if (asm_driver_eat_comma(d)) {
     op->no_base = 1;
     op->index = (u8)parse_reg(d, NULL, NULL);
     op->has_index = 1;
     expect_comma(d);
     op->scale = (u8)x64_scale_to_log2(d, asm_driver_parse_const(d));
     asm_driver_expect_punct(d, ')', "')' in x64 memory operand");
     return;
   }
   asm_driver_panic(d, "x64 asm: expected register in memory operand");
 }
 
 /* Consume an optional `@MOD` relocation suffix after a symbol and return the
  * RelocKind it selects, or `dflt` when no suffix is present. */
 static RelocKind x64_parse_reloc_suffix(AsmDriver* d, RelocKind dflt) {
   if (!asm_driver_tok_is_punct(asm_driver_peek(d), '@')) return dflt;
   (void)asm_driver_next(d); /* '@' */
   AsmTok n = asm_driver_next(d);
   if (n.kind != ASM_TOK_IDENT)
     asm_driver_panic(d, "x64 asm: expected relocation name after '@'");
   Slice s = pool_slice(asm_driver_pool(d), n.v.ident);
   if (slice_eq_cstr(s, "PLT")) return R_X64_PLT32;
   if (slice_eq_cstr(s, "GOTPCREL")) return R_X64_REX_GOTPCRELX;
   if (slice_eq_cstr(s, "GOTPCRELX")) return R_X64_GOTPCRELX;
   asm_driver_panic(d, "x64 asm: unsupported relocation suffix");
 }
 
 static X64AsmOperand parse_operand(AsmDriver* d) {
   X64AsmOperand op;
   AsmTok t;
   memset(&op, 0, sizeof op);
   t = asm_driver_peek(d);
   if (asm_driver_eat_punct(d, '*')) {
     op.kind = X64_ASM_OP_IND_REG;
     op.reg = (u8)parse_reg(d, NULL, NULL);
     return op;
   }
   if (asm_driver_eat_punct(d, '$')) {
     op.kind = X64_ASM_OP_IMM;
     op.imm = asm_driver_parse_const(d);
     return op;
   }
   if (asm_driver_tok_is_punct(t, '%')) {
     u32 width = 8;
     u32 high8 = 0;
     AsmTok ident;
     (void)asm_driver_next(d);
     ident = asm_driver_next(d);
     if (ident.kind != ASM_TOK_IDENT)
       asm_driver_panic(d, "x64 asm: bad register");
     if (x64_segment_prefix_from_name(d, ident.v.ident, &op.seg)) {
       asm_driver_expect_punct(d, ':', "':' after x64 segment register");
       op.kind = X64_ASM_OP_MEM;
       if (!asm_driver_tok_is_punct(asm_driver_peek(d), '('))
         op.disp = (i32)asm_driver_parse_const(d);
       if (asm_driver_eat_punct(d, '(')) {
         op.base = (u8)parse_reg(d, NULL, NULL);
         asm_driver_expect_punct(d, ')', "')' in x64 memory operand");
       } else {
         op.no_base = 1;
       }
       return op;
     }
     if (x64_xmm_from_name(d, ident.v.ident, &width)) {
       op.kind = X64_ASM_OP_XMM;
       op.reg = (u8)width;
       op.width = 16;
       return op;
     }
     {
       u32 reg = 0;
       if (!x64_reg_from_name(d, ident.v.ident, &reg, &width, &high8))
         asm_driver_panic(d, "x64 asm: bad register");
       op.kind = X64_ASM_OP_REG;
       op.reg = (u8)reg;
     }
     op.width = (u8)width;
     op.high8 = (u8)high8;
     return op;
   }
   op.kind = X64_ASM_OP_MEM;
   op.disp = 0;
   if (!asm_driver_tok_is_punct(t, '(')) {
     /* A symbolic displacement (`sym(%rip)`, `sym@GOTPCREL(%rip)`) becomes a
      * relocation; a numeric displacement stays literal. */
     if (asm_driver_peek(d).kind == ASM_TOK_IDENT) {
       asm_driver_parse_sym_expr(d, &op.reloc_sym, &op.reloc_off);
       op.reloc_kind = x64_parse_reloc_suffix(d, R_PC32);
       op.has_reloc = 1;
     } else {
       op.disp = (i32)asm_driver_parse_const(d);
     }
   }
   asm_driver_expect_punct(d, '(', "'(' in x64 memory operand");
   parse_mem_paren_body(d, &op);
   if (op.has_reloc && !op.rip_relative)
     asm_driver_panic(d,
                      "x64 asm: symbolic memory displacement requires (%rip)");
   return op;
 }
 
 /* Emit the relocation a symbolic `(%rip)` memory operand carries, if any. The
  * disp32 field is the last 4 bytes of the instruction except for an immediate
  * store, where `trailing` immediate bytes follow it. R_X86_64_PC32-style
  * relocs use addend (off - 4 - trailing) so S+A-P yields the rip-relative
  * displacement to the end of the instruction. */
 static void x64_emit_mem_reloc(AsmDriver* d, MCEmitter* mc,
                                const X64AsmOperand* m, u32 trailing) {
   if (!m->has_reloc) return;
   u32 disp_pos = mc->pos(mc) - 4u - trailing;
   mc->emit_reloc_at(mc, asm_driver_cur_section(d), disp_pos, m->reloc_kind,
                     m->reloc_sym, m->reloc_off - 4 - (i64)trailing, 1, 0);
 }
 
 static u32 x64_pack_rex_mem_operand(u8* out, int w, u32 reg,
                                     X64AsmOperand mem) {
   /* RIP-relative carries no base/index registers (rm=101, no SIB). */
   if (mem.rip_relative) return x64_pack_rex(out, w, reg, 0, 0u);
   /* SIB forms supply REX.X from the index register (and REX.B from base
    * unless the base is omitted in the index-only form). */
   if (mem.has_index)
     return x64_pack_rex(out, w, reg, mem.index, mem.no_base ? 0u : mem.base);
   return x64_pack_rex(out, w, reg, 0, mem.no_base ? 0u : mem.base);
 }
 
 static u32 x64_pack_mem_operand(u8* out, u32 reg, X64AsmOperand mem) {
   if (mem.rip_relative) return x64_pack_mem_rip(out, reg, mem.disp);
   if (mem.has_index) {
     /* Index-only form (no base): mod=00 with SIB.base=101 → disp32. */
     if (mem.no_base) {
       out[0] = x64_modrm(0u, reg, X64_MODRM_RM_SIB);
       out[1] = x64_sib(mem.scale, mem.index, X64_SIB_NO_BASE);
       return 2u + x64_put_u32le(out + 2, (u32)mem.disp);
     }
     return x64_pack_mem_sib(out, reg, mem.base, mem.index, mem.scale, mem.disp);
   }
   if (mem.no_base) {
     out[0] = x64_modrm(0u, reg, X64_MODRM_RM_SIB);
     out[1] = x64_sib(0u, X64_SIB_NO_INDEX, X64_SIB_NO_BASE);
     return 2u + x64_put_u32le(out + 2, (u32)mem.disp);
   }
   return x64_pack_mem(out, reg, mem.base, mem.disp);
 }
 
 /* reg ← mem with an explicit single-byte opcode (e.g. 0x8B MOV, 0x8D LEA).
  * Routes the full memory-operand variety (plain / SIB / RIP / segment)
  * through the shared pack helpers. */
 static void emit_reg_mem_operand(AsmDriver* d, MCEmitter* mc, u32 size, u8 opc,
                                  u32 dst, X64AsmOperand src) {
   u8 buf[16];
   u32 n = 0;
   if (size == 2u) buf[n++] = X64_OPSIZE_PFX;
   if (src.seg) buf[n++] = src.seg;
   n += x64_pack_rex_mem_operand(buf + n, size == 8u, dst, src);
   buf[n++] = opc;
   n += x64_pack_mem_operand(buf + n, dst, src);
   mc->emit_bytes(mc, buf, n);
   x64_emit_mem_reloc(d, mc, &src, 0);
 }
 
 static void emit_mov_load_operand(AsmDriver* d, MCEmitter* mc, u32 size,
                                   u32 dst, X64AsmOperand src) {
   emit_reg_mem_operand(d, mc, size, X64_OPC_MOV_R_RM, dst, src);
 }
 
 /* reg → mem store with an explicit reg-to-r/m opcode.  Used by MOV
  * (0x89/0x88) and the ALU /r stores (ADD 0x01, OR 0x09, AND 0x21,
  * SUB 0x29, XOR 0x31, CMP 0x39).  The register operand occupies the
  * ModR/M reg field; the memory operand the r/m field. */
 static void emit_reg_store_operand(AsmDriver* d, MCEmitter* mc, u32 size,
                                    u8 opc, u32 src, X64AsmOperand dst,
                                    int force_rex) {
   u8 buf[16];
   u32 n = 0;
   if (size == 2u) buf[n++] = X64_OPSIZE_PFX;
   if (dst.seg) buf[n++] = dst.seg;
   if (force_rex)
     n += x64_pack_rex_force(buf + n, size == 8u, src, 0,
                             dst.no_base ? 0u : dst.base);
   else
     n += x64_pack_rex_mem_operand(buf + n, size == 8u, src, dst);
   buf[n++] = opc;
   n += x64_pack_mem_operand(buf + n, src, dst);
   mc->emit_bytes(mc, buf, n);
   x64_emit_mem_reloc(d, mc, &dst, 0);
 }
 
 static void emit_mov_store_operand(AsmDriver* d, MCEmitter* mc, u32 size,
                                    u32 src, X64AsmOperand dst, int force_rex) {
   emit_reg_store_operand(d, mc, size,
                          size == 1u ? X64_OPC_MOV_RM_R8 : X64_OPC_MOV_RM_R, src,
                          dst, force_rex);
 }
 
 /* imm → mem store via a group-1 /digit opcode (group-1 ALU 0x80/0x81/0x83,
  * or MOV C6/C7).  `opc8`/`opc32` select the 8-bit-immediate vs
  * 32-bit-immediate (sign-extended) opcode; pass equal values when the
  * encoding has no imm8 short form (e.g. MOV).  `imm8` forces the short
  * form when the immediate fits. */
 static void emit_rm_imm_store_operand(AsmDriver* d, MCEmitter* mc, u32 size,
                                       u8 opc8, u8 opc32, u32 sub,
                                       X64AsmOperand dst, i64 imm,
                                       int allow_i8) {
   u8 buf[16];
   u32 n = 0;
   int use_i8 = allow_i8 && imm_fits_i8(imm);
   if (!use_i8 && !imm_fits_i32(imm) && size != 1u)
     asm_driver_panic(d, "x64 asm: immediate out of range");
   if (size == 2u) buf[n++] = X64_OPSIZE_PFX;
   if (dst.seg) buf[n++] = dst.seg;
   n += x64_pack_rex_mem_operand(buf + n, size == 8u, 0, dst);
   buf[n++] = use_i8 ? opc8 : opc32;
   n += x64_pack_mem_operand(buf + n, sub, dst);
   u32 trailing;
   if (size == 1u) {
     buf[n++] = (u8)imm;
     trailing = 1u;
   } else if (use_i8) {
     buf[n++] = (u8)(i8)imm;
     trailing = 1u;
   } else if (size == 2u) {
     /* 16-bit operand size: a 2-byte immediate (under the 0x66 prefix). */
     u16 v = (u16)imm;
     buf[n++] = (u8)v;
     buf[n++] = (u8)(v >> 8);
     trailing = 2u;
   } else {
     n += x64_put_u32le(buf + n, (u32)(i32)imm);
     trailing = 4u;
   }
   mc->emit_bytes(mc, buf, n);
   x64_emit_mem_reloc(d, mc, &dst, trailing);
 }
 
 static void expect_comma(AsmDriver* d) {
   if (!asm_driver_eat_comma(d)) asm_driver_panic(d, "x64 asm: expected ','");
 }
 
 static void emit_indirect_branch(MCEmitter* mc, u32 sub, u32 reg) {
   u8 op = 0xff;
   emit_rex(mc, 0, 0, 0, reg);
   mc->emit_bytes(mc, &op, 1);
   {
     u8 mr = modrm(3u, sub, reg);
     mc->emit_bytes(mc, &mr, 1);
   }
 }
 
 static void emit_packed(MCEmitter* mc, const u8* bytes, u32 n) {
   mc->emit_bytes(mc, bytes, n);
 }
 
 static int byte_reg_needs_rex(const X64AsmOperand* op) {
   return op && !op->high8 && op->reg >= 4u;
 }
 
 static void reject_high8_with_rex(AsmDriver* d, const X64AsmOperand* a,
                                   const X64AsmOperand* b) {
   if ((a && a->high8 && byte_reg_needs_rex(b)) ||
       (b && b->high8 && byte_reg_needs_rex(a))) {
     asm_driver_panic(d, "x64 asm: high-byte register cannot use REX");
   }
 }
 
 static __attribute__((unused)) void emit_movb_rr_operand(AsmDriver* d,
                                                          MCEmitter* mc,
                                                          X64AsmOperand dst,
                                                          X64AsmOperand src) {
   u8 ob = 0x88;
   reject_high8_with_rex(d, &dst, &src);
   if (byte_reg_needs_rex(&dst) || byte_reg_needs_rex(&src))
     emit_rex_force(mc, 0, src.reg, 0, dst.reg);
   else
     emit_rex(mc, 0, src.reg, 0, dst.reg);
   mc->emit_bytes(mc, &ob, 1);
   {
     u8 mr = modrm(3u, src.reg, dst.reg);
     mc->emit_bytes(mc, &mr, 1);
   }
 }
 
 static __attribute__((unused)) void emit_movb_store_operand(AsmDriver* d,
                                                             MCEmitter* mc,
                                                             X64AsmOperand src,
                                                             X64AsmOperand dst) {
   if (src.high8) {
     u8 ob = 0x88;
     if (dst.no_base || dst.base >= 8u)
       asm_driver_panic(d, "x64 asm: high-byte register cannot use REX");
     if (dst.seg) mc->emit_bytes(mc, &dst.seg, 1);
     mc->emit_bytes(mc, &ob, 1);
     emit_mem_operand(mc, src.reg, dst.base, dst.disp);
     return;
   }
   emit_mov_store_operand(d, mc, 1, src.reg, dst, 1);
 }
 
 static __attribute__((unused)) void emit_rm_imm(AsmDriver* d, MCEmitter* mc,
                                                 u32 width, u8 opc, u32 sub,
                                                 X64AsmOperand dst, i32 imm,
                                                 int imm32) {
   u8 buf[16];
   u32 n = 0;
   if (dst.kind != X64_ASM_OP_REG && dst.kind != X64_ASM_OP_MEM)
     asm_driver_panic(d, "x64 asm: expected register or memory destination");
   if (width == 2u) buf[n++] = X64_OPSIZE_PFX;
   if (dst.kind == X64_ASM_OP_REG) {
     n += x64_pack_rex(buf + n, width == 8u, 0, 0, dst.reg);
     buf[n++] = opc;
     buf[n++] = x64_modrm(3u, sub, dst.reg);
   } else {
     n += x64_pack_rex(buf + n, width == 8u, 0, 0, dst.base);
     buf[n++] = opc;
     n += x64_pack_mem(buf + n, sub, dst.base, dst.disp);
   }
   if (imm32)
     n += x64_put_u32le(buf + n, (u32)imm);
   else
     buf[n++] = (u8)(i8)imm;
   emit_packed(mc, buf, n);
 }
 
 static __attribute__((unused)) void emit_rm_op(AsmDriver* d, MCEmitter* mc,
                                                u32 width, u8 opc, u32 sub,
                                                X64AsmOperand dst) {
   u8 buf[16];
   u32 n = 0;
   if (dst.kind != X64_ASM_OP_REG && dst.kind != X64_ASM_OP_MEM)
     asm_driver_panic(d, "x64 asm: expected register or memory operand");
   if (width == 2u) buf[n++] = X64_OPSIZE_PFX;
   if (dst.kind == X64_ASM_OP_REG) {
     n += x64_pack_rex(buf + n, width == 8u, 0, 0, dst.reg);
     buf[n++] = opc;
     buf[n++] = x64_modrm(3u, sub, dst.reg);
   } else {
     n += x64_pack_rex(buf + n, width == 8u, 0, 0, dst.base);
     buf[n++] = opc;
     n += x64_pack_mem(buf + n, sub, dst.base, dst.disp);
   }
   emit_packed(mc, buf, n);
 }
 
 static __attribute__((unused)) void emit_reg_rm_twobyte(
     AsmDriver* d, MCEmitter* mc, u32 width, u8 opcode2, u32 dst,
     X64AsmOperand src, int force_rex, u8 prefix) {
   u8 buf[16];
   u32 n = 0;
   if (src.kind != X64_ASM_OP_REG && src.kind != X64_ASM_OP_MEM)
     asm_driver_panic(d, "x64 asm: expected register or memory source");
   if (prefix) buf[n++] = prefix;
   if (src.kind == X64_ASM_OP_REG) {
     if (force_rex)
       n += x64_pack_rex_force(buf + n, width == 8u, dst, 0, src.reg);
     else
       n += x64_pack_rex(buf + n, width == 8u, dst, 0, src.reg);
     buf[n++] = X64_OPC_TWOBYTE;
     buf[n++] = opcode2;
     buf[n++] = x64_modrm(3u, dst, src.reg);
   } else {
     /* Route the full memory-operand variety (plain / SIB-indexed / RIP /
      * segment) through the shared pack helpers so a SIB index register is
      * preserved (e.g. `movzbl (%rcx,%rsi,1), %edx`). */
     if (src.seg) buf[n++] = src.seg;
     n += x64_pack_rex_mem_operand(buf + n, width == 8u, dst, src);
     buf[n++] = X64_OPC_TWOBYTE;
     buf[n++] = opcode2;
     n += x64_pack_mem_operand(buf + n, dst, src);
   }
   emit_packed(mc, buf, n);
   if (src.kind == X64_ASM_OP_MEM) x64_emit_mem_reloc(d, mc, &src, 0);
 }
 
 /* ====================================================================
  * Descriptor-driven mnemonic dispatch.
  *
  * The disassembler's `x64_insn_table` (src/arch/x64/isa.c) lists every
  * encoding kit emits with its X64Format.  We reuse the SAME table for
  * the assembler: linear-scan to find the row whose mnemonic matches the
  * user's AT&T spelling (after stripping the size suffix b/w/l/q), then
  * dispatch to a per-format parser that consumes the operands and calls
  * the existing `emit_*` helpers in emit.c.
  *
  * The width comes from the suffix (or the row's width flags); per-format
  * parsers receive it via a small X64ParseCtx so they can pick the right
  * emit overload (e.g., MOV r,r at 32 vs 64 bits).
  *
  * Note: a single mnemonic ("mov") has multiple table rows for different
  * formats (MOV_RI, ALU_RR, MOV_RM_LOAD).  We return the FIRST row that
  * matches the mnemonic + width filter; per-format parsers that need a
  * different row (e.g., MOV imm→reg uses MOV_RI but our scan may have
  * returned ALU_RR first) fall through to operand-kind dispatch and
  * select the correct emit helper directly.  Phase 1 of this refactor
  * only exercises the mnemonics asm.c handled before; richer disambiguation
  * lands in follow-ups. */
 
 #define X64_SFX_NONE 0u
 #define X64_SFX_B 1u
 #define X64_SFX_W 2u
 #define X64_SFX_L 4u
 #define X64_SFX_Q 8u
 
 typedef struct X64MnInfo {
   char base[16]; /* stripped mnemonic (table-spelling) */
   u32 base_len;
   u32 width; /* X64_SFX_* — 0 if mnemonic carries no size letter */
   u32 cc;    /* condition nibble for jcc/cmovcc/setcc, or 16 if none */
 } X64MnInfo;
 
 /* Parse the user-supplied mnemonic into (root, width, cc).  Handles:
  *   - trailing size letter (b/w/l/q) when the table mnemonic has none
  *   - jXX → ("j", cc)
  *   - cmovXX[q|l|w|b] → ("cmov", cc, width)
  *   - setXX → ("set", cc)
  *   - exact-match mnemonics carried verbatim (movslq, movzbl, ud2, ...) */
 static int parse_mnemonic(const char* s, size_t n, X64MnInfo* out) {
   static const struct {
     const char* name;
     u8 cc;
   } kCC[] = {
       /* Two-letter codes first so e.g. "ne" beats "n" if we ever add it. */
       {"ae", X64_CC_AE}, {"be", X64_CC_BE}, {"ge", X64_CC_GE},
       {"le", X64_CC_LE}, {"ne", X64_CC_NE}, {"no", X64_CC_NO},
       {"np", X64_CC_NP}, {"ns", X64_CC_NS}, {"a", X64_CC_A},
       {"b", X64_CC_B},   {"e", X64_CC_E},   {"g", X64_CC_G},
       {"l", X64_CC_L},   {"o", X64_CC_O},   {"p", X64_CC_P},
       {"s", X64_CC_S},
   };
   out->base_len = 0;
   out->width = X64_SFX_NONE;
   out->cc = 16u;
   if (n == 0 || n >= sizeof out->base) return 0;
   memcpy(out->base, s, n);
   out->base[n] = '\0';
 
   /* Exact-match mnemonics that carry their own width letters or are
    * already canonical table spellings. */
   if (n >= 6 && memcmp(s, "movslq", 6) == 0) {
     memcpy(out->base, "movslq", 6);
     out->base_len = 6;
     out->width = X64_SFX_Q;
     return 1;
   }
   if (n >= 6 && (memcmp(s, "movzbl", 6) == 0 || memcmp(s, "movzwl", 6) == 0 ||
                  memcmp(s, "movsbl", 6) == 0 || memcmp(s, "movswl", 6) == 0 ||
                  memcmp(s, "movzbq", 6) == 0 || memcmp(s, "movzwq", 6) == 0 ||
                  memcmp(s, "movsbq", 6) == 0 || memcmp(s, "movswq", 6) == 0)) {
     memcpy(out->base, s, 6);
     out->base_len = 6;
     return 1;
   }
   if (n == 3 && memcmp(s, "ud2", 3) == 0) {
     out->base_len = 3;
     return 1;
   }
   if (n == 3 && memcmp(s, "nop", 3) == 0) {
     out->base_len = 3;
     return 1;
   }
   if (n == 3 && memcmp(s, "ret", 3) == 0) {
     out->base_len = 3;
     return 1;
   }
   /* "syscall" ends in 'l' — return early so the generic size-suffix
    * stripper below does not mistake it for a movl-style width letter. */
   if (n == 7 && memcmp(s, "syscall", 7) == 0) {
     out->base_len = 7;
     return 1;
   }
 
   /* Indirect-branch spellings carry an explicit 'q' suffix that must be
    * preserved — the BR_RM rows in the table are keyed on "jmpq"/"callq". */
   if (n == 4 && memcmp(s, "call", 4) == 0) {
     memcpy(out->base, "callq", 5);
     out->base[5] = '\0';
     out->base_len = 5;
     out->width = X64_SFX_Q;
     return 1;
   }
   if (n == 4 && memcmp(s, "jmpq", 4) == 0) {
     out->base_len = 4;
     out->width = X64_SFX_Q;
     return 1;
   }
   if (n == 5 && memcmp(s, "callq", 5) == 0) {
     out->base_len = 5;
     out->width = X64_SFX_Q;
     return 1;
   }
 
   /* CMOVcc: cmov<cc>[suffix].  Strip optional trailing q/l/w/b first. */
   if (n >= 5 && memcmp(s, "cmov", 4) == 0) {
     size_t after = 4;
     size_t tail = n;
     char last = s[n - 1];
     if (last == 'b' || last == 'w' || last == 'l' || last == 'q') {
       out->width = (last == 'b')   ? X64_SFX_B
                    : (last == 'w') ? X64_SFX_W
                    : (last == 'l') ? X64_SFX_L
                                    : X64_SFX_Q;
       tail = n - 1;
     }
     if (tail > after) {
       Slice cc = {{s + after}, tail - after};
       for (size_t i = 0; i < sizeof kCC / sizeof kCC[0]; ++i) {
         if (slice_eq_cstr(cc, kCC[i].name)) {
           out->cc = kCC[i].cc;
           memcpy(out->base, "cmov", 4);
           out->base[4] = '\0';
           out->base_len = 4;
           return 1;
         }
       }
     }
   }
 
   /* SETcc: set<cc>. */
   if (n > 3 && memcmp(s, "set", 3) == 0) {
     Slice cc = {{s + 3}, n - 3};
     for (size_t i = 0; i < sizeof kCC / sizeof kCC[0]; ++i) {
       if (slice_eq_cstr(cc, kCC[i].name)) {
         out->cc = kCC[i].cc;
         memcpy(out->base, "set", 3);
         out->base[3] = '\0';
         out->base_len = 3;
         return 1;
       }
     }
   }
 
   /* Jcc: j<cc> — but NOT "jmp" / "jmpq" (handled above). */
   if (n > 1 && s[0] == 'j' && !(n >= 3 && s[1] == 'm' && s[2] == 'p')) {
     Slice cc = {{s + 1}, n - 1};
     for (size_t i = 0; i < sizeof kCC / sizeof kCC[0]; ++i) {
       if (slice_eq_cstr(cc, kCC[i].name)) {
         out->cc = kCC[i].cc;
         out->base[0] = 'j';
         out->base[1] = '\0';
         out->base_len = 1;
         return 1;
       }
     }
   }
 
   /* Generic: strip trailing size letter b/w/l/q. */
   {
     char last = s[n - 1];
     if (last == 'b' || last == 'w' || last == 'l' || last == 'q') {
       out->width = (last == 'b')   ? X64_SFX_B
                    : (last == 'w') ? X64_SFX_W
                    : (last == 'l') ? X64_SFX_L
                                    : X64_SFX_Q;
       out->base_len = (u32)(n - 1);
       out->base[out->base_len] = '\0';
       return 1;
     }
   }
 
   out->base_len = (u32)n;
   return 1;
 }
 
 /* Width implied by a descriptor row, given the mnemonic's parsed width. */
 static u32 row_implied_width(const X64InsnDesc* d) {
   if (d->flags & X64_ASMFL_FORCE_W64) return 8u;
   if (d->flags & X64_ASMFL_BYTE) return 1u;
   if (d->flags & X64_ASMFL_W16) return 2u;
   if (d->flags & X64_ASMFL_W_FROM_REX) return 0u; /* any */
   if (d->leg_pfx == X64_PFX_66) return 2u;
   return 0u; /* any */
 }
 
 /* Linear scan for the first table row whose mnemonic matches `info->base`
  * AND whose width filter is compatible.  Returns NULL on miss. */
 static const X64InsnDesc* find_mnemonic_row(const X64MnInfo* info) {
   u32 want_w = info->width; /* 0 = any */
   Slice base = {{info->base}, info->base_len};
   for (u32 i = 0; i < x64_insn_table_n; ++i) {
     const X64InsnDesc* d = &x64_insn_table[i];
     if (!slice_eq(d->mnemonic, base)) continue;
     if (want_w != 0) {
       u32 rw = row_implied_width(d);
       if (rw != 0 && rw != want_w) continue;
     }
     return d;
   }
   return NULL;
 }
 
 /* Per-format parse context. */
 typedef struct X64ParseCtx {
   AsmDriver* d;
   MCEmitter* mc;
   const X64InsnDesc* desc;
   u32 width; /* 1/2/4/8 — derived from suffix or row */
   u32 cc;    /* condition nibble (jcc/cmovcc/setcc); 16 if unused */
 } X64ParseCtx;
 
 /* w-bit for emit_rex / emit_alu_rr / emit_mov_rr etc. */
 static int width_to_w(u32 w) { return w == 8u ? 1 : 0; }
 
 /* ---- per-format parsers ---- */
 
 static void parse_nullary(X64ParseCtx* p) {
   /* nop / ret / ud2 / leave / cltd / cqto. */
   u8 buf[4];
   u32 n = 0;
   if (p->desc->leg_pfx) buf[n++] = p->desc->leg_pfx;
   if (p->desc->rex_w_req == X64_W_REQ_1) buf[n++] = X64_REX_BASE | X64_REX_W;
   for (u32 i = 0; i < p->desc->opc_len; ++i) buf[n++] = p->desc->opc[i];
   if (p->desc->opc_len >= 1u) {
     p->mc->emit_bytes(p->mc, buf, n);
     return;
   }
   asm_driver_panic(p->d, "x64 asm: nullary form not implemented");
 }
 
 static void parse_br_rm(X64ParseCtx* p) {
   /* jmpq *%reg or callq *%reg.  /digit picks sub (2 = call, 4 = jmp). */
   X64AsmOperand op = parse_operand(p->d);
   if (op.kind != X64_ASM_OP_IND_REG)
     asm_driver_panic(p->d, "x64 asm: indirect branch form");
   emit_indirect_branch(p->mc, p->desc->modrm_reg, op.reg);
 }
 
 /* Look up the ALU_RM_IMM8 row for a given mnemonic root; the /digit
  * picks the operation (0=add, 1=or, 4=and, 5=sub, 6=xor, 7=cmp). */
 static const X64InsnDesc* find_alu_imm_row(Slice root) {
   for (u32 i = 0; i < x64_insn_table_n; ++i) {
     const X64InsnDesc* d = &x64_insn_table[i];
     if (d->fmt != X64_FMT_ALU_RM_IMM8) continue;
     if (!slice_eq(d->mnemonic, root)) continue;
     return d;
   }
   return NULL;
 }
 
 static void parse_alu_rr(X64ParseCtx* p) {
   /* op src, dst  in AT&T.  Row's opc[0] is the ALU opcode (0x01/0x09/...
    * 0x31/0x85/0x89).  The byte/16-bit forms are handled by the
    * existing emit.c helpers for w=0/w=1 + size suffix; here phase-1
    * supports only the regular 32/64 forms used by the prior asm.c. */
   X64AsmOperand src;
   X64AsmOperand dst;
   src = parse_operand(p->d);
   expect_comma(p->d);
   dst = parse_operand(p->d);
 
   /* Immediate source → not an ALU_RR encoding.  Redirect to the
    * ALU_RM_IMM row for this mnemonic. */
   if (src.kind == X64_ASM_OP_IMM &&
       (dst.kind == X64_ASM_OP_REG || dst.kind == X64_ASM_OP_MEM)) {
     const X64InsnDesc* imm_row = find_alu_imm_row(p->desc->mnemonic);
     if (!imm_row) asm_driver_panic(p->d, "x64 asm: no alu-imm row");
     if (dst.kind == X64_ASM_OP_MEM) {
       emit_rm_imm_store_operand(p->d, p->mc, p->width, X64_OPC_ALU_IMM8,
                                 X64_OPC_ALU_IMM32, imm_row->modrm_reg, dst,
                                 src.imm, 1);
       return;
     }
     /* Stack-pointer adjustments (`add/sub $imm, %rsp`, 64-bit) always use the
      * imm32 form in codegen — the prologue and alloca patch a fixed-width
      * placeholder, so they never shrink to imm8 even for a small frame. Match
      * that here so `cc -S | as` reproduces codegen's bytes exactly; %rsp is a
      * reserved register, so codegen never emits an imm8 ALU op against it. */
     if (dst.reg == X64_RSP && p->width == 8u && imm_fits_i32(src.imm))
       emit_alu_imm32(p->mc, 1, imm_row->modrm_reg, dst.reg, (i32)src.imm);
     else if (imm_fits_i8(src.imm))
       emit_alu_imm8(p->mc, width_to_w(p->width), imm_row->modrm_reg, dst.reg,
                     (i8)src.imm);
     else if (imm_fits_i32(src.imm))
       emit_alu_imm32(p->mc, width_to_w(p->width), imm_row->modrm_reg, dst.reg,
                      (i32)src.imm);
     else
       asm_driver_panic(p->d, "x64 asm: alu-imm out of range");
     return;
   }
 
   if (src.kind == X64_ASM_OP_REG && dst.kind == X64_ASM_OP_REG) {
     u8 op = p->desc->opc[0];
     if (p->width == 2u) {
       u8 pfx = X64_OPSIZE_PFX;
       p->mc->emit_bytes(p->mc, &pfx, 1);
     }
     if (op == 0x89u) {
       /* MOV r/m, r — phase-1 keeps the existing helper. */
       emit_mov_rr(p->mc, width_to_w(p->width), dst.reg, src.reg);
       return;
     }
     if (op == 0x88u) {
       emit_movb_rr_operand(p->d, p->mc, dst, src);
       return;
     }
     if (op == 0x88u) {
       /* MOV r/m8, r8 — byte form (preserved from prior asm.c). */
       u8 ob = 0x88;
       emit_rex(p->mc, 0, src.reg, 0, dst.reg);
       p->mc->emit_bytes(p->mc, &ob, 1);
       {
         u8 mr = modrm(3u, src.reg, dst.reg);
         p->mc->emit_bytes(p->mc, &mr, 1);
       }
       return;
     }
     /* xor/test/and/... — emit_alu_rr handles the generic shape. */
     emit_alu_rr(p->mc, width_to_w(p->width), op, dst.reg, src.reg);
     return;
   }
   /* MOV r, r/m goes through MOV_RM_LOAD; MOV r, m goes through
    * MOV_RM_LOAD (load) or ALU_RR with mem dst (store).  We handle the
    * store side here only when the mnemonic is "mov" (opc 0x89). */
   if (p->desc->opc[0] == 0x89u && src.kind == X64_ASM_OP_REG &&
       dst.kind == X64_ASM_OP_MEM) {
     if (p->width == 1u)
       emit_movb_store_operand(p->d, p->mc, src, dst);
     else
       emit_mov_store_operand(p->d, p->mc, p->width, src.reg, dst, 0);
     return;
   }
   if (p->desc->opc[0] == 0x89u && src.kind == X64_ASM_OP_MEM &&
       dst.kind == X64_ASM_OP_REG) {
     emit_mov_load_operand(p->d, p->mc, p->width, dst.reg, src);
     return;
   }
   /* ALU reg → mem store (add/or/and/sub/xor/cmp %reg, mem): the reg-to-r/m
    * /r opcode (opc[0]) with a memory ModR/M.  The byte form clears the
    * opcode's W bit (e.g. ADD r/m,r 0x01 → r/m8,r8 0x00). */
   if (src.kind == X64_ASM_OP_REG && dst.kind == X64_ASM_OP_MEM) {
     u8 op = p->width == 1u ? (u8)(p->desc->opc[0] & ~1u) : p->desc->opc[0];
     emit_reg_store_operand(p->d, p->mc, p->width, op, src.reg, dst,
                            p->width == 1u && byte_reg_needs_rex(&src));
     return;
   }
   asm_driver_panic(p->d, "x64 asm: unsupported alu_rr form");
 }
 
 static void parse_mov_ri(X64ParseCtx* p) {
   X64AsmOperand src;
   X64AsmOperand dst;
   src = parse_operand(p->d);
   expect_comma(p->d);
   dst = parse_operand(p->d);
   if (src.kind != X64_ASM_OP_IMM ||
       (dst.kind != X64_ASM_OP_REG && dst.kind != X64_ASM_OP_MEM))
     asm_driver_panic(p->d, "x64 asm: mov-imm form");
   /* MOV $imm → mem: C6 /0 (byte) or C7 /0 (32/64 sign-extended imm32). */
   if (dst.kind == X64_ASM_OP_MEM) {
     if (p->width != 8u && !imm_fits_i32(src.imm))
       asm_driver_panic(p->d, "x64 asm: mov immediate out of range");
     emit_rm_imm_store_operand(
         p->d, p->mc, p->width, X64_OPC_MOV_RM_IMM8,
         p->width == 1u ? X64_OPC_MOV_RM_IMM8 : X64_OPC_MOV_RM_IMM32,
         X64_MOV_RM_IMM_SUB, dst, src.imm, 0);
     return;
   }
   if (p->width != 4u && p->width != 8u)
     asm_driver_panic(p->d, "x64 asm: mov imm only supports l/q forms");
   x64_emit_load_imm(p->mc, p->width == 8u ? 1 : 0, dst.reg, src.imm);
 }
 
 static void parse_mov_rm_load(X64ParseCtx* p) {
   /* MOV r, r/m (0x8B) or LEA r, m (0x8D).  AT&T order is src, dst.
    * Phase-1 covers reg-reg, reg←mem (load) and lea. */
   X64AsmOperand src;
   X64AsmOperand dst;
   src = parse_operand(p->d);
   expect_comma(p->d);
   dst = parse_operand(p->d);
   if (p->desc->opc[0] == 0x8Du) {
     if (src.kind != X64_ASM_OP_MEM || dst.kind != X64_ASM_OP_REG)
       asm_driver_panic(p->d, "x64 asm: lea form");
     emit_reg_mem_operand(p->d, p->mc, p->width, X64_OPC_LEA, dst.reg, src);
     return;
   }
   if (src.kind == X64_ASM_OP_MEM && dst.kind == X64_ASM_OP_REG) {
     emit_mov_load_operand(p->d, p->mc, p->width, dst.reg, src);
     return;
   }
   if (src.kind == X64_ASM_OP_REG && dst.kind == X64_ASM_OP_REG) {
     if (p->width == 2u) {
       u8 pfx = X64_OPSIZE_PFX;
       p->mc->emit_bytes(p->mc, &pfx, 1);
     }
     emit_mov_rr(p->mc, width_to_w(p->width), dst.reg, src.reg);
     return;
   }
   asm_driver_panic(p->d, "x64 asm: mov-load form");
 }
 
 static void parse_movsxd(X64ParseCtx* p) {
   X64AsmOperand src;
   X64AsmOperand dst;
   src = parse_operand(p->d);
   expect_comma(p->d);
   dst = parse_operand(p->d);
   if (dst.kind != X64_ASM_OP_REG)
     asm_driver_panic(p->d, "x64 asm: movslq form");
   if (src.kind == X64_ASM_OP_REG) {
     emit_extend_rr(p->mc, 1, 1, 4, dst.reg, src.reg);
   } else if (src.kind == X64_ASM_OP_MEM) {
     u8 buf[16];
     u32 n = x64_mov_rm_load_pack((X64MovRMLoad){.w = 1,
                                                 .opc0 = X64_OPC_MOVSXD,
                                                 .dst = dst.reg,
                                                 .base = src.base,
                                                 .disp = src.disp},
                                  buf);
     emit_packed(p->mc, buf, n);
   } else {
     asm_driver_panic(p->d, "x64 asm: movslq source");
   }
 }
 
 static void parse_alu_rm_imm(X64ParseCtx* p) {
   X64AsmOperand src;
   X64AsmOperand dst;
   src = parse_operand(p->d);
   expect_comma(p->d);
   dst = parse_operand(p->d);
   if (src.kind != X64_ASM_OP_IMM || dst.kind != X64_ASM_OP_REG)
     asm_driver_panic(p->d, "x64 asm: alu-imm form");
   if (imm_fits_i8(src.imm))
     emit_alu_imm8(p->mc, width_to_w(p->width), p->desc->modrm_reg, dst.reg,
                   (i8)src.imm);
   else if (imm_fits_i32(src.imm))
     emit_alu_imm32(p->mc, width_to_w(p->width), p->desc->modrm_reg, dst.reg,
                    (i32)src.imm);
   else
     asm_driver_panic(p->d, "x64 asm: alu-imm out of range");
 }
 
 static void parse_cmovcc(X64ParseCtx* p) {
   X64AsmOperand src;
   X64AsmOperand dst;
   src = parse_operand(p->d);
   expect_comma(p->d);
   dst = parse_operand(p->d);
   if (src.kind != X64_ASM_OP_REG || dst.kind != X64_ASM_OP_REG)
     asm_driver_panic(p->d, "x64 asm: cmovcc form");
   {
     u8 op[2] = {0x0f, (u8)(0x40u | (p->cc & 0xfu))};
     if (p->width == 2u) {
       u8 pfx = X64_OPSIZE_PFX;
       p->mc->emit_bytes(p->mc, &pfx, 1);
     }
     emit_rex(p->mc, width_to_w(p->width), dst.reg, 0, src.reg);
     p->mc->emit_bytes(p->mc, op, 2);
     emit_rm_reg(p->mc, dst.reg, src.reg);
   }
 }
 
 static void parse_push_pop(X64ParseCtx* p) {
   X64AsmOperand op = parse_operand(p->d);
   u8 base = p->desc->opc[0];
   u8 ob;
   if (op.kind != X64_ASM_OP_REG)
     asm_driver_panic(p->d, "x64 asm: push/pop register");
   emit_rex(p->mc, 0, 0, 0, op.reg);
   ob = (u8)(base | (op.reg & 7u));
   p->mc->emit_bytes(p->mc, &ob, 1);
 }
 
 static void parse_movzx_movsx(X64ParseCtx* p) {
   X64AsmOperand src = parse_operand(p->d);
   X64AsmOperand dst;
   expect_comma(p->d);
   dst = parse_operand(p->d);
   if (dst.kind != X64_ASM_OP_REG)
     asm_driver_panic(p->d, "x64 asm: movx dst register");
   /* REX.W follows the destination register width: `movsbq …, %rcx` (64-bit)
    * needs REX.W; `movsbl …, %ecx` (32-bit) does not. The disassembler spells
    * the q/l form from REX.W, so honoring dst width here round-trips it. */
   emit_reg_rm_twobyte(
       p->d, p->mc, dst.width == 8u ? 8u : 4u, p->desc->opc[1], dst.reg, src,
       p->desc->opc[1] == X64_OPC_MOVZX_B || p->desc->opc[1] == X64_OPC_MOVSX_B,
       0);
 }
 
 static void parse_imul_rr(X64ParseCtx* p) {
   X64AsmOperand src = parse_operand(p->d);
   X64AsmOperand dst;
   if (src.kind == X64_ASM_OP_IMM) {
     X64AsmOperand real_src;
     expect_comma(p->d);
     real_src = parse_operand(p->d);
     expect_comma(p->d);
     dst = parse_operand(p->d);
     if (dst.kind != X64_ASM_OP_REG)
       asm_driver_panic(p->d, "x64 asm: imul dst register");
     if (real_src.kind == X64_ASM_OP_REG) {
       if (imm_fits_i8(src.imm))
         emit_imul_imm8(p->mc, width_to_w(p->width), dst.reg, real_src.reg,
                        (i8)src.imm);
       else if (imm_fits_i32(src.imm))
         emit_imul_imm32(p->mc, width_to_w(p->width), dst.reg, real_src.reg,
                         (i32)src.imm);
       else
         asm_driver_panic(p->d, "x64 asm: imul imm out of range");
       return;
     }
     if (real_src.kind == X64_ASM_OP_MEM) {
       u8 buf[16];
       u32 n = 0;
       int imm32 = !imm_fits_i8(src.imm);
       if (imm32 && !imm_fits_i32(src.imm))
         asm_driver_panic(p->d, "x64 asm: imul imm out of range");
       n += x64_pack_rex(buf + n, width_to_w(p->width), dst.reg, 0,
                         real_src.base);
       buf[n++] = imm32 ? X64_OPC_IMUL_IMM32 : X64_OPC_IMUL_IMM8;
       n += x64_pack_mem(buf + n, dst.reg, real_src.base, real_src.disp);
       if (imm32)
         n += x64_put_u32le(buf + n, (u32)(i32)src.imm);
       else
         buf[n++] = (u8)(i8)src.imm;
       emit_packed(p->mc, buf, n);
       return;
     }
     asm_driver_panic(p->d, "x64 asm: imul source");
   }
   expect_comma(p->d);
   dst = parse_operand(p->d);
   if (dst.kind != X64_ASM_OP_REG)
     asm_driver_panic(p->d, "x64 asm: imul dst register");
   emit_reg_rm_twobyte(p->d, p->mc, p->width, X64_OPC_IMUL_2B, dst.reg, src, 0,
                       0);
 }
 
 static void parse_imul_rri(X64ParseCtx* p) {
   X64AsmOperand imm = parse_operand(p->d);
   X64AsmOperand src;
   X64AsmOperand dst;
   if (imm.kind != X64_ASM_OP_IMM) asm_driver_panic(p->d, "x64 asm: imul imm");
   expect_comma(p->d);
   src = parse_operand(p->d);
   expect_comma(p->d);
   dst = parse_operand(p->d);
   if (dst.kind != X64_ASM_OP_REG)
     asm_driver_panic(p->d, "x64 asm: imul dst register");
   if (src.kind == X64_ASM_OP_REG) {
     if (p->desc->opc[0] == X64_OPC_IMUL_IMM8 || imm_fits_i8(imm.imm))
       emit_imul_imm8(p->mc, width_to_w(p->width), dst.reg, src.reg,
                      (i8)imm.imm);
     else if (imm_fits_i32(imm.imm))
       emit_imul_imm32(p->mc, width_to_w(p->width), dst.reg, src.reg,
                       (i32)imm.imm);
     else
       asm_driver_panic(p->d, "x64 asm: imul imm out of range");
     return;
   }
   if (src.kind == X64_ASM_OP_MEM) {
     u8 buf[16];
     u32 n = 0;
     int imm32 = !(p->desc->opc[0] == X64_OPC_IMUL_IMM8 || imm_fits_i8(imm.imm));
     if (imm32 && !imm_fits_i32(imm.imm))
       asm_driver_panic(p->d, "x64 asm: imul imm out of range");
     n += x64_pack_rex(buf + n, width_to_w(p->width), dst.reg, 0, src.base);
     buf[n++] = imm32 ? X64_OPC_IMUL_IMM32 : X64_OPC_IMUL_IMM8;
     n += x64_pack_mem(buf + n, dst.reg, src.base, src.disp);
     if (imm32)
       n += x64_put_u32le(buf + n, (u32)(i32)imm.imm);
     else
       buf[n++] = (u8)(i8)imm.imm;
     emit_packed(p->mc, buf, n);
     return;
   }
   asm_driver_panic(p->d, "x64 asm: imul source");
 }
 
 static void parse_f7_rm(X64ParseCtx* p) {
   X64AsmOperand op = parse_operand(p->d);
   emit_rm_op(p->d, p->mc, p->width, X64_OPC_F7, p->desc->modrm_reg, op);
 }
 
 static void parse_shift(X64ParseCtx* p) {
   X64AsmOperand src = parse_operand(p->d);
   X64AsmOperand dst;
   expect_comma(p->d);
   dst = parse_operand(p->d);
   if (src.kind == X64_ASM_OP_REG && src.reg == X64_RCX && src.width == 1u) {
     emit_rm_op(p->d, p->mc, p->width, X64_OPC_SHIFT_CL, p->desc->modrm_reg,
                dst);
     return;
   }
   if (src.kind != X64_ASM_OP_IMM) asm_driver_panic(p->d, "x64 asm: shift imm");
   emit_rm_imm(p->d, p->mc, p->width, X64_OPC_SHIFT_IMM, p->desc->modrm_reg, dst,
               (i32)src.imm, 0);
 }
 
 static void parse_rel32_branch(X64ParseCtx* p) {
   ObjSymId sym = OBJ_SYM_NONE;
   i64 off = 0;
   u32 disp_pos;
   if (p->desc->fmt == X64_FMT_JCC_REL32) {
     u8 op[2] = {0x0f, (u8)(0x80u | (p->cc & 0xfu))};
     p->mc->emit_bytes(p->mc, op, 2);
   } else {
     u8 op = (p->desc->fmt == X64_FMT_CALL_REL32) ? X64_OPC_CALL_REL32
                                                  : X64_OPC_JMP_REL32;
     p->mc->emit_bytes(p->mc, &op, 1);
   }
   disp_pos = p->mc->pos(p->mc);
   emit_u32le(p->mc, 0);
   asm_driver_parse_sym_expr(p->d, &sym, &off);
   if (sym == OBJ_SYM_NONE)
     asm_driver_panic(p->d, "x64 asm: symbolic branch target required");
   /* A `@PLT` suffix forces the PLT32 reloc (the default for `call`); plain
    * `jmp sym` uses PC32. */
   RelocKind dflt = p->desc->fmt == X64_FMT_CALL_REL32 ? R_X64_PLT32 : R_PC32;
   RelocKind k = x64_parse_reloc_suffix(p->d, dflt);
   if (k != R_X64_PLT32 && k != R_PC32)
     asm_driver_panic(p->d, "x64 asm: only @PLT is valid on a branch target");
   p->mc->emit_reloc_at(p->mc, asm_driver_cur_section(p->d), disp_pos, k, sym,
                        off - 4, 1, 0);
 }
 
 static void parse_setcc(X64ParseCtx* p) {
   X64AsmOperand dst = parse_operand(p->d);
   if (dst.kind == X64_ASM_OP_REG) {
     if (dst.high8) {
       u8 op[2] = {0x0f, (u8)(0x90u | (p->cc & 0xfu))};
       p->mc->emit_bytes(p->mc, op, 2);
       emit_rm_reg(p->mc, 0, dst.reg);
     } else {
       emit_setcc(p->mc, p->cc, dst.reg);
     }
     return;
   }
   if (dst.kind == X64_ASM_OP_MEM) {
     u8 buf[16];
     u32 n = x64_pack_rex(buf, 0, 0, 0, dst.base);
     buf[n++] = X64_OPC_TWOBYTE;
     buf[n++] = (u8)(X64_OPC_SETCC_BASE | (p->cc & 0xfu));
     n += x64_pack_mem(buf + n, 0, dst.base, dst.disp);
     emit_packed(p->mc, buf, n);
     return;
   }
   asm_driver_panic(p->d, "x64 asm: setcc destination");
 }
 
 static void parse_sse_rr(X64ParseCtx* p) {
   X64AsmOperand src = parse_operand(p->d);
   X64AsmOperand dst;
   int cvt_to_int = p->desc->opc[1] == 0x2cu;
   int cvt_from_int = p->desc->opc[1] == 0x2au;
   expect_comma(p->d);
   dst = parse_operand(p->d);
   if (cvt_to_int) {
     /* cvttsd2si/cvttss2si XMM/m -> GPR: REX.W follows the GPR destination
      * width (`%rdx` = 64-bit, `%edx` = 32-bit), not the mnemonic — these rows
      * carry no size suffix. */
     if (dst.kind != X64_ASM_OP_REG)
       asm_driver_panic(p->d, "x64 asm: cvtt dst register");
     u32 gpr_w = dst.width == 8u ? 8u : 4u;
     if (src.kind == X64_ASM_OP_XMM)
       emit_sse_rr_w(p->mc, p->desc->leg_pfx, p->desc->opc[1], width_to_w(gpr_w),
                     dst.reg, src.reg);
     else if (src.kind == X64_ASM_OP_MEM)
       emit_reg_rm_twobyte(p->d, p->mc, gpr_w, p->desc->opc[1], dst.reg, src, 0,
                           p->desc->leg_pfx);
     else
       asm_driver_panic(p->d, "x64 asm: cvtt source");
     return;
   }
   if (cvt_from_int) {
     /* cvtsi2sd/cvtsi2ss GPR/m -> XMM: REX.W follows the GPR source width. */
     if (dst.kind != X64_ASM_OP_XMM)
       asm_driver_panic(p->d, "x64 asm: cvtsi dst xmm");
     if (src.kind == X64_ASM_OP_REG) {
       u32 gpr_w = src.width == 8u ? 8u : 4u;
       emit_sse_rr_w(p->mc, p->desc->leg_pfx, p->desc->opc[1], width_to_w(gpr_w),
                     dst.reg, src.reg);
     } else if (src.kind == X64_ASM_OP_MEM)
       emit_sse_load(p->mc, p->desc->leg_pfx, p->desc->opc[1], dst.reg, src.base,
                     src.disp);
     else
       asm_driver_panic(p->d, "x64 asm: cvtsi source");
     return;
   }
   if (dst.kind == X64_ASM_OP_MEM && src.kind == X64_ASM_OP_XMM &&
       p->desc->opc[1] == 0x10u &&
       (slice_eq_cstr(p->desc->mnemonic, "movsd") ||
        slice_eq_cstr(p->desc->mnemonic, "movss"))) {
     emit_sse_store(p->mc, p->desc->leg_pfx, 0x11, src.reg, dst.base, dst.disp);
     return;
   }
   if (dst.kind == X64_ASM_OP_MEM && src.kind == X64_ASM_OP_XMM &&
       p->desc->opc[1] == 0x28u && slice_eq_cstr(p->desc->mnemonic, "movaps")) {
     emit_sse_store(p->mc, p->desc->leg_pfx, 0x29, src.reg, dst.base, dst.disp);
     return;
   }
   if (dst.kind != X64_ASM_OP_XMM)
     asm_driver_panic(p->d, "x64 asm: sse dst xmm");
   if (src.kind == X64_ASM_OP_XMM)
     emit_sse_rr(p->mc, p->desc->leg_pfx, p->desc->opc[1], dst.reg, src.reg);
   else if (src.kind == X64_ASM_OP_MEM)
     emit_sse_load(p->mc, p->desc->leg_pfx, p->desc->opc[1], dst.reg, src.base,
                   src.disp);
   else
     asm_driver_panic(p->d, "x64 asm: sse source");
 }
 
 static void parse_bswap(X64ParseCtx* p) {
   X64AsmOperand reg = parse_operand(p->d);
   u8 op[2];
   if (reg.kind != X64_ASM_OP_REG) asm_driver_panic(p->d, "x64 asm: bswap reg");
   emit_rex(p->mc, width_to_w(p->width), 0, 0, reg.reg);
   op[0] = 0x0f;
   op[1] = (u8)(0xc8u | (reg.reg & 7u));
   p->mc->emit_bytes(p->mc, op, 2);
 }
 
 static void parse_bs_popcnt(X64ParseCtx* p) {
   X64AsmOperand src = parse_operand(p->d);
   X64AsmOperand dst;
   expect_comma(p->d);
   dst = parse_operand(p->d);
   if (dst.kind != X64_ASM_OP_REG)
     asm_driver_panic(p->d, "x64 asm: bit-scan dst register");
   emit_reg_rm_twobyte(p->d, p->mc, p->width, p->desc->opc[1], dst.reg, src, 0,
                       p->desc->leg_pfx);
 }
 
 static void parse_atomic(X64ParseCtx* p) {
   X64AsmOperand src = parse_operand(p->d);
   X64AsmOperand dst;
   u8 buf[16];
   u32 n = 0;
   expect_comma(p->d);
   dst = parse_operand(p->d);
   if (src.kind != X64_ASM_OP_REG ||
       (dst.kind != X64_ASM_OP_REG && dst.kind != X64_ASM_OP_MEM))
     asm_driver_panic(p->d, "x64 asm: atomic form");
   n += x64_pack_rex(buf + n, width_to_w(p->width), src.reg, 0,
                     dst.kind == X64_ASM_OP_REG ? dst.reg : dst.base);
   if (p->desc->opc_len == 2) {
     buf[n++] = X64_OPC_TWOBYTE;
     buf[n++] = p->desc->opc[1];
   } else {
     buf[n++] = p->desc->opc[0];
   }
   if (dst.kind == X64_ASM_OP_REG)
     buf[n++] = x64_modrm(3u, src.reg, dst.reg);
   else
     n += x64_pack_mem(buf + n, src.reg, dst.base, dst.disp);
   emit_packed(p->mc, buf, n);
 }
 
 static void parse_nop_multi(X64ParseCtx* p) {
   u8 nop6[6] = {X64_NOP6_BYTE0, X64_NOP6_BYTE1, X64_NOP6_BYTE2,
                 X64_NOP6_BYTE3, X64_NOP6_BYTE4, X64_NOP6_BYTE5};
   p->mc->emit_bytes(p->mc, nop6, sizeof nop6);
 }
 
 static void parse_and_emit_for_format(X64ParseCtx* p) {
   switch ((X64Format)p->desc->fmt) {
     case X64_FMT_NULLARY:
       parse_nullary(p);
       return;
     case X64_FMT_NOP_MULTI:
       parse_nop_multi(p);
       return;
     case X64_FMT_PUSH_POP:
       parse_push_pop(p);
       return;
     case X64_FMT_BR_RM:
       parse_br_rm(p);
       return;
     case X64_FMT_ALU_RR:
       parse_alu_rr(p);
       return;
     case X64_FMT_MOV_RI:
       parse_mov_ri(p);
       return;
     case X64_FMT_MOV_RM_LOAD:
       parse_mov_rm_load(p);
       return;
     case X64_FMT_MOVZX_MOVSX:
       parse_movzx_movsx(p);
       return;
     case X64_FMT_MOVSXD:
       parse_movsxd(p);
       return;
     case X64_FMT_ALU_RM_IMM8:
     case X64_FMT_ALU_RM_IMM32:
       parse_alu_rm_imm(p);
       return;
     case X64_FMT_CMOVCC_RR:
       parse_cmovcc(p);
       return;
     case X64_FMT_IMUL_RR:
       parse_imul_rr(p);
       return;
     case X64_FMT_IMUL_RRI:
       parse_imul_rri(p);
       return;
     case X64_FMT_F7_RM:
       parse_f7_rm(p);
       return;
     case X64_FMT_SHIFT_IMM:
     case X64_FMT_SHIFT_CL:
       parse_shift(p);
       return;
     case X64_FMT_JCC_REL32:
     case X64_FMT_JMP_REL32:
     case X64_FMT_CALL_REL32:
       parse_rel32_branch(p);
       return;
     case X64_FMT_SETCC_RM:
       parse_setcc(p);
       return;
     case X64_FMT_SSE_RR:
     case X64_FMT_SSE_LOAD:
     case X64_FMT_SSE_STORE:
       parse_sse_rr(p);
       return;
     case X64_FMT_BSWAP:
       parse_bswap(p);
       return;
     case X64_FMT_BS:
     case X64_FMT_POPCNT:
       parse_bs_popcnt(p);
       return;
     case X64_FMT_XADD_MEM:
     case X64_FMT_XCHG_MEM:
     case X64_FMT_CMPXCHG_MEM:
       parse_atomic(p);
       return;
     default:
       asm_driver_panic(p->d, "x64 asm: format not implemented");
   }
 }
 
 /* Width letter (b/w/l/q) → width in bytes.  Falls back to row-implied
  * width if the suffix is absent. */
 static u32 width_from_info(const X64MnInfo* info, const X64InsnDesc* d) {
   if (info->width != 0) return info->width;
   {
     u32 rw = row_implied_width(d);
     return rw ? rw : 4u;
   }
 }
 
 static void x64_arch_asm_insn(ArchAsm* base, AsmDriver* d, Sym mnemonic) {
   X64Asm* a = (X64Asm*)base;
   MCEmitter* mc = asm_driver_mc(d);
   Slice mnsl = pool_slice(asm_driver_pool(d), mnemonic);
   const char* p = mnsl.s;
   size_t n = mnsl.len;
   X64MnInfo info;
   const X64InsnDesc* desc;
   X64ParseCtx ctx;
   (void)a;
   (void)asm_driver_cur_section(d);
 
   if (!p || !parse_mnemonic(p, n, &info))
     asm_driver_panic(d, "x64 asm: bad mnemonic");
 
   if (n == 4 && memcmp(p, "lock", 4) == 0) {
     AsmTok next;
     u8 pfx = 0xf0;
     mc->emit_bytes(mc, &pfx, 1);
     next = asm_driver_next(d);
     if (next.kind != ASM_TOK_IDENT)
       asm_driver_panic(d, "x64 asm: lock requires an instruction");
     x64_arch_asm_insn(base, d, next.v.ident);
     return;
   }
 
   /* Special case: imm→reg "mov" still spelled "movl"/"movq" but the
    * generic scan returns ALU_RR (0x89) first.  When we see a "$"
    * immediate as the first operand, we want MOV_RI instead.  Defer this
    * disambiguation to parse_alu_rr would force pre-parsing operands;
    * simpler is to special-case MOV here. */
   if (info.base_len == 3 && memcmp(info.base, "mov", 3) == 0) {
     /* Peek for leading '$' → immediate form. */
     AsmTok t = asm_driver_peek(d);
     if (asm_driver_tok_is_punct(t, '$')) {
       /* Find the MOV_RI row. */
       for (u32 i = 0; i < x64_insn_table_n; ++i) {
         const X64InsnDesc* dr = &x64_insn_table[i];
         if (dr->fmt == X64_FMT_MOV_RI && slice_eq_cstr(dr->mnemonic, "mov")) {
           ctx.d = d;
           ctx.mc = mc;
           ctx.desc = dr;
           ctx.width = info.width ? info.width : 4u;
           ctx.cc = info.cc;
           parse_mov_ri(&ctx);
           return;
         }
       }
     }
     /* For mov reg,mem and mov mem,reg we need MOV_RM_LOAD (0x8B) for
      * the load side.  Easiest: pre-parse src; if mem and dst is reg →
      * MOV_RM_LOAD.  Doing so re-uses the AT&T parser cleanly. */
     {
       X64AsmOperand src = parse_operand(d);
       expect_comma(d);
       {
         X64AsmOperand dst = parse_operand(d);
         u32 w = info.width ? info.width : 4u;
         if (src.kind == X64_ASM_OP_REG && dst.kind == X64_ASM_OP_REG) {
           if (w == 1u) {
             /* MOV r/m8, r8 — opcode 0x88. */
             emit_movb_rr_operand(d, mc, dst, src);
             return;
           }
           if (w == 2u) {
             u8 pfx = X64_OPSIZE_PFX;
             mc->emit_bytes(mc, &pfx, 1);
           }
           emit_mov_rr(mc, width_to_w(w), dst.reg, src.reg);
           return;
         }
         if (src.kind == X64_ASM_OP_REG && dst.kind == X64_ASM_OP_MEM) {
           if (w == 1u)
             emit_movb_store_operand(d, mc, src, dst);
           else
             emit_mov_store_operand(d, mc, w, src.reg, dst, 0);
           return;
         }
         if (src.kind == X64_ASM_OP_MEM && dst.kind == X64_ASM_OP_REG) {
           emit_mov_load_operand(d, mc, w, dst.reg, src);
           return;
         }
         /* movd/movq between a GPR and an XMM register: 66 [REX.W] 0F 6E (to
          * xmm) / 7E (to gpr). The xmm is always the ModRM.reg field, the gpr
          * the r/m; movq sets REX.W (w==8), movd does not (w==4). */
         if ((src.kind == X64_ASM_OP_REG && dst.kind == X64_ASM_OP_XMM) ||
             (src.kind == X64_ASM_OP_XMM && dst.kind == X64_ASM_OP_REG)) {
           int to_xmm = (dst.kind == X64_ASM_OP_XMM);
           u32 xmm = to_xmm ? dst.reg : src.reg;
           u32 gpr = to_xmm ? src.reg : dst.reg;
           emit_sse_rr_w(mc, X64_OPSIZE_PFX, to_xmm ? 0x6Eu : 0x7Eu,
                         width_to_w(w), xmm, gpr);
           return;
         }
         asm_driver_panic(d, "x64 asm: mov form");
       }
     }
   }
 
   desc = find_mnemonic_row(&info);
   if (!desc) asm_driver_panic(d, "x64 asm: unknown mnemonic");
 
   /* If the user wrote an indirect branch (`*%reg`), prefer the BR_RM row
    * over the rel32 row that may sort first in the table. */
   if (desc->fmt == X64_FMT_CALL_REL32 || desc->fmt == X64_FMT_JMP_REL32) {
     AsmTok t = asm_driver_peek(d);
     if (asm_driver_tok_is_punct(t, '*')) {
       for (u32 i = 0; i < x64_insn_table_n; ++i) {
         const X64InsnDesc* dr = &x64_insn_table[i];
         if (dr->fmt != X64_FMT_BR_RM) continue;
         if (!slice_eq(dr->mnemonic, (Slice){{info.base}, info.base_len}))
           continue;
         desc = dr;
         break;
       }
     }
   }
 
   ctx.d = d;
   ctx.mc = mc;
   ctx.desc = desc;
   ctx.width = width_from_info(&info, desc);
   ctx.cc = info.cc;
   parse_and_emit_for_format(&ctx);
 }
 
 static void x64_arch_asm_destroy(ArchAsm* base) {
   x64_asm_close((X64Asm*)base);
 }
 
 X64Asm* x64_asm_open(Compiler* c) {
   X64Asm* a = arena_new(c->tu, X64Asm);
   memset(a, 0, sizeof *a);
   a->base.insn = x64_arch_asm_insn;
   a->base.destroy = x64_arch_asm_destroy;
   a->c = c;
   return a;
 }
 
 void x64_asm_close(X64Asm* a) { (void)a; }
 
 /* ---- cc -S symbolization hooks (printer <-> this parser) ------------------
  *
  * Inverse of the operand-syntax this parser accepts (parse_rel32_branch,
  * x64_parse_reloc_suffix): how the printer spells a relocated x64 operand so it
  * re-assembles. x64 relocs store addend-4 (rel32 bias), so addend_bias=4 makes
  * the printed offset the symbol offset. R_PC32 covers BOTH a branch target and
  * a RIP-relative lea/mov, so surgery is chosen from the operand text by the
  * printer (an `(%rip)` operand uses RIP surgery); we just supply the modifier.
  * Calls (R_X64_PLT32) print as a bare symbol — both kit-as (call default) and
  * clang resolve a same-TU callee, so execution matches regardless of the exact
  * reloc kind each assembler picks. */
 static int x64_reloc_operand(u16 kind, KitObjFmt fmt, ArchRelocOperand* out) {
   const char* suffix;
   (void)fmt; /* x64 cc -S cross-targets ELF; one spelling */
   switch (kind) {
     case R_PC32:      /* jmp/jcc target, or RIP-relative lea/mov */
     case R_X64_PLT32: /* call target -> bare symbol */
       suffix = "";
       break;
     case R_X64_GOTPCREL:
     case R_X64_GOTPCRELX:
     case R_X64_REX_GOTPCRELX:
       suffix = "@GOTPCREL"; /* RIP-relative GOT load */
       break;
     default:
       return 0; /* data (R_ABS*) via emit_data_range; TLS/etc. unsymbolized */
   }
   out->surg =
       ARCH_RELOC_SURG_TAIL; /* promoted to RIP by the printer if (%rip) */
   out->prefix = "";
   out->suffix = suffix;
   out->addend_bias = 4;
   return 1;
 }
 
 /* Intra-section local branches whose target codegen resolved in place (no
  * relocation): jmp and the Jcc family. Excludes call (always relocated) and
  * indirect/register-form jumps (no numeric target to relabel). */
 static int x64_is_local_branch(KitSlice m) {
   static const char* const br[] = {
       "jmp",
       "jo",
       "jno",
       "jb",
       "jae",
       "je",
       "jne",
       "jbe",
       "ja",
       "js",
       "jns",
       "jp",
       "jnp",
       "jl",
       "jge",
       "jle",
       "jg",
       /* aliases the disassembler may not emit but harmless to accept */
       "jz",
       "jnz",
       "jc",
       "jnc",
   };
   u32 i;
   for (i = 0; i < sizeof br / sizeof br[0]; ++i) {
     size_t n = strlen(br[i]);
     if (m.len == (u32)n && memcmp(m.s, br[i], n) == 0) return 1;
   }
   return 0;
 }
 
 const ArchAsmOps x64_asm_ops = {
     .reloc_operand = x64_reloc_operand,
     .is_local_branch = x64_is_local_branch,
 };
 
 ArchAsm* x64_arch_asm_new(Compiler* c) { return &x64_asm_open(c)->base; }
 
 void x64_inline_bind(X64Asm* a, const AsmConstraint* outs, u32 nout,
                      Operand* out_ops, const AsmConstraint* ins, u32 nin,
                      const Operand* in_ops, const Sym* clobbers, u32 nclob) {
   a->outs = outs;
   a->out_ops = out_ops;
   a->ins = ins;
   a->in_ops = in_ops;
   a->clobbers = clobbers;
   a->nout = nout;
   a->nin = nin;
   a->nclob = nclob;
 }
 
 #define X64_INLINE_LINE_CAP 1024
 
 _Noreturn static void inline_panic(X64Asm* a, const char* msg) {
   SrcLoc loc = {0, 0, 0};
   compiler_panic(a->c, loc, "x64 inline asm: %.*s",
                  SLICE_ARG(slice_from_cstr(msg)));
 }
 
 /* Width selector for x64_reg_spelling: matches the operand-modifier
  * forms recognised by the template walker. */
 #define X64_REG_WIDTH_64 0
 #define X64_REG_WIDTH_32 1
 #define X64_REG_WIDTH_8 2
 #define X64_REG_WIDTH_16 3
 #define X64_REG_WIDTH_H8 4
 
 static void render_xmm(StrBuf* sb, u32 reg) {
   strbuf_putc(sb, '%');
   strbuf_puts(sb, "xmm");
   reg &= 15u;
   if (reg >= 10u) strbuf_putc(sb, (char)('0' + (reg / 10u)));
   strbuf_putc(sb, (char)('0' + (reg % 10u)));
 }
 
 static const char* x64_reg_spelling(u32 reg, int width) {
   static const char* r64[16] = {
       "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi",
       "r8",  "r9",  "r10", "r11", "r12", "r13", "r14", "r15",
   };
   static const char* r32[16] = {
       "eax", "ecx", "edx",  "ebx",  "esp",  "ebp",  "esi",  "edi",
       "r8d", "r9d", "r10d", "r11d", "r12d", "r13d", "r14d", "r15d",
   };
   static const char* r8[16] = {
       "al",  "cl",  "dl",   "bl",   "spl",  "bpl",  "sil",  "dil",
       "r8b", "r9b", "r10b", "r11b", "r12b", "r13b", "r14b", "r15b",
   };
   static const char* r16[16] = {
       "ax",  "cx",  "dx",   "bx",   "sp",   "bp",   "si",   "di",
       "r8w", "r9w", "r10w", "r11w", "r12w", "r13w", "r14w", "r15w",
   };
   static const char* rh8[4] = {"ah", "ch", "dh", "bh"};
   if (width == X64_REG_WIDTH_H8) return reg < 4u ? rh8[reg] : NULL;
   if (width == X64_REG_WIDTH_16) return r16[reg & 15u];
   if (width == X64_REG_WIDTH_8) return r8[reg & 15u];
   if (width == X64_REG_WIDTH_32) return r32[reg & 15u];
   return r64[reg & 15u];
 }
 
 static int x64_type_prefers_32(KitCgTypeId type) {
   if (type == 0) return 0;
   return !type_is_64(type);
 }
 
 static void render_reg(StrBuf* sb, u32 reg, int width) {
   const char* name = x64_reg_spelling(reg, width);
   strbuf_putc(sb, '%');
   if (name) strbuf_puts(sb, name);
 }
 
 static void render_imm(StrBuf* sb, i64 v) {
   strbuf_putc(sb, '$');
   strbuf_put_i64(sb, v);
 }
 
 static void render_indirect(StrBuf* sb, Reg base, i32 ofs) {
   if (ofs) strbuf_put_i64(sb, (i64)ofs);
   strbuf_putc(sb, '(');
   render_reg(sb, (u32)base, X64_REG_WIDTH_64);
   strbuf_putc(sb, ')');
 }
 
 /* Operand-modifier forms used by the template walker. */
 #define X64_FORM_DEFAULT 0
 #define X64_FORM_W 1 /* %w — 16-bit */
 #define X64_FORM_X 2 /* %x — 64-bit */
 #define X64_FORM_A 3 /* %a — address / memory */
 #define X64_FORM_B 4 /* %b — 8-bit (byte) register */
 #define X64_FORM_K 5 /* %k — 32-bit */
 #define X64_FORM_H 6 /* %h — high 8-bit register (a/c/d/b only) */
 
 static char x64_size_suffix_for_operand(X64Asm* a, u32 idx) {
   u32 ntot = a->nout + a->nin;
   const Operand* op;
   u32 size;
   if (idx >= ntot) inline_panic(a, "operand index out of range");
   op = (idx < a->nout) ? &a->out_ops[idx] : &a->in_ops[idx - a->nout];
   if (op->type)
     size = type_byte_size(op->type);
   else if (op->kind == OPK_IMM)
     size = 4;
   else
     size = 8;
   switch (size) {
     case 1:
       return 'b';
     case 2:
       return 'w';
     case 4:
       return 'l';
     case 8:
       return 'q';
     default:
       inline_panic(a, "%z requires a scalar 1/2/4/8-byte operand");
   }
 }
 
 static void render_operand(X64Asm* a, StrBuf* sb, u32 idx, int form) {
   u32 ntot = a->nout + a->nin;
   const Operand* op;
   if (idx >= ntot) inline_panic(a, "operand index out of range");
   op = (idx < a->nout) ? &a->out_ops[idx] : &a->in_ops[idx - a->nout];
   if (form == X64_FORM_A) {
     if (op->kind != OPK_INDIRECT) inline_panic(a, "%a on non-memory operand");
     if (op->v.ind.index != REG_NONE)
       inline_panic(a, "inline asm: indexed addressing not supported");
     render_indirect(sb, op->v.ind.base, op->v.ind.ofs);
     return;
   }
   if ((form == X64_FORM_B || form == X64_FORM_H) &&
       op->kind != X64_INLINE_OPK_REG) {
     inline_panic(a, "byte-register modifier requires a register operand");
   }
   if (op->kind == X64_INLINE_OPK_REG) {
     int width;
     if (op->pad[0] == X64_INLINE_OPCLS_FP) {
       render_xmm(sb, (u32)op->v.local);
       return;
     }
     if (form == X64_FORM_B)
       width = X64_REG_WIDTH_8;
     else if (form == X64_FORM_H) {
       if (op->v.local > X64_RBX) {
         inline_panic(a, "%h modifier requires ax/cx/dx/bx register");
       }
       width = X64_REG_WIDTH_H8;
     } else if (form == X64_FORM_W)
       width = X64_REG_WIDTH_16;
     else if (form == X64_FORM_K)
       width = X64_REG_WIDTH_32;
     else if (form == X64_FORM_X)
       width = X64_REG_WIDTH_64;
     else
       width =
           x64_type_prefers_32(op->type) ? X64_REG_WIDTH_32 : X64_REG_WIDTH_64;
     render_reg(sb, (u32)op->v.local, width);
     return;
   }
   if (op->kind == OPK_IMM) {
     render_imm(sb, op->v.imm);
     return;
   }
   if (op->kind == OPK_INDIRECT) {
     if (op->v.ind.index != REG_NONE)
       inline_panic(a, "inline asm: indexed addressing not supported");
     render_indirect(sb, op->v.ind.base, op->v.ind.ofs);
     return;
   }
   inline_panic(a, "unsupported operand kind");
 }
 
 static u32 find_named_operand(X64Asm* a, const char* name, size_t len) {
   Sym needle = pool_intern_slice(a->c->global, (Slice){.s = name, .len = len});
   u32 i;
   for (i = 0; i < a->nout; ++i) {
     if (a->outs[i].name == needle) return i;
   }
   for (i = 0; i < a->nin; ++i) {
     if (a->ins[i].name == needle) return a->nout + i;
   }
   inline_panic(a, "%[name] does not match any constraint");
 }
 
 static void run_one_line(X64Asm* a, MCEmitter* mc, const char* text,
                          size_t len) {
   size_t i;
   AsmLexer* lx;
   AsmDriver* d;
   AsmTok t;
   for (i = 0; i < len; ++i) {
     if (text[i] != ' ' && text[i] != '\t') break;
   }
   if (i == len) return;
   lx = asm_lex_open_mem(a->c, "<inline-asm>", text, len);
   d = asm_driver_open_inline(a->c, mc, lx);
   t = asm_driver_peek(d);
   while (t.kind == ASM_TOK_NEWLINE || t.kind == ASM_TOK_HASH) {
     (void)asm_driver_next(d);
     if (t.kind == ASM_TOK_HASH) {
       while (!asm_driver_at_eol(d)) (void)asm_driver_next(d);
     }
     t = asm_driver_peek(d);
   }
   if (t.kind == ASM_TOK_EOF) {
     asm_driver_close_inline(d);
     asm_lex_close(lx);
     return;
   }
   if (t.kind != ASM_TOK_IDENT) inline_panic(a, "expected mnemonic");
   (void)asm_driver_next(d);
   x64_arch_asm_insn(&a->base, d, t.v.ident);
   asm_driver_close_inline(d);
   asm_lex_close(lx);
 }
 
 static void render_and_run_line(X64Asm* a, MCEmitter* mc, StrBuf* sb,
                                 const char* start, const char* end) {
   strbuf_reset(sb);
   for (const char* p = start; p < end; ++p) {
     char c = *p;
     char n;
     int form = 0;
     if (c != '%') {
       strbuf_putc(sb, c);
       continue;
     }
     if (p + 1 >= end) inline_panic(a, "trailing '%' in template");
     n = *(p + 1);
     if (n == '%') {
       strbuf_putc(sb, '%');
       ++p;
       continue;
     }
     if (n == 'w' || n == 'x' || n == 'a' || n == 'b' || n == 'k' || n == 'h' ||
         n == 'z') {
       form = (n == 'w')   ? X64_FORM_W
              : (n == 'x') ? X64_FORM_X
              : (n == 'a') ? X64_FORM_A
              : (n == 'b') ? X64_FORM_B
              : (n == 'k') ? X64_FORM_K
              : (n == 'h') ? X64_FORM_H
                           : -1;
       ++p;
       if (p + 1 >= end) inline_panic(a, "trailing '%' modifier");
       n = *(p + 1);
     }
     if (n == '[') {
       const char* nbeg = p + 2;
       const char* nend = nbeg;
       u32 idx;
       while (nend < end && *nend != ']') ++nend;
       if (nend == end) inline_panic(a, "unterminated %[name]");
       idx = find_named_operand(a, nbeg, (size_t)(nend - nbeg));
       p = nend;
       if (form == -1)
         strbuf_putc(sb, x64_size_suffix_for_operand(a, idx));
       else
         render_operand(a, sb, idx, form);
       continue;
     }
     if (n < '0' || n > '9') inline_panic(a, "expected digit after '%'");
     {
       u32 idx = (u32)(n - '0');
       ++p;
       if (p + 1 < end && *(p + 1) >= '0' && *(p + 1) <= '9') {
         idx = idx * 10u + (u32)(*(p + 1) - '0');
         ++p;
       }
       if (form == -1)
         strbuf_putc(sb, x64_size_suffix_for_operand(a, idx));
       else
         render_operand(a, sb, idx, form);
     }
   }
   if (sb->truncated) inline_panic(a, "inline asm line buffer overflow");
   run_one_line(a, mc, strbuf_cstr(sb), strbuf_len(sb));
 }
 
 void x64_asm_run_template(X64Asm* a, MCEmitter* mc, const char* tmpl) {
   char buf[X64_INLINE_LINE_CAP];
   StrBuf sb;
   const char* line_start;
   int bracket = 0;
   char quote = 0;
   if (!tmpl || !*tmpl) return;
   strbuf_init(&sb, buf, sizeof buf);
   line_start = tmpl;
   for (const char* p = tmpl;; ++p) {
     char c = *p;
     if (c == '\0') {
       render_and_run_line(a, mc, &sb, line_start, p);
       break;
     }
     if (quote) {
       if (c == '\\' && *(p + 1)) {
         ++p;
         continue;
       }
       if (c == quote) quote = 0;
       continue;
     }
     if (c == '"' || c == '\'') {
       quote = c;
       continue;
     }
     if (c == '[') {
       ++bracket;
       continue;
     }
     if (c == ']') {
       if (bracket) --bracket;
       continue;
     }
     if (bracket == 0 && (c == '\n' || c == ';')) {
       render_and_run_line(a, mc, &sb, line_start, p);
       line_start = p + 1;
     }
   }
 }