kit

mc.c (32281B)

---

 /* Generic MCEmitter implementation.
  *
  * MCEmitter sits between CGTarget (or asm_parse) and ObjBuilder. It owns
  * the current section, byte position, machine label table, and forwards
  * relocations / source-location stamps. Encoding is the caller's job —
  * MCEmitter writes whatever bytes it's handed.
  *
  * One MCEmitter serves every supported arch. Label fixup encoding delegates
  * through ArchImpl so MCEmitter owns label bookkeeping only.
  *
  * MCLabel handling: ids are 1-based (0 = MC_LABEL_NONE). Each label
  * carries either a placement (sec_id, offset) or a list of pending
  * fixups for forward references. emit_label_ref records a fixup; on
  * label_place, all pending fixups for that label are applied. Fixup
  * application uses RelocKind to choose how to encode the resolved
  * displacement into the already-emitted bytes.
  *
  * For v1 we support these label-ref reloc kinds for intra-section
  * fixups:
  *   R_PC32          — write 32-bit signed displacement at fixup ofs
  *   R_AARCH64_CALL26 — 26-bit imm26 << 2 in the BL instruction
  *   R_AARCH64_JUMP26 — same encoding as CALL26 (B vs BL only differs
  *                       in the parent opcode the caller already emitted)
  *
  * Anything else for a label_ref panics; cross-section references go
  * through emit_reloc against an ObjSymId instead. */
 
 #include <string.h>
 
 #include "arch/arch.h"
 #include "core/arena.h"
 #include "core/buf.h"
 #include "core/bytes.h"
 #include "core/heap.h"
 #include "core/pool.h"
 #include "core/strbuf.h"
 #include "debug/dwarf_defs.h"
 #include "obj/obj.h"
 
 typedef struct MCFixup {
   u32 sec_id;
   u32 offset;
   u32 width; /* bytes the encoding occupies */
   RelocKind kind;
   i64 addend;
   struct MCFixup* next;
 } MCFixup;
 
 typedef struct MCDataLabelRef {
   /* Where in the data section to write the relocation. */
   u32 data_sec;
   u32 data_offset;
   RelocKind kind;
   u32 width;
   i64 extra_addend;
   struct MCDataLabelRef* next;
   /* func_sym + func_start are read from MCEmitter at label_place time
    * (when the label's offset becomes known). Under -O1 the queue-time
    * call comes during opt IR recording — before any backend func_begin
    * has set cur_func_* — so capturing them here would be wrong. The
    * label is always placed inside its owning function's emit, so the
    * MCEmitter's current function tracks the right symbol at that
    * moment. */
 } MCDataLabelRef;
 
 typedef struct MCLabelInfo {
   u8 placed;
   u8 pad[3];
   u32 sec_id;
   u32 offset;
   MCFixup* pending;
   MCDataLabelRef* pending_data;
   /* Lazily-minted SB_LOCAL symbol for this label, for code-location
    * references that must survive a re-encoding assembler: switch jump-table
    * entries (.quad <sym>) and `&&label` address-takes (a PC-relative reloc
    * against <sym>). OBJ_SYM_NONE until first requested via mc_label_symbol;
    * defined at the label's offset in mc_label_place (forward-ref safe). */
   ObjSymId block_sym;
 } MCLabelInfo;
 
 /* ---- CFI buffering (.eh_frame producer) ----
  *
  * Each cfi_startproc opens a new FDE record; the per-arch backend then
  * calls cfi_def_cfa / cfi_offset as the prologue is laid
  * down. Each directive snapshots either the current section offset or the
  * override set by cfi_set_next_pc_offset — which is STICKY until cfi_endproc
  * (used by backends that emit the whole CFI batch in func_end, after the
  * epilogue, but want every rule pinned to the post-prologue PC). The
  * .eh_frame section is synthesised at mc_emit_eh_frame() time. */
 typedef enum CfiOpKind {
   CFI_OP_DEF_CFA,
   CFI_OP_OFFSET,
 } CfiOpKind;
 
 typedef struct CfiDirective {
   u32 pc_offset; /* offset within the function from func_start */
   u8 kind;       /* CfiOpKind */
   u8 pad[3];
   u32 reg;
   i32 imm;
 } CfiDirective;
 
 typedef struct CfiFde {
   ObjSymId func_sym;
   u32 func_section;
   u32 func_start;
   u32 func_end;
   u32 dir_start; /* index of this FDE's first directive in MCImpl.dirs */
   u32 ndir;
 } CfiFde;
 
 typedef struct MCImpl {
   MCEmitter base;
   /* Per-function scratch: labels, forward-reference fixups, and data-label
    * refs. All are resolved within the function that produced them (immediately
    * for a backward ref, at mc_label_place for a forward one); cross-function
    * references go through per-label SB_LOCAL object symbols, and the emitted
    * bytes/relocations live in the ObjBuilder, not here. So this arena is reset
    * at every mc_begin_function — it grows only to the largest single function's
    * scratch instead of accumulating the whole TU's in c->tu. The whole-object
    * CFI vectors (fdes/dirs) are heap-realloc'd separately, not from here. */
   Arena func_arena;
   /* `loc` lives on MCEmitter base now (so per-arch emit hooks can read it
    * to feed debug_emit_row). Use base.loc through impl_of(...)->base.loc
    * or directly mc->base.loc. */
   MCLabelInfo* labels; /* index 0 unused (MC_LABEL_NONE) */
   u32 nlabels;
   u32 cap;
   CfiFde* fdes;
   u32 nfdes;
   u32 fdes_cap;
   i32 cur_fde;
   /* All FDEs' CFI directives share one growable vector; each FDE owns the
    * contiguous range [dir_start, dir_start+ndir). Single-pass emission keeps a
    * function's directives contiguous, so a function never costs its own heap
    * block -- the vector doubles a handful of times for the whole object. */
   CfiDirective* dirs;
   u32 ndirs;
   u32 dirs_cap;
   u8 eh_frame_emitted;
   u8 has_pc_override;
   u8 pad_cfi[2];
   u32 pc_override;
 } MCImpl;
 
 /* ---- helpers ---- */
 
 static MCImpl* impl_of(MCEmitter* m) { return (MCImpl*)m; }
 
 static void labels_grow(MCImpl* mc, u32 want) {
   if (want <= mc->cap) return;
   u32 ncap = mc->cap ? mc->cap * 2 : 16;
   while (ncap < want) ncap *= 2;
   MCLabelInfo* nbuf = arena_array(&mc->func_arena, MCLabelInfo, ncap);
   if (mc->labels) memcpy(nbuf, mc->labels, sizeof(MCLabelInfo) * mc->nlabels);
   /* The grown tail is left uninitialized: mc_label_new fully assigns every
    * field of the one slot it hands out before any consumer indexes it, and
    * nothing ever reads labels[i] for i >= nlabels (every access guards on
    * id < nlabels and rejects MC_LABEL_NONE). */
   mc->labels = nbuf;
   mc->cap = ncap;
 }
 
 static void emit_label_data_reloc_now(MCImpl* mc, MCLabel label,
                                       const MCDataLabelRef* r) {
   /* Reference the label's per-block local symbol (its value IS the label's
    * offset) rather than the enclosing function symbol + a baked byte offset.
    * That makes the entry genuinely relocatable: a third-party assembler that
    * re-encodes the function to different instruction lengths still resolves it
    * to the right address (a fixed fn+offset would point into the wrong
    * instruction). */
   ObjSymId sym = mc_label_symbol(&mc->base, label);
   i64 addend = r->extra_addend;
   u8 bytes[8];
   u32 i;
   int big_endian = mc->base.c->target.big_endian;
   /* Patch the inline addend (Mach-O ARM64_RELOC_UNSIGNED reads only the inline
    * value) and also pass it in the reloc record (ELF RELA / the JIT linker's
    * link_reloc_apply, where the inline gets overwritten by S + A). Both paths
    * converge on sym + addend at runtime. */
   memset(bytes, 0, sizeof bytes);
   for (i = 0; i < r->width && i < sizeof bytes; ++i) {
     u32 shift = big_endian ? (r->width - 1u - i) * 8u : i * 8u;
     bytes[i] = (u8)((u64)addend >> shift);
   }
   obj_patch(mc->base.obj, r->data_sec, r->data_offset, bytes, r->width);
   mc_emit_reloc_at(&mc->base, r->data_sec, r->data_offset, r->kind, sym,
                          addend, /*explicit_addend=*/1, /*pair=*/0);
 }
 
 static void apply_fixup(MCImpl* mc, const MCFixup* fx, u32 target_offset) {
   /* signed displacement from end-of-instruction position to target. */
   ArchLabelFixup desc;
   const ArchImpl* arch;
 
   memset(&desc, 0, sizeof desc);
   desc.obj = mc->base.obj;
   desc.sec_id = fx->sec_id;
   desc.offset = fx->offset;
   desc.width = fx->width;
   desc.kind = fx->kind;
   desc.disp = (i64)target_offset - (i64)fx->offset + fx->addend;
   desc.cur_func_sym = mc->base.cur_func_sym;
   desc.cur_func_start = mc->base.cur_func_start;
 
   arch = arch_for_compiler(mc->base.c);
   if (!arch || !arch->apply_label_fixup ||
       arch->apply_label_fixup(mc->base.c, &desc) != 0) {
     compiler_panic(mc->base.c, mc->base.loc,
                    "MCEmitter: unsupported label-ref reloc kind %d",
                    (int)fx->kind);
   }
 }
 
 /* Lazily mint (and return) a per-label SB_LOCAL symbol defined at the label's
  * placement, for code-location references an encoding-divergent assembler must
  * be able to recompute: switch jump-table entries and `&&label` address-takes
  * relocate against it instead of baking a fixed offset. Created undefined if
  * the label is not yet placed (a forward reference) and defined in
  * mc_label_place; defined immediately otherwise. The name is per-object-unique
  * (MCLabel ids are monotonic within a TU). */
 ObjSymId mc_label_symbol(MCEmitter* m, MCLabel id) {
   MCImpl* mc = impl_of(m);
   MCLabelInfo* li;
   char buf[40];
   StrBuf sb;
   Sym name;
   if (id == MC_LABEL_NONE || id >= mc->nlabels) {
     compiler_panic(m->c, m->loc, "MCEmitter: bad label %u for symbol",
                    (unsigned)id);
   }
   li = &mc->labels[id];
   if (li->block_sym != OBJ_SYM_NONE) return li->block_sym;
   strbuf_init(&sb, buf, sizeof buf);
   strbuf_put_slice(&sb, SLICE_LIT(".Lcfblk."));
   strbuf_put_u64(&sb, (u64)id);
   name = pool_intern_slice(m->c->global, strbuf_slice(&sb));
   li->block_sym = obj_symbol(m->obj, name, SB_LOCAL, SK_NOTYPE,
                              li->placed ? li->sec_id : OBJ_SEC_NONE,
                              li->placed ? (u64)li->offset : 0u, 0);
   /* A block-address symbol is a relocation anchor within its function, not a
    * separately collectible/reorderable content unit. This distinction is
    * material on Mach-O, where every ordinary symbol under
    * MH_SUBSECTIONS_VIA_SYMBOLS otherwise starts a new atom. */
   obj_symbol_set_atom_subordinate(m->obj, li->block_sym, 1);
   return li->block_sym;
 }
 
 /* ---- emission ops (called directly by the arch backends) ---- */
 
 /* (Re)point the typed-store cursor at the active section's tail chunk write
  * frontier. Called after set_section and after any non-cursor write path
  * (mc_emit_bytes / fills) that may have grown or replaced the tail chunk. The
  * cursor is left NULL/empty (forcing the slow path) when there is no PROGBITS
  * buffer or the tail chunk is full or absent — buf_write_slow then mints/grows
  * the chunk and the next repoint picks it up. The invariant maintained
  * everywhere: cur == cur_chunk->data + cur_chunk->used (when cur_chunk != NULL),
  * with cur_chunk->used / cur_bytes->total kept coherent by mc_emit32. */
 static void mc_cursor_repoint(MCEmitter* m) {
   Buf* b = m->cur_bytes;
   BufChunk* t = b ? b->tail : NULL;
   if (t) {
     m->cur_chunk = t;
     m->cur = t->data + t->used;
     m->cur_end = t->data + t->cap;
   } else {
     m->cur_chunk = NULL;
     m->cur = NULL;
     m->cur_end = NULL;
   }
 }
 
 void mc_emit_bytes(MCEmitter* m, const u8* data, size_t n) {
   /* Fast path: append straight to the cached section buffer (inlined
    * buf_write). cur_bytes is NULL for NOBITS/.bss/none, where obj_write does
    * the bss_size accounting instead. */
   if (m->cur_bytes)
     buf_write(m->cur_bytes, data, n);
   else
     obj_write(m->obj, m->section_id, data, n);
   /* buf_write may have grown/replaced the tail chunk and bumped used/total;
    * re-point the typed-store cursor so the next mc_emit32 stays coherent. */
   mc_cursor_repoint(m);
 }
 
 /* Out-of-line slow path for mc_emit32: the tail chunk is full, absent, or the
  * section is NOBITS. Route through the normal byte sink (which keeps used/total
  * coherent and re-points the cursor), so the fast path can branch on a simple
  * pointer-bounds test. */
 static void mc_emit32_slow(MCEmitter* m, u32 word) {
   u8 b[4];
   wr_u32_le(b, word);
   mc_emit_bytes(m, b, sizeof b);
 }
 
 void mc_emit32(MCEmitter* m, u32 word) {
   /* Typed-store fast path: one const-width 4-byte store through the cursor,
    * then bump the chunk/buffer lengths to keep the Buf coherent. No staging
    * buffer, no out-of-line memcpy call. */
   u8* cur = m->cur;
   if (cur && cur + 4 <= m->cur_end) {
     wr_u32_le(cur, word);
     m->cur = cur + 4;
     m->cur_chunk->used += 4u;
     m->cur_bytes->total += 4u;
   } else {
     mc_emit32_slow(m, word);
   }
 }
 
 u32 mc_pos(MCEmitter* m) { return obj_pos(m->obj, m->section_id); }
 
 void mc_set_loc(MCEmitter* m, SrcLoc loc) { m->loc = loc; }
 
 void mc_set_section(MCEmitter* m, u32 section_id) {
   m->section_id = section_id;
   /* Cache the active section's byte buffer so the hot emit path avoids the
    * per-instruction Sections_at deref + nobits branch. NULL for NOBITS/.bss
    * (or none): emit then falls back to obj_write for bss_size accounting. */
   m->cur_bytes = obj_section_bytes(m->obj, section_id);
   mc_cursor_repoint(m);
 }
 
 MCLabel mc_label_new(MCEmitter* m) {
   MCImpl* mc = impl_of(m);
   if (mc->nlabels == 0) {
     labels_grow(mc, 1);
     mc->nlabels = 1;
   } /* skip 0 */
   labels_grow(mc, mc->nlabels + 1);
   u32 id = mc->nlabels++;
   MCLabelInfo* li = &mc->labels[id];
   li->placed = 0;
   li->sec_id = 0;
   li->offset = 0;
   li->pending = NULL;
   li->pending_data = NULL;
   li->block_sym = OBJ_SYM_NONE;
   return (MCLabel)id;
 }
 
 void mc_label_place(MCEmitter* m, MCLabel id) {
   MCImpl* mc = impl_of(m);
   if (id == MC_LABEL_NONE || id >= mc->nlabels) {
     compiler_panic(m->c, mc->base.loc, "MCEmitter: bad label %u", (unsigned)id);
   }
   MCLabelInfo* li = &mc->labels[id];
   if (li->placed) {
     compiler_panic(m->c, mc->base.loc, "MCEmitter: label %u placed twice",
                    (unsigned)id);
   }
   li->placed = 1;
   li->sec_id = m->section_id;
   li->offset = obj_pos(m->obj, m->section_id);
   /* Define the lazily-minted block symbol (if any) now that the offset is
    * known — resolves the forward-reference case for jump-table / &&label
    * relocations emitted before the label was placed. */
   if (li->block_sym != OBJ_SYM_NONE)
     obj_symbol_define(m->obj, li->block_sym, li->sec_id, (u64)li->offset, 0);
   /* Apply pending intra-section fixups. */
   for (MCFixup* fx = li->pending; fx; fx = fx->next) {
     apply_fixup(mc, fx, li->offset);
   }
   li->pending = NULL;
   /* Resolve any deferred data-section relocations referencing this label.
    * MCEmitter's cur_func_sym/cur_func_start track the function whose
    * body is currently being emitted; the label is always placed inside
    * its owning function's emit, so the active function context matches. */
   for (MCDataLabelRef* r = li->pending_data; r; r = r->next) {
     emit_label_data_reloc_now(mc, id, r);
   }
   li->pending_data = NULL;
 }
 
 void mc_emit_fill(MCEmitter* m, size_t n, u8 byte) {
   u8 buf[64];
   memset(buf, byte, sizeof buf);
   while (n > 0) {
     size_t k = n < sizeof buf ? n : sizeof buf;
     obj_write(m->obj, m->section_id, buf, k);
     n -= k;
   }
   /* obj_write grew the tail directly (bypassing the typed-store cursor);
    * re-point so a following mc_emit32 stays coherent. */
   mc_cursor_repoint(m);
 }
 
 void mc_emit_align(MCEmitter* m, u32 align, u8 fill) {
   const Section* s;
   if (align <= 1) return;
   s = obj_section_get(m->obj, m->section_id);
   if (s && align > s->align)
     obj_section_set_align(m->obj, m->section_id, align);
   u32 cur = obj_pos(m->obj, m->section_id);
   u32 misalign = cur & (align - 1);
   if (misalign == 0) return;
   mc_emit_fill(m, align - misalign, fill);
 }
 
 void mc_emit_reloc(MCEmitter* m, RelocKind k, ObjSymId sym, i64 addend) {
   obj_reloc(m->obj, m->section_id, obj_pos(m->obj, m->section_id), k, sym,
             addend);
 }
 
 void mc_emit_reloc_at(MCEmitter* m, u32 section_id, u32 offset,
                             RelocKind k, ObjSymId sym, i64 addend,
                             int explicit_addend, int pair) {
   obj_reloc_ex(m->obj, section_id, offset, k, sym, addend, explicit_addend,
                pair);
 }
 
 void mc_emit_label_ref(MCEmitter* m, MCLabel id, RelocKind kind,
                              u32 width, i64 addend) {
   MCImpl* mc = impl_of(m);
   if (id == MC_LABEL_NONE || id >= mc->nlabels) {
     compiler_panic(m->c, mc->base.loc, "MCEmitter: bad label %u", (unsigned)id);
   }
   MCLabelInfo* li = &mc->labels[id];
   MCFixup* fx = arena_new(&mc->func_arena, MCFixup);
   fx->sec_id = m->section_id;
   fx->offset = obj_pos(m->obj, m->section_id) -
                width; /* fixup site is the just-emitted insn */
   fx->width = width;
   fx->kind = kind;
   fx->addend = addend;
   fx->next = NULL;
   if (li->placed) {
     apply_fixup(mc, fx, li->offset);
   } else {
     fx->next = li->pending;
     li->pending = fx;
   }
 }
 
 void mc_emit_label_data_reloc(MCEmitter* m, u32 data_sec, u32 data_offset,
                                     MCLabel id, RelocKind kind, u32 width,
                                     i64 extra_addend) {
   MCImpl* mc = impl_of(m);
   MCLabelInfo* li;
   if (id == MC_LABEL_NONE || id >= mc->nlabels) {
     compiler_panic(m->c, m->loc, "MCEmitter: bad label %u", (unsigned)id);
   }
   li = &mc->labels[id];
   if (li->placed) {
     MCDataLabelRef tmp;
     tmp.data_sec = data_sec;
     tmp.data_offset = data_offset;
     tmp.kind = kind;
     tmp.width = width;
     tmp.extra_addend = extra_addend;
     tmp.next = NULL;
     emit_label_data_reloc_now(mc, id, &tmp);
     return;
   }
   {
     MCDataLabelRef* r = arena_new(&mc->func_arena, MCDataLabelRef);
     r->data_sec = data_sec;
     r->data_offset = data_offset;
     r->kind = kind;
     r->width = width;
     r->extra_addend = extra_addend;
     r->next = li->pending_data;
     li->pending_data = r;
   }
 }
 
 
 /* CFI: buffered for .eh_frame emission. Backend calls cfi_startproc to
  * open a per-function FDE record, then cfi_def_cfa / cfi_offset / ...
  * around the prologue; mc_emit_eh_frame builds the section at TU
  * finalize. */
 
 static void fde_push(MCImpl* mc, u8 kind, u32 reg, i32 imm) {
   CfiFde* fde;
   CfiDirective* d;
   Heap* heap;
   u32 pc_off;
   if (mc->cur_fde < 0) {
     compiler_panic(mc->base.c, mc->base.loc,
                    "MCEmitter: CFI directive outside cfi_startproc");
   }
   fde = &mc->fdes[mc->cur_fde];
   if (mc->base.section_id != fde->func_section) {
     compiler_panic(mc->base.c, mc->base.loc,
                    "MCEmitter: CFI directive in wrong section");
   }
   heap = mc->base.c->ctx->heap;
   /* The current FDE is always the last one, so its directives sit at the tail
    * of the shared vector (dir_start + ndir == ndirs); append there. */
   if (mc->ndirs == mc->dirs_cap) {
     u32 new_cap = mc->dirs_cap ? mc->dirs_cap * 2u : 64u;
     CfiDirective* nbuf = (CfiDirective*)heap->alloc(
         heap, sizeof(CfiDirective) * new_cap, _Alignof(CfiDirective));
     if (!nbuf) compiler_panic(mc->base.c, mc->base.loc, "MCEmitter: CFI OOM");
     if (mc->dirs) {
       memcpy(nbuf, mc->dirs, sizeof(CfiDirective) * mc->ndirs);
       heap->free(heap, mc->dirs, sizeof(CfiDirective) * mc->dirs_cap);
     }
     mc->dirs = nbuf;
     mc->dirs_cap = new_cap;
   }
   if (mc->has_pc_override) {
     /* Sticky until cfi_endproc: every directive in a func_end prologue batch
      * shares the post-prologue PC set once before cfi_def_cfa. (A one-shot
      * override only covered the first directive, so the saved-register and
      * return-address offset rules fell back to obj_pos() — the function END —
      * making mid-function unwind unable to recover them.) */
     pc_off = mc->pc_override;
   } else {
     pc_off = obj_pos(mc->base.obj, mc->base.section_id) - fde->func_start;
   }
   d = &mc->dirs[mc->ndirs++];
   fde->ndir++;
   d->pc_offset = pc_off;
   d->kind = kind;
   d->reg = reg;
   d->imm = imm;
 }
 
 void mc_cfi_startproc(MCEmitter* m) {
   MCImpl* mc = impl_of(m);
   Heap* heap = m->c->ctx->heap;
   if (mc->cur_fde >= 0) {
     compiler_panic(m->c, m->loc, "MCEmitter: nested cfi_startproc");
   }
   if (m->cur_func_sym == OBJ_SYM_NONE) {
     /* Backend must call mc_begin_function before cfi_startproc; tolerate
      * the no-op for stand-ins. */
     return;
   }
   if (mc->nfdes == mc->fdes_cap) {
     u32 new_cap = mc->fdes_cap ? mc->fdes_cap * 2u : 8u;
     CfiFde* nbuf =
         (CfiFde*)heap->alloc(heap, sizeof(CfiFde) * new_cap, _Alignof(CfiFde));
     if (!nbuf) compiler_panic(m->c, m->loc, "MCEmitter: CFI OOM");
     if (mc->fdes) {
       memcpy(nbuf, mc->fdes, sizeof(CfiFde) * mc->nfdes);
       heap->free(heap, mc->fdes, sizeof(CfiFde) * mc->fdes_cap);
     }
     mc->fdes = nbuf;
     mc->fdes_cap = new_cap;
   }
   mc->cur_fde = (i32)mc->nfdes;
   mc->has_pc_override = 0; /* no override carries across an FDE boundary */
   {
     CfiFde* fde = &mc->fdes[mc->nfdes++];
     fde->func_sym = m->cur_func_sym;
     fde->func_section = m->section_id;
     fde->func_start = obj_pos(m->obj, m->section_id);
     fde->func_end = fde->func_start;
     fde->dir_start = mc->ndirs;
     fde->ndir = 0;
   }
 }
 
 void mc_cfi_endproc(MCEmitter* m) {
   MCImpl* mc = impl_of(m);
   CfiFde* fde;
   if (mc->cur_fde < 0) return;
   fde = &mc->fdes[mc->cur_fde];
   fde->func_end = obj_pos(m->obj, m->section_id);
   mc->cur_fde = -1;
   mc->has_pc_override =
       0; /* the sticky prologue-PC override ends with the FDE */
 }
 
 void mc_cfi_def_cfa(MCEmitter* m, u32 r, i32 o) {
   MCImpl* mc = impl_of(m);
   if (mc->cur_fde < 0) return;
   fde_push(mc, CFI_OP_DEF_CFA, r, o);
 }
 void mc_cfi_offset(MCEmitter* m, u32 r, i32 o) {
   MCImpl* mc = impl_of(m);
   if (mc->cur_fde < 0) return;
   fde_push(mc, CFI_OP_OFFSET, r, o);
 }
 void mc_cfi_set_next_pc_offset(MCEmitter* m, u32 pc_offset) {
   MCImpl* mc = impl_of(m);
   mc->has_pc_override = 1;
   mc->pc_override = pc_offset;
 }
 
 /* ---- construction ---- */
 
 static void mc_cleanup(void* arg) { mc_free((MCEmitter*)arg); }
 
 MCEmitter* mc_new(Compiler* c, ObjBuilder* o) {
   MCImpl* mc = arena_new(c->tu, MCImpl);
   memset(mc, 0, sizeof *mc);
 
   MCEmitter* base = &mc->base;
   base->c = c;
   base->obj = o;
   base->section_id = OBJ_SEC_NONE;
   base->cur_func_sym = OBJ_SYM_NONE;
   base->cur_func_section = 0;
   base->cur_func_start = 0;
 
   arena_init(&mc->func_arena, c->ctx->heap, 0);
   mc->labels = NULL;
   mc->nlabels = 0;
   mc->cap = 0;
   mc->fdes = NULL;
   mc->nfdes = 0;
   mc->fdes_cap = 0;
   mc->cur_fde = -1;
   mc->dirs = NULL;
   mc->ndirs = 0;
   mc->dirs_cap = 0;
   mc->eh_frame_emitted = 0;
   mc->has_pc_override = 0;
   mc->pc_override = 0;
 
   compiler_defer(c, mc_cleanup, base);
   return base;
 }
 
 void mc_free(MCEmitter* m) {
   MCImpl* mc;
   Heap* heap;
   if (!m) return;
   mc = impl_of(m);
   /* Release any CFI directive buffers when the caller never invoked
    * mc_emit_eh_frame (e.g. test harness or early teardown). */
   if (!mc->eh_frame_emitted && mc->fdes) {
     heap = m->c->ctx->heap;
     if (mc->dirs) {
       heap->free(heap, mc->dirs, sizeof(CfiDirective) * mc->dirs_cap);
       mc->dirs = NULL;
       mc->dirs_cap = 0;
       mc->ndirs = 0;
     }
     heap->free(heap, mc->fdes, sizeof(CfiFde) * mc->fdes_cap);
     mc->fdes = NULL;
     mc->fdes_cap = 0;
     mc->nfdes = 0;
   }
   arena_fini(&mc->func_arena);
 }
 
 void mc_begin_function(MCEmitter* m, ObjSymId sym, u32 section_id,
                        u32 start_offset) {
   if (!m) return;
   /* Reclaim the previous function's label/fixup/dataref scratch (see the
    * func_arena note on MCImpl). The labels vector is re-grown from the reset
    * arena on demand, so drop the stale pointer/count here. */
   {
     MCImpl* mc = impl_of(m);
     arena_reset(&mc->func_arena);
     mc->labels = NULL;
     mc->nlabels = 0;
     mc->cap = 0;
   }
   m->cur_func_sym = sym;
   m->cur_func_section = section_id;
   m->cur_func_start = start_offset;
 }
 
 void mc_end_function(MCEmitter* m) {
   if (!m) return;
   m->cur_func_sym = OBJ_SYM_NONE;
   m->cur_func_section = 0;
   m->cur_func_start = 0;
 }
 
 /* ============================================================
  * .eh_frame emitter
  * ============================================================ */
 
 static void buf_uleb(Buf* b, u64 v) {
   u8 tmp[10];
   u32 n = 0;
   do {
     u8 byte = (u8)(v & 0x7fu);
     v >>= 7;
     if (v) byte |= 0x80u;
     tmp[n++] = byte;
   } while (v);
   buf_write(b, tmp, n);
 }
 
 static void buf_sleb(Buf* b, i64 v) {
   u8 tmp[10];
   u32 n = 0;
   int more = 1;
   while (more) {
     u8 byte = (u8)(v & 0x7fu);
     v >>= 7;
     if ((v == 0 && (byte & 0x40u) == 0) || (v == -1 && (byte & 0x40u) != 0)) {
       more = 0;
     } else {
       byte |= 0x80u;
     }
     tmp[n++] = byte;
   }
   buf_write(b, tmp, n);
 }
 
 static void buf_u8(Buf* b, u8 v) { buf_write(b, &v, 1); }
 
 static void buf_u32le(Buf* b, u32 v) {
   u8 t[4];
   t[0] = (u8)v;
   t[1] = (u8)(v >> 8);
   t[2] = (u8)(v >> 16);
   t[3] = (u8)(v >> 24);
   buf_write(b, t, 4);
 }
 
 static void buf_pad_to(Buf* b, u32 entry_start, u32 align) {
   u32 cur = buf_pos(b);
   u32 rel = cur - entry_start;
   u32 mis = rel & (align - 1u);
   u32 pad;
   if (mis == 0) return;
   pad = align - mis;
   while (pad--) buf_u8(b, 0);
 }
 
 static void encode_cfi_directive(Buf* prog, const CfiDirective* d, u32* cur_loc,
                                  i32 code_align, i32 data_align) {
   u32 delta = d->pc_offset - *cur_loc;
   if (delta) {
     u32 fac = (code_align > 0) ? (delta / (u32)code_align) : delta;
     if (fac < 0x40u) {
       buf_u8(prog, DW_CFA_advance_loc | (u8)fac);
     } else if (fac < 0x100u) {
       buf_u8(prog, DW_CFA_advance_loc1);
       buf_u8(prog, (u8)fac);
     } else if (fac < 0x10000u) {
       buf_u8(prog, DW_CFA_advance_loc2);
       buf_u8(prog, (u8)(fac & 0xff));
       buf_u8(prog, (u8)(fac >> 8));
     } else {
       buf_u8(prog, DW_CFA_advance_loc4);
       buf_u32le(prog, fac);
     }
     *cur_loc = d->pc_offset;
   }
   switch ((CfiOpKind)d->kind) {
     case CFI_OP_DEF_CFA:
       buf_u8(prog, DW_CFA_def_cfa);
       buf_uleb(prog, d->reg);
       buf_uleb(prog, (u64)(d->imm < 0 ? 0 : d->imm));
       break;
     case CFI_OP_OFFSET: {
       i64 fac;
       if (data_align == 0)
         fac = d->imm;
       else
         fac = (i64)d->imm / (i64)data_align;
       if (d->reg < 0x40u && fac >= 0) {
         buf_u8(prog, DW_CFA_offset | (u8)d->reg);
         buf_uleb(prog, (u64)fac);
       } else {
         buf_u8(prog, DW_CFA_offset_extended_sf);
         buf_uleb(prog, d->reg);
         buf_sleb(prog, fac);
       }
     } break;
   }
 }
 
 void mc_emit_eh_frame(MCEmitter* m) {
   MCImpl* mc;
   const ArchImpl* arch;
   Heap* heap;
   Buf body;
   ObjSecId eh_sec;
   Sym sec_name;
   u32 cie_offset_in_buf;
   u32 cie_len;
   u32 entry_start;
   u32 i;
   u8 fde_pe;
   if (!m) return;
   mc = impl_of(m);
   if (mc->eh_frame_emitted) return;
   if (mc->nfdes == 0) {
     mc->eh_frame_emitted = 1;
     return;
   }
   arch = arch_for_compiler(m->c);
   if (!arch || arch->cfi_return_addr_reg == 0u) {
     mc->eh_frame_emitted = 1;
     return;
   }
   /* Freestanding (bare-metal): emit no .eh_frame. kit marks .eh_frame
    * SF_ALLOC so a hosted unwinder can consume it, but a bare-metal link
    * (e.g. riscv32-none-elf) has no unwinder and would otherwise have to
    * /DISCARD/ the orphaned ALLOC section. emits_eh_frame is 0 exactly for
    * those freestanding targets and 1 for hosted output (linux/macos/windows/
    * freebsd/wasi), which is unaffected and byte-identical. */
   if (!m->c->target.emits_eh_frame) {
     heap = m->c->ctx->heap;
     if (mc->dirs) {
       heap->free(heap, mc->dirs, sizeof(CfiDirective) * mc->dirs_cap);
       mc->dirs = NULL;
       mc->dirs_cap = 0;
       mc->ndirs = 0;
     }
     if (mc->fdes) {
       heap->free(heap, mc->fdes, sizeof(CfiFde) * mc->fdes_cap);
       mc->fdes = NULL;
       mc->fdes_cap = 0;
       mc->nfdes = 0;
     }
     mc->eh_frame_emitted = 1;
     return;
   }
   heap = m->c->ctx->heap;
   fde_pe = (u8)(DW_EH_PE_pcrel | DW_EH_PE_sdata4);
 
   buf_init(&body, heap);
 
   /* CIE */
   cie_offset_in_buf = buf_pos(&body);
   buf_u32le(&body, 0);
   entry_start = buf_pos(&body);
   buf_u32le(&body, 0); /* CIE_id */
   buf_u8(&body, 1);    /* version */
   buf_u8(&body, 'z');
   buf_u8(&body, 'R');
   buf_u8(&body, 0);
   buf_uleb(&body, (u64)(u32)arch->cfi_code_align_factor);
   buf_sleb(&body, (i64)arch->cfi_data_align_factor);
   buf_uleb(&body, (u64)arch->cfi_return_addr_reg);
   buf_uleb(&body, 1);
   buf_u8(&body, fde_pe);
   buf_u8(&body, DW_CFA_def_cfa);
   buf_uleb(&body, (u64)arch->cfi_cfa_init_reg);
   buf_uleb(
       &body,
       (u64)(arch->cfi_cfa_init_offset < 0 ? 0 : arch->cfi_cfa_init_offset));
   buf_pad_to(&body, entry_start, 4u);
   cie_len = buf_pos(&body) - entry_start;
   {
     u8 lbytes[4];
     lbytes[0] = (u8)cie_len;
     lbytes[1] = (u8)(cie_len >> 8);
     lbytes[2] = (u8)(cie_len >> 16);
     lbytes[3] = (u8)(cie_len >> 24);
     buf_patch(&body, cie_offset_in_buf, lbytes, 4);
   }
 
   {
     u32* pc_slot_rels =
         (u32*)heap->alloc(heap, sizeof(u32) * mc->nfdes, _Alignof(u32));
     ObjSymId* fde_syms = (ObjSymId*)heap->alloc(
         heap, sizeof(ObjSymId) * mc->nfdes, _Alignof(ObjSymId));
     if (!pc_slot_rels || !fde_syms) {
       if (pc_slot_rels) heap->free(heap, pc_slot_rels, sizeof(u32) * mc->nfdes);
       if (fde_syms) heap->free(heap, fde_syms, sizeof(ObjSymId) * mc->nfdes);
       buf_fini(&body);
       compiler_panic(m->c, m->loc, "MCEmitter: CFI OOM");
     }
     for (i = 0; i < mc->nfdes; ++i) {
       const CfiFde* fde = &mc->fdes[i];
       u32 fde_offset_in_buf = buf_pos(&body);
       u32 fde_entry_start;
       u32 fde_len;
       u32 pc_slot;
       u32 cur_loc = 0;
       u32 j;
       i64 cie_back_off;
       buf_u32le(&body, 0);
       fde_entry_start = buf_pos(&body);
       cie_back_off = (i64)fde_entry_start - (i64)cie_offset_in_buf;
       buf_u32le(&body, (u32)cie_back_off);
       pc_slot = buf_pos(&body);
       pc_slot_rels[i] = pc_slot;
       fde_syms[i] = fde->func_sym;
       buf_u32le(&body, 0); /* initial_location (reloc) */
       buf_u32le(&body, fde->func_end - fde->func_start); /* range */
       buf_uleb(&body, 0);                                /* aug_data_len = 0 */
       for (j = 0; j < fde->ndir; ++j) {
         encode_cfi_directive(&body, &mc->dirs[fde->dir_start + j], &cur_loc,
                              arch->cfi_code_align_factor,
                              arch->cfi_data_align_factor);
       }
       buf_pad_to(&body, fde_entry_start, 4u);
       fde_len = buf_pos(&body) - fde_entry_start;
       {
         u8 lbytes[4];
         lbytes[0] = (u8)fde_len;
         lbytes[1] = (u8)(fde_len >> 8);
         lbytes[2] = (u8)(fde_len >> 16);
         lbytes[3] = (u8)(fde_len >> 24);
         buf_patch(&body, fde_offset_in_buf, lbytes, 4);
       }
     }
     /* Terminator zero-length entry. */
     buf_u32le(&body, 0);
 
     /* Section name: Mach-O wants "__TEXT,__eh_frame", ELF wants
      * ".eh_frame". The Mach-O emitter splits on comma; the ELF emitter
      * uses the literal as section name. */
     if (m->c->target.obj == KIT_OBJ_MACHO) {
       sec_name =
           pool_intern_slice(m->c->global, SLICE_LIT("__TEXT,__eh_frame"));
     } else {
       sec_name = pool_intern_slice(m->c->global, SLICE_LIT(".eh_frame"));
     }
     eh_sec = obj_section(m->obj, sec_name, SEC_OTHER, SF_ALLOC, 8);
     {
       u32 total = buf_pos(&body);
       u8* bytes = (u8*)heap->alloc(heap, total, 1);
       if (!bytes) {
         heap->free(heap, pc_slot_rels, sizeof(u32) * mc->nfdes);
         heap->free(heap, fde_syms, sizeof(ObjSymId) * mc->nfdes);
         buf_fini(&body);
         compiler_panic(m->c, m->loc, "MCEmitter: CFI OOM");
       }
       buf_flatten(&body, bytes);
       obj_write(m->obj, eh_sec, bytes, total);
       heap->free(heap, bytes, total);
     }
     for (i = 0; i < mc->nfdes; ++i) {
       /* R_PC32 against the function symbol: linker writes
        * (S + A - P) into the 4-byte slot, yielding a pc-relative
        * displacement that the unwinder can decode via DW_EH_PE_pcrel
        * | DW_EH_PE_sdata4. */
       obj_reloc_ex(m->obj, eh_sec, pc_slot_rels[i], R_PC32, fde_syms[i],
                    /*addend=*/0, /*explicit_addend=*/1, /*pair=*/0);
     }
     heap->free(heap, pc_slot_rels, sizeof(u32) * mc->nfdes);
     heap->free(heap, fde_syms, sizeof(ObjSymId) * mc->nfdes);
   }
 
   buf_fini(&body);
 
   if (mc->dirs) {
     heap->free(heap, mc->dirs, sizeof(CfiDirective) * mc->dirs_cap);
     mc->dirs = NULL;
     mc->dirs_cap = 0;
     mc->ndirs = 0;
   }
   if (mc->fdes) {
     heap->free(heap, mc->fdes, sizeof(CfiFde) * mc->fdes_cap);
     mc->fdes = NULL;
     mc->fdes_cap = 0;
     mc->nfdes = 0;
   }
   mc->eh_frame_emitted = 1;
 }