kit

emit.c (186276B)

---

 /* Wasm CGTarget emission.
  *
  * Records CGTarget operations into a per-function WIR list, then linearizes
  * to a WasmFunc body at func_end. Each SSA Reg becomes a Wasm local; control
  * flow that fits the kit_cg_if_begin/else/end pattern lowers to
  * if/else/end, while CG scopes (SCOPE_LOOP) lower to (block (loop ...)).
  *
  * TODO: complete IR-to-WASM coverage for:
  *   - bitfield load/store
  *   - multiple call/return results
  *   - address-taken parameters
  *   - ABI multipart params
  *   - dynamic memcpy/memset via memory.* calls
  *   - file-scope asm
  *   - atomics
  *   - intrinsics
  */
 
 #include <stdarg.h>
 #include <string.h>
 
 #include "abi/abi.h"
 #include "arch/wasm/internal.h"
 #include "cg/type.h"
 #include "core/arena.h"
 #include "core/buf.h"
 #include "core/heap.h"
 #include "core/pool.h"
 #include "obj/obj.h"
 #include "obj/wasm_imports.h"
 
 /* Shared Wasm core: in-memory WasmModule, helpers (wasm_add_func,
  * wasm_intern_func_type, wasm_func_add_insn, ...), and wasm_encode for
  * the final flush from emit_wasm. */
 #include "wasm/wasm.h"
 
 /* -----------------------------------------------------------------
  * Helpers
  * ----------------------------------------------------------------- */
 
 static SrcLoc cur_loc(WTarget* t) {
   /* Prefer the most recent statement loc the frontend reported via
    * wasm_set_loc — gives diagnostics the actual failing line, not the
    * function definition's line. Fall back to the function loc when no
    * statement loc has been set (line == 0). */
   if (t->cur_stmt_loc.line) return t->cur_stmt_loc;
   if (t->cur_fn_desc) return t->cur_fn_desc->loc;
   SrcLoc l = {0, 0, 0};
   return l;
 }
 
 static _Noreturn void wfail(WTarget* t, const char* fmt, ...) {
   va_list ap;
   va_start(ap, fmt);
   compiler_panicv(t->c, cur_loc(t), fmt, ap);
 }
 
 static _Noreturn void wfail_at(WTarget* t, SrcLoc loc, const char* fmt, ...) {
   va_list ap;
   va_start(ap, fmt);
   compiler_panicv(t->c, loc, fmt, ap);
 }
 
 static struct WasmModule* ensure_module(WTarget* t);
 
 static const char* pool_sym_cstr(Pool* p, Sym sym, size_t* len_out) {
   Slice sl = pool_slice(p, sym);
   if (len_out) *len_out = sl.len;
   return sl.s;
 }
 
 static WasmValType valtype_for_size_kind(WTarget* t, u32 size, u8 scalar_kind) {
   if (scalar_kind == ABI_SC_FLOAT) {
     if (size == 4) return WASM_VAL_F32;
     if (size == 8) return WASM_VAL_F64;
     /* The only C float wider than f64 is binary128 long double, which has
      * no wasm value type. Report it specifically rather than as a generic
      * size error. */
     if (size == 16) wfail(t, "wasm: long double not supported");
     wfail(t, "wasm: unsupported float size %u", size);
   }
   if (size <= 4) return WASM_VAL_I32;
   if (size == 8) return WASM_VAL_I64;
   wfail(t, "wasm: unsupported integer size %u", size);
 }
 
 static WasmValType valtype_for_type(WTarget* t, KitCgTypeId ty) {
   ABITypeInfo ti = abi_cg_type_info(t->c->abi, ty);
   if (ti.scalar_kind == ABI_SC_VOID) {
     wfail(t, "wasm: void value type requested");
   }
   if (ti.scalar_kind == ABI_SC_PTR) return WASM_VAL_I32; /* wasm32 ILP32 */
   return valtype_for_size_kind(t, ti.size, ti.scalar_kind);
 }
 
 static u32 align_to_u32(u32 v, u32 a) {
   if (!a) return v;
   return (v + a - 1u) & ~(a - 1u);
 }
 
 /* Low-memory guard: leaves the first NULL_GUARD bytes of linear memory
  * unassigned so addr==0 never resolves to a real symbol. */
 #define WASM_DATA_NULL_GUARD 16u
 
 static void type_size_align(WTarget* t, KitCgTypeId ty, u32 fallback_size,
                             u32 fallback_align, u32* size_out, u32* align_out) {
   ABITypeInfo ti;
   if (ty) {
     ti = abi_cg_type_info(t->c->abi, ty);
     *size_out = ti.size ? ti.size : fallback_size;
     *align_out = ti.align ? ti.align : fallback_align;
   } else {
     *size_out = fallback_size;
     *align_out = fallback_align;
   }
   if (!*size_out) *size_out = 1;
   if (!*align_out) *align_out = 1;
 }
 
 static u32 add_wasm_local(WTarget* t, WasmValType vt) {
   if (!t->cur_func) wfail(t, "wasm: local allocation outside a function");
   return wasm_func_push_local(t->c, t->module, t->cur_func, vt);
 }
 
 static void ensure_linear_memory(WTarget* t) {
   ensure_module(t);
   if (!t->has_memory) {
     WasmMemory* mem = wasm_add_memory(t->c, t->module);
     mem->min_pages = 1;
     t->has_memory = 1;
   }
 }
 
 /* Atomic memory ops require the linear memory to be declared shared and to
  * carry a maximum size. We promote the (single) module memory to shared on
  * first atomic emission. max_pages is provisionally set to the wasm32 limit
  * (65536, i.e. 4 GiB); wasm_materialize_data tightens it to match min_pages
  * after the final layout is known so embedders can pre-reserve a snug arena. */
 static void ensure_shared_memory(WTarget* t) {
   ensure_linear_memory(t);
   WasmMemory* mem = &t->module->memories[0];
   if (!mem->shared) {
     mem->shared = 1;
   }
   if (!mem->has_max) {
     mem->has_max = 1;
     mem->max_pages = 65536u;
   }
 }
 
 static void ensure_stack_pointer(WTarget* t) {
   ensure_linear_memory(t);
   if (!t->has_stack_pointer) {
     WasmGlobal* g = wasm_add_global(t->c, t->module);
     g->name = wasm_strdup(t->module->heap, "__stack_pointer",
                           sizeof("__stack_pointer") - 1u);
     g->type = WASM_VAL_I32;
     g->mutable_ = 1;
     g->init.kind = WASM_INSN_I32_CONST;
     g->init.imm = 65536;
     t->stack_pointer_global = t->module->nglobals - 1u;
     t->stack_size = 65536;
     t->has_stack_pointer = 1;
   }
 }
 
 /* Map an SSA Reg to its WasmFunc local index, allocating on first use. */
 static u32 reg_local(WTarget* t, Reg r, KitCgTypeId ty, RegClass cls) {
   Heap* h = t->c->ctx->heap;
   if (r == REG_NONE) wfail(t, "wasm: REG_NONE used as operand");
   if (r >= t->reg_cap) {
     u32 nc = t->reg_cap ? t->reg_cap : 16u;
     while (nc <= r) nc *= 2u;
     u32* nl = (u32*)h->realloc(h, t->reg_to_local, sizeof(u32) * t->reg_cap,
                                sizeof(u32) * nc, _Alignof(u32));
     KitCgTypeId* nt = (KitCgTypeId*)h->realloc(
         h, t->reg_type, sizeof(KitCgTypeId) * t->reg_cap,
         sizeof(KitCgTypeId) * nc, _Alignof(KitCgTypeId));
     u8* nc_arr = (u8*)h->realloc(h, t->reg_cls, t->reg_cap, nc, 1);
     if (!nl || !nt || !nc_arr) wfail(t, "wasm: out of memory");
     for (u32 i = t->reg_cap; i < nc; ++i) {
       nl[i] = 0xffffffffu;
       nt[i] = KIT_CG_TYPE_NONE;
       nc_arr[i] = 0;
     }
     t->reg_to_local = nl;
     t->reg_type = nt;
     t->reg_cls = nc_arr;
     t->reg_cap = nc;
   }
   /* CG may reuse the same Reg id with different value types: api_ensure_reg
    * for an SV_CMP reuses one of the cmp operands' Regs (originally e.g. i64)
    * to hold the i32 cmp result. Detect a type change and rebind to a fresh
    * wasm local; the previous binding is dead from CG's point of view. */
   if (t->reg_to_local[r] != 0xffffffffu && t->reg_type[r]) {
     WasmValType cached_vt = valtype_for_type(t, t->reg_type[r]);
     WasmValType want_vt = valtype_for_type(t, ty);
     if (cached_vt == want_vt) return t->reg_to_local[r];
     /* fall through to allocate fresh */
   }
   {
     WasmValType vt = valtype_for_type(t, ty);
     t->reg_to_local[r] = add_wasm_local(t, vt);
     t->reg_type[r] = ty;
     t->reg_cls[r] = (u8)cls;
     if (r + 1u > t->reg_hwm) t->reg_hwm = r + 1u;
   }
   return t->reg_to_local[r];
 }
 
 /* -----------------------------------------------------------------
  * WIR appending
  * ----------------------------------------------------------------- */
 
 static WIR* wir_push(WTarget* t) {
   Heap* h = t->c->ctx->heap;
   if (t->nwir == t->wir_cap) {
     u32 nc = t->wir_cap ? t->wir_cap * 2u : 64u;
     void* p = h->realloc(h, t->wir, sizeof(WIR) * t->wir_cap, sizeof(WIR) * nc,
                          _Alignof(WIR));
     if (!p) wfail(t, "wasm: out of memory");
     t->wir = (WIR*)p;
     t->wir_cap = nc;
   }
   WIR* w = &t->wir[t->nwir++];
   memset(w, 0, sizeof *w);
   return w;
 }
 
 /* Operand-kind encoding stored in WIR's imm_kind / imm_kind_b. */
 enum {
   WOP_REG = 0,
   WOP_IMM = 1,
   WOP_LOCAL = 2,
   WOP_WASM_LOCAL = 3,
   WOP_ADDR = 4
 };
 
 static void wir_capture_operand(WIR* w, int which, Operand op) {
   u32 kind;
   i64 ival;
   Reg r = REG_NONE;
   switch (op.kind) {
     case OPK_REG:
       kind = WOP_REG;
       r = op.v.reg;
       ival = 0;
       break;
     case OPK_IMM:
       kind = WOP_IMM;
       ival = op.v.imm;
       break;
     case OPK_LOCAL:
       kind = WOP_LOCAL;
       ival = (i64)op.v.frame_slot;
       break;
     default:
       kind = 99u;
       ival = 0;
       break;
   }
   if (which == 0) {
     w->imm_kind = kind;
     w->imm_a = ival;
     w->a = r;
   } else {
     w->imm_kind_b = kind;
     w->imm_b = ival;
     w->b = r;
   }
 }
 
 /* -----------------------------------------------------------------
  * Labels
  * ----------------------------------------------------------------- */
 
 Label wasm_label_new(CGTarget* tg) {
   WTarget* t = (WTarget*)tg;
   Heap* h = t->c->ctx->heap;
   if (t->nlabels == t->labels_cap) {
     u32 nc = t->labels_cap ? t->labels_cap * 2u : 16u;
     void* p = h->realloc(h, t->labels, sizeof(WLabel) * t->labels_cap,
                          sizeof(WLabel) * nc, _Alignof(WLabel));
     if (!p) wfail(t, "wasm: out of memory");
     t->labels = (WLabel*)p;
     t->labels_cap = nc;
   }
   u32 id = t->nlabels++;
   memset(&t->labels[id], 0, sizeof t->labels[id]);
   return (Label)(id + 1u);
 }
 
 static WLabel* lookup_label(WTarget* t, Label l) {
   if (l == LABEL_NONE || l - 1u >= t->nlabels) return NULL;
   return &t->labels[l - 1u];
 }
 
 void wasm_label_place(CGTarget* tg, Label l) {
   WTarget* t = (WTarget*)tg;
   WLabel* lbl = lookup_label(t, l);
   if (!lbl) wfail(t, "wasm: label_place on unknown label");
   /* If this label is registered to a scope, the scope ops drive the wasm
    * structure — placement here is a no-op. */
   if (lbl->kind == WLBL_SCOPE_BREAK || lbl->kind == WLBL_SCOPE_CONT) {
     lbl->placed = 1;
     t->dead = 0;
     return;
   }
   /* Free-standing label placement: record but emit nothing. The CG layer
    * sometimes places scope continue/break labels just before/after a
    * scope_begin/end pair; for wasm the structured scope ops drive the
    * `br N` targets, so these placements are no-ops. A subsequent jump or
    * cmp_branch that lands on an unbound label will diagnose. */
   lbl->placed = 1;
   lbl->wir_index = t->nwir;
   if (lbl->kind == WLBL_UNBOUND) lbl->kind = WLBL_FORWARD;
   WIR* w = wir_push(t);
   w->op = WIR_LABEL;
   w->labels[0] = l;
   t->dead = 0;
 }
 
 void wasm_jump(CGTarget* tg, Label l) {
   WTarget* t = (WTarget*)tg;
   if (t->dead) return;
   WLabel* lbl = lookup_label(t, l);
   if (!lbl) wfail(t, "wasm: jump to unknown label");
   WIR* w = wir_push(t);
   w->op = WIR_JUMP;
   w->labels[0] = l;
   t->dead = 1;
 }
 
 void wasm_cmp_branch(CGTarget* tg, CmpOp op, Operand a, Operand b, Label l) {
   WTarget* t = (WTarget*)tg;
   if (t->dead) return;
   if (!lookup_label(t, l)) wfail(t, "wasm: cmp_branch to unknown label");
   WIR* w = wir_push(t);
   w->op = WIR_CMP_BRANCH;
   w->cgop = (u8)op;
   wir_capture_operand(w, 0, a);
   wir_capture_operand(w, 1, b);
   w->type = a.type ? a.type : b.type;
   w->labels[0] = l;
 }
 
 void wasm_switch(CGTarget* tg, const CGSwitchDesc* d) {
   WTarget* t = (WTarget*)tg;
   WIR* w;
   if (t->dead) return;
   if (!d) wfail(t, "wasm: switch without descriptor");
   if (d->default_label == LABEL_NONE)
     wfail(t, "wasm: switch without default label");
   if (d->ncases && !d->cases) wfail(t, "wasm: switch case count without cases");
   if (d->selector.kind != OPK_REG && d->selector.kind != OPK_IMM &&
       d->selector.kind != OPK_LOCAL)
     wfail(t, "wasm: switch selector has unsupported operand kind");
   if (!lookup_label(t, d->default_label))
     wfail(t, "wasm: switch default label is unknown");
   for (u32 i = 0; i < d->ncases; ++i) {
     if (!lookup_label(t, d->cases[i].label))
       wfail(t, "wasm: switch case label is unknown");
   }
 
   w = wir_push(t);
   w->op = WIR_SWITCH;
   wir_capture_operand(w, 0, d->selector);
   w->type = d->selector_type;
   w->labels[0] = d->default_label;
   w->switch_ncases = d->ncases;
   if (d->ncases) {
     Heap* h = t->c->ctx->heap;
     w->switch_cases = (CGSwitchCase*)h->alloc(
         h, sizeof(CGSwitchCase) * d->ncases, _Alignof(CGSwitchCase));
     if (!w->switch_cases) wfail(t, "wasm: out of memory");
     memcpy(w->switch_cases, d->cases, sizeof(CGSwitchCase) * d->ncases);
   }
   t->dead = 1;
 }
 
 /* -----------------------------------------------------------------
  * Scopes
  * ----------------------------------------------------------------- */
 
 CGScope wasm_scope_begin(CGTarget* tg, const CGScopeDesc* d) {
   WTarget* t = (WTarget*)tg;
   if (t->nscopes >= 32u) wfail(t, "wasm: too many nested scopes (max 32)");
   WScope* s = &t->scopes[t->nscopes];
   memset(s, 0, sizeof *s);
   s->id = ++t->next_scope_id;
   s->cg_kind = d->kind;
   s->break_lbl = d->break_label;
   s->cont_lbl = d->continue_label;
   s->result_type = d->result_type;
 
   /* Wire scope's break/continue labels to this scope so jump()/cmp_branch()
    * to them can lower to wasm `br`. */
   if (d->break_label != LABEL_NONE) {
     WLabel* lbl = lookup_label(t, d->break_label);
     if (lbl) {
       lbl->kind = WLBL_SCOPE_BREAK;
       lbl->scope_id = s->id;
     }
   }
   if (d->continue_label != LABEL_NONE) {
     WLabel* lbl = lookup_label(t, d->continue_label);
     if (lbl) {
       lbl->kind = WLBL_SCOPE_CONT;
       lbl->scope_id = s->id;
     }
   }
 
   WIR* open = wir_push(t);
   open->op = WIR_SCOPE_OPEN;
   open->scope_id = s->id;
   open->cgop = d->kind;
   open->dst = REG_NONE;
 
   s->placed_in_wir = 1;
   t->nscopes++;
   return (CGScope)s->id;
 }
 
 static WScope* scope_by_id(WTarget* t, u32 id) {
   for (u32 i = 0; i < t->nscopes; ++i) {
     if (t->scopes[i].id == id) return &t->scopes[i];
   }
   return NULL;
 }
 
 void wasm_scope_end(CGTarget* tg, CGScope sc) {
   WTarget* t = (WTarget*)tg;
   WScope* s = scope_by_id(t, (u32)sc);
   if (!s) wfail(t, "wasm: scope_end on unknown scope");
   WIR* w = wir_push(t);
   w->op = WIR_SCOPE_CLOSE;
   w->scope_id = s->id;
   /* Pop the scope from the stack. CG always closes in LIFO order. */
   if (t->nscopes == 0 || t->scopes[t->nscopes - 1u].id != s->id)
     wfail(t, "wasm: scope_end out of LIFO order");
   t->nscopes--;
   t->dead = 0;
 }
 
 void wasm_break_to(CGTarget* tg, CGScope sc) {
   WTarget* t = (WTarget*)tg;
   if (t->dead) return;
   WScope* s = scope_by_id(t, (u32)sc);
   if (!s) wfail(t, "wasm: break_to unknown scope");
   WIR* w = wir_push(t);
   w->op = WIR_JUMP;
   w->labels[0] = s->break_lbl;
   t->dead = 1;
 }
 
 void wasm_continue_to(CGTarget* tg, CGScope sc) {
   WTarget* t = (WTarget*)tg;
   if (t->dead) return;
   WScope* s = scope_by_id(t, (u32)sc);
   if (!s) wfail(t, "wasm: continue_to unknown scope");
   WIR* w = wir_push(t);
   w->op = WIR_JUMP;
   w->labels[0] = s->cont_lbl;
   t->dead = 1;
 }
 
 /* -----------------------------------------------------------------
  * Function lifecycle
  * ----------------------------------------------------------------- */
 
 /* Forward decl — promotes an undefined function symbol's WasmFunc to an
  * import using the supplied ABI to build the wasm signature. */
 static void promote_import_func(WTarget* t, ObjSymId sym, WasmFunc* f,
                                 const ABIFuncInfo* abi);
 
 /* Lookup or allocate a Wasm function index for an ObjSymId. Returns
  * (wasm_func_idx, *out_func) on success. */
 static u32 sym_to_wasm_func(WTarget* t, ObjSymId sym, WasmFunc** out_func) {
   Heap* h = t->c->ctx->heap;
   if (sym >= t->sym_to_func_cap) {
     u32 nc = t->sym_to_func_cap ? t->sym_to_func_cap : 16u;
     while (nc <= sym) nc *= 2u;
     u32* p =
         (u32*)h->realloc(h, t->sym_to_func, sizeof(u32) * t->sym_to_func_cap,
                          sizeof(u32) * nc, _Alignof(u32));
     if (!p) wfail(t, "wasm: out of memory");
     for (u32 i = t->sym_to_func_cap; i < nc; ++i) p[i] = 0;
     t->sym_to_func = p;
     t->sym_to_func_cap = nc;
   }
   if (t->sym_to_func[sym]) {
     u32 idx = t->sym_to_func[sym] - 1u;
     if (out_func) *out_func = &t->module->funcs[idx];
     return idx;
   }
   /* Create a fresh WasmFunc and link. */
   WasmFunc* f = wasm_add_func(t->c, t->module);
   u32 idx = t->module->nfuncs - 1u;
   t->sym_to_func[sym] = idx + 1u;
   if (out_func) *out_func = f;
   return idx;
 }
 
 static WSlot* slot_push(WTarget* t) {
   Heap* h = t->c->ctx->heap;
   if (t->nslots == t->slots_cap) {
     u32 nc = t->slots_cap ? t->slots_cap * 2u : 16u;
     WSlot* ns = (WSlot*)h->realloc(h, t->slots, sizeof(WSlot) * t->slots_cap,
                                    sizeof(WSlot) * nc, _Alignof(WSlot));
     if (!ns) wfail(t, "wasm: out of memory");
     t->slots = ns;
     t->slots_cap = nc;
   }
   WSlot* s = &t->slots[t->nslots++];
   memset(s, 0, sizeof *s);
   return s;
 }
 
 /* True iff `ty` maps to a single wasm value type (i32/i64/f32/f64) and so can
  * live directly in a wasm local. Aggregates (records/arrays) and anything
  * wider than 8 bytes must be homed in linear memory. */
 static int type_is_wasm_scalar(WTarget* t, KitCgTypeId ty) {
   ABITypeInfo ti;
   if (!ty) return 0;
   ti = abi_cg_type_info(t->c->abi, ty);
   if (ti.scalar_kind == ABI_SC_VOID) return 0;
   if (ti.scalar_kind == ABI_SC_PTR) return 1;
   return ti.size <= 8u;
 }
 
 static FrameSlot alloc_frame_slot_kind(WTarget* t, KitCgTypeId ty, u32 size,
                                        u32 align, int stack_backed) {
   WSlot* s;
   u32 slot_id;
   type_size_align(t, ty, size, align, &size, &align);
   /* A non-scalar type has no wasm value type; force it into linear memory. */
   if (!stack_backed && !type_is_wasm_scalar(t, ty)) stack_backed = 1;
   s = slot_push(t);
   s->type = ty;
   s->size = size;
   s->align = align;
   if (stack_backed) {
     ensure_stack_pointer(t);
     t->frame_size = align_to_u32(t->frame_size, align);
     s->kind = W_SLOT_STACK;
     s->frame_offset = t->frame_size;
     t->frame_size += size;
     if (align > t->frame_align) t->frame_align = align;
     t->has_stack_frame = 1;
   } else {
     s->kind = W_SLOT_LOCAL;
     s->wasm_local = add_wasm_local(t, valtype_for_type(t, ty));
   }
   slot_id = t->nslots;
   return (FrameSlot)slot_id;
 }
 
 static WSlot* slot_for(WTarget* t, FrameSlot fs) {
   if (fs == FRAME_SLOT_NONE) wfail(t, "wasm: FRAME_SLOT_NONE used");
   u32 idx = fs - 1u;
   if (idx >= t->nslots) wfail(t, "wasm: bad frame slot id");
   return &t->slots[idx];
 }
 
 static void promote_slot_to_stack(WTarget* t, WSlot* s) {
   if (s->kind == W_SLOT_STACK) return;
   ensure_stack_pointer(t);
   t->frame_size = align_to_u32(t->frame_size, s->align ? s->align : 1u);
   s->frame_offset = t->frame_size;
   t->frame_size += s->size ? s->size : 1u;
   if (s->align > t->frame_align) t->frame_align = s->align;
   s->kind = W_SLOT_STACK;
   t->has_stack_frame = 1;
 }
 
 void wasm_func_begin(CGTarget* tg, const CGFuncDesc* d) {
   WTarget* t = (WTarget*)tg;
   WasmFunc* f;
   u32 idx;
   const CgType* fnty;
   const ABIFuncInfo* abi;
   Heap* h = t->c->ctx->heap;
 
   ensure_module(t);
   t->cur_fn_desc = d;
   memset(&t->cur_stmt_loc, 0, sizeof t->cur_stmt_loc);
   t->nwir = 0;
   t->nlabels = 0;
   t->nslots = 0;
   t->nscopes = 0;
   t->next_scope_id = 0;
   t->frame_size = 0;
   t->frame_align = 1;
   t->frame_base_local = 0xffffffffu;
   t->frame_saved_sp_local = 0xffffffffu;
   t->has_stack_frame = 0;
   t->dead = 0;
   t->sret_param_local = 0xffffffffu;
   t->va_ptr_param_local = 0xffffffffu;
   t->cur_has_sret = 0;
   t->cur_is_variadic = 0;
   t->varcall_saved_sp_local = 0xffffffffu;
   t->varcall_buf_local = 0xffffffffu;
   t->va_arg_tmp_addr_local = 0xffffffffu;
   t->nparams_cg = 0;
   t->nbyval_copies = 0;
   /* Wipe reg map. Only [0, reg_hwm) can hold a non-sentinel binding from a
    * prior function — reg_cap grows monotonically but slots beyond the
    * high-water mark are sentinel-initialized at grow time and never bound, so
    * wiping the full reg_cap each function is wasted work on large TUs. */
   for (u32 i = 0; i < t->reg_hwm; ++i) t->reg_to_local[i] = 0xffffffffu;
 
   idx = sym_to_wasm_func(t, d->sym, &f);
   t->cur_func_idx = idx;
   t->cur_func = f;
 
   fnty = cg_type_get(t->c, d->fn_type);
   if (!fnty || fnty->kind != KIT_CG_TYPE_FUNC)
     wfail(t, "wasm: func_begin without function type");
   abi = d->abi;
   if (!abi) wfail(t, "wasm: func_begin with no ABI info");
 
   /* Build the wasm function's param layout:
    *   [sret_ptr]? [param_0] [param_1] ...
    * with IGNORE'd CG params dropped. Record per-CG-param the wasm-local
    * index so wasm_param can place each frame slot on the right local.
    * params/locals/local_names grow on demand — no fixed cap. */
   f->nparams = 0;
   if (abi->has_sret) {
     t->sret_param_local =
         wasm_func_push_param(t->c, t->module, f, WASM_VAL_I32);
     t->cur_has_sret = 1;
     ensure_linear_memory(t);
   }
   if (fnty->func.nparams > t->param_local_idx_cap) {
     u32 nc = t->param_local_idx_cap ? t->param_local_idx_cap : 4u;
     while (nc < fnty->func.nparams) nc *= 2u;
     u32* p = (u32*)h->realloc(h, t->param_local_idx,
                               sizeof(u32) * t->param_local_idx_cap,
                               sizeof(u32) * nc, _Alignof(u32));
     if (!p) wfail(t, "wasm: out of memory");
     t->param_local_idx = p;
     t->param_local_idx_cap = nc;
   }
   t->nparams_cg = fnty->func.nparams;
   for (u32 i = 0; i < fnty->func.nparams; ++i) {
     const ABIArgInfo* ai = &abi->params[i];
     if (ai->kind == ABI_ARG_IGNORE) {
       t->param_local_idx[i] = 0xffffffffu;
       continue;
     }
     if (ai->kind == ABI_ARG_INDIRECT) {
       t->param_local_idx[i] =
           wasm_func_push_param(t->c, t->module, f, WASM_VAL_I32);
       ensure_linear_memory(t);
     } else {
       if (ai->nparts != 1)
         wfail(t, "wasm: multi-part DIRECT param %u not yet implemented", i);
       const ABIArgPart* p = &ai->parts[0];
       WasmValType vt = valtype_for_size_kind(
           t, p->size, p->cls == ABI_CLASS_FP ? ABI_SC_FLOAT : ABI_SC_INT);
       t->param_local_idx[i] = wasm_func_push_param(t->c, t->module, f, vt);
     }
   }
   /* Variadic: append hidden i32 va_ptr trailing param. Must match
    * abi_to_wasm_func_type so indirect calls' signature interning agrees. */
   if (abi->variadic) {
     t->va_ptr_param_local =
         wasm_func_push_param(t->c, t->module, f, WASM_VAL_I32);
     t->cur_is_variadic = 1;
     ensure_linear_memory(t);
   }
   f->nresults = 0;
   if (!abi->has_sret && abi->ret.kind == ABI_ARG_DIRECT &&
       abi->ret.nparts == 1) {
     const ABIArgPart* p = &abi->ret.parts[0];
     wasm_func_push_result(
         t->c, t->module, f,
         valtype_for_size_kind(
             t, p->size, p->cls == ABI_CLASS_FP ? ABI_SC_FLOAT : ABI_SC_INT));
   }
   f->typeidx = wasm_intern_func_type(t->c, t->module, f);
 
   /* Export under the symbol's name when the symbol is globally bound. */
   const ObjSym* sym = obj_symbol_get(t->obj, d->sym);
   if (sym && sym->bind != SB_LOCAL) {
     const char* name = pool_sym_cstr(t->c->global, sym->name, NULL);
     if (name && *name) {
       Heap* h = t->c->ctx->heap;
       size_t nlen = strlen(name);
       char* dup = (char*)h->alloc(h, nlen + 1u, 1);
       memcpy(dup, name, nlen + 1u);
       f->export_name = dup;
       WasmExport* e = wasm_add_export(t->c, t->module);
       char* exp_name = (char*)h->alloc(h, nlen + 1u, 1);
       memcpy(exp_name, name, nlen + 1u);
       e->name = exp_name;
       e->kind = 0; /* function export */
       e->index = idx;
     }
   }
 }
 
 CGLocalStorage wasm_param(CGTarget* tg, const CGParamDesc* d) {
   WTarget* t = (WTarget*)tg;
   CGLocalStorage ls;
   Heap* h = t->c->ctx->heap;
   u32 wli;
   WSlot* s;
   memset(&ls, 0, sizeof ls);
   if (d->index >= t->nparams_cg)
     wfail(t, "wasm: param index %u out of range (nparams=%u)", d->index,
           t->nparams_cg);
   wli = t->param_local_idx[d->index];
   if (wli == 0xffffffffu) {
     /* ABI_ARG_IGNORE — no wasm storage. Push a placeholder slot so the
      * returned FrameSlot is meaningful to CG; it never gets emitted. */
     s = slot_push(t);
     s->type = d->type;
     s->size = d->size;
     s->align = d->align ? d->align : 1u;
     s->kind = W_SLOT_LOCAL;
     s->wasm_local = 0;
     ls.kind = CG_LOCAL_STORAGE_FRAME;
     ls.v.frame_slot = (FrameSlot)t->nslots;
     return ls;
   }
   if (d->abi && d->abi->kind == ABI_ARG_INDIRECT) {
     /* byval: callee receives an i32 pointer; copy the aggregate into a
      * caller-isolated stack-backed slot at function entry. */
     u32 size = d->size;
     u32 align = d->align ? d->align : 1u;
     type_size_align(t, d->type, size, align, &size, &align);
     ensure_stack_pointer(t);
     s = slot_push(t);
     s->type = d->type;
     s->size = size;
     s->align = align;
     s->kind = W_SLOT_STACK;
     t->frame_size = align_to_u32(t->frame_size, align);
     s->frame_offset = t->frame_size;
     t->frame_size += size;
     if (align > t->frame_align) t->frame_align = align;
     t->has_stack_frame = 1;
     /* Queue prologue copy-in from the pointer's wasm-local into &slot. */
     if (t->nbyval_copies == t->byval_copies_cap) {
       u32 nc = t->byval_copies_cap ? t->byval_copies_cap * 2u : 4u;
       WByvalCopy* nb = (WByvalCopy*)h->realloc(
           h, t->byval_copies, sizeof(WByvalCopy) * t->byval_copies_cap,
           sizeof(WByvalCopy) * nc, _Alignof(WByvalCopy));
       if (!nb) wfail(t, "wasm: out of memory");
       t->byval_copies = nb;
       t->byval_copies_cap = nc;
     }
     WByvalCopy* bc = &t->byval_copies[t->nbyval_copies++];
     bc->ptr_wasm_local = wli;
     bc->dst_slot_id = t->nslots - 1u;
     ls.kind = CG_LOCAL_STORAGE_FRAME;
     ls.v.frame_slot = (FrameSlot)t->nslots;
     return ls;
   }
   if (d->flags & CG_LOCAL_ADDR_TAKEN) {
     wfail(t, "wasm: address-taken parameter not yet implemented");
   }
   s = slot_push(t);
   s->type = d->type;
   s->size = d->size;
   s->align = d->align ? d->align : 1u;
   s->kind = W_SLOT_LOCAL;
   s->wasm_local = wli;
   ls.kind = CG_LOCAL_STORAGE_FRAME;
   ls.v.frame_slot = (FrameSlot)t->nslots;
   return ls;
 }
 
 /* Allocate a frame slot backed by a fresh wasm local. */
 static FrameSlot alloc_frame_slot(WTarget* t, KitCgTypeId ty) {
   return alloc_frame_slot_kind(t, ty, 0, 0, 0);
 }
 
 FrameSlot wasm_frame_slot(CGTarget* tg, const FrameSlotDesc* d) {
   WTarget* t = (WTarget*)tg;
   if (!d->type && !d->size) wfail(t, "wasm: frame slot without type/size");
   return alloc_frame_slot_kind(
       t, d->type, d->size, d->align,
       (d->flags & FSF_ADDR_TAKEN) != 0 || d->kind == FS_ALLOCA);
 }
 
 CGLocalStorage wasm_local(CGTarget* tg, const CGLocalDesc* d) {
   WTarget* t = (WTarget*)tg;
   CGLocalStorage ls;
   memset(&ls, 0, sizeof ls);
   if (d->flags & (CG_LOCAL_ADDR_TAKEN | CG_LOCAL_MEMORY_REQUIRED)) {
     ls.kind = CG_LOCAL_STORAGE_FRAME;
     ls.v.frame_slot = alloc_frame_slot_kind(t, d->type, d->size, d->align, 1);
     return ls;
   }
   ls.kind = CG_LOCAL_STORAGE_FRAME;
   ls.v.frame_slot = alloc_frame_slot(t, d->type);
   return ls;
 }
 
 /* -----------------------------------------------------------------
  * Data-movement records
  * ----------------------------------------------------------------- */
 
 void wasm_load_imm(CGTarget* tg, Operand dst, i64 imm) {
   WTarget* t = (WTarget*)tg;
   if (t->dead) return;
   if (dst.kind != OPK_REG) wfail(t, "wasm: load_imm dst must be REG");
   WIR* w = wir_push(t);
   w->op = WIR_LOAD_IMM;
   w->dst = dst.v.reg;
   w->imm = imm;
   w->type = dst.type;
   w->cls = dst.cls;
 }
 
 void wasm_load_const(CGTarget* tg, Operand dst, ConstBytes cb) {
   WTarget* t = (WTarget*)tg;
   if (t->dead) return;
   if (dst.kind != OPK_REG) wfail(t, "wasm: load_const dst must be REG");
   WasmValType vt = valtype_for_type(t, cb.type);
   WIR* w = wir_push(t);
   w->dst = dst.v.reg;
   w->type = cb.type;
   w->cls = dst.cls;
   if (vt == WASM_VAL_F32) {
     if (cb.size != 4) wfail(t, "wasm: f32 const must be 4 bytes");
     float f;
     memcpy(&f, cb.bytes, 4);
     w->op = WIR_LOAD_CONST_F;
     w->fp_imm = (double)f;
   } else if (vt == WASM_VAL_F64) {
     if (cb.size != 8) wfail(t, "wasm: f64 const must be 8 bytes");
     double v;
     memcpy(&v, cb.bytes, 8);
     w->op = WIR_LOAD_CONST_F;
     w->fp_imm = v;
   } else {
     i64 v = 0;
     memcpy(&v, cb.bytes, cb.size < 8 ? cb.size : 8u);
     /* Sign-extend for small signed types so the immediate has the expected
      * bit pattern. */
     if (cb.size == 1)
       v = (i64)(i8)v;
     else if (cb.size == 2)
       v = (i64)(i16)v;
     else if (cb.size == 4)
       v = (i64)(i32)v;
     w->op = WIR_LOAD_IMM;
     w->imm = v;
   }
 }
 
 void wasm_copy(CGTarget* tg, Operand dst, Operand src) {
   WTarget* t = (WTarget*)tg;
   if (t->dead) return;
   if (dst.kind != OPK_REG || src.kind != OPK_REG)
     wfail(t, "wasm: copy operands must both be REG");
   WIR* w = wir_push(t);
   w->op = WIR_COPY;
   w->dst = dst.v.reg;
   w->a = src.v.reg;
   w->type = dst.type;
 }
 
 void wasm_binop(CGTarget* tg, BinOp op, Operand dst, Operand a, Operand b) {
   WTarget* t = (WTarget*)tg;
   if (t->dead) return;
   if (dst.kind != OPK_REG) wfail(t, "wasm: binop dst must be REG");
   WIR* w = wir_push(t);
   w->op = WIR_BINOP;
   w->cgop = (u8)op;
   w->dst = dst.v.reg;
   wir_capture_operand(w, 0, a);
   wir_capture_operand(w, 1, b);
   w->type = dst.type;
   w->cls = dst.cls;
 }
 
 void wasm_unop(CGTarget* tg, UnOp op, Operand dst, Operand a) {
   WTarget* t = (WTarget*)tg;
   if (t->dead) return;
   if (dst.kind != OPK_REG) wfail(t, "wasm: unop dst must be REG");
   WIR* w = wir_push(t);
   w->op = WIR_UNOP;
   w->cgop = (u8)op;
   w->dst = dst.v.reg;
   wir_capture_operand(w, 0, a);
   w->type = dst.type;
   w->cls = dst.cls;
 }
 
 void wasm_cmp(CGTarget* tg, CmpOp op, Operand dst, Operand a, Operand b) {
   WTarget* t = (WTarget*)tg;
   if (t->dead) return;
   if (dst.kind != OPK_REG) wfail(t, "wasm: cmp dst must be REG");
   WIR* w = wir_push(t);
   w->op = WIR_CMP;
   w->cgop = (u8)op;
   w->dst = dst.v.reg;
   wir_capture_operand(w, 0, a);
   wir_capture_operand(w, 1, b);
   w->type = dst.type;
   w->type2 = a.type ? a.type : b.type;
   w->cls = dst.cls;
 }
 
 void wasm_convert(CGTarget* tg, ConvKind ck, Operand dst, Operand src) {
   WTarget* t = (WTarget*)tg;
   if (t->dead) return;
   if (dst.kind != OPK_REG) wfail(t, "wasm: convert dst must be REG");
   WIR* w = wir_push(t);
   w->op = WIR_CONVERT;
   w->cgop = (u8)ck;
   w->dst = dst.v.reg;
   wir_capture_operand(w, 0, src);
   w->type = dst.type;
   w->type2 = src.type;
   w->cls = dst.cls;
 }
 
 /* Build (or reuse) the wasm typeidx for a function-typed indirect call. The
  * signature shape must exactly match a direct call to the same C type:
  * - hidden i32 sret pointer prepended when ABI has_sret
  * - per-param: i32 pointer for ABI_ARG_INDIRECT, else the DIRECT scalar
  *   produced by the wasm32 BasicCABI classifier (IGNORE params dropped)
  * - result: empty when has_sret, else the DIRECT scalar
  *
  * call_indirect's runtime type check compares this typeidx against the
  * funcref's recorded type, so any mismatch with the direct-call path would
  * trap. The temporary WasmFunc is stack-allocated; wasm_intern_func_type
  * copies the param array on insertion. */
 /* Translate an ABI function signature into the wasm-level param/result list.
  * Used both for indirect-call signature interning and for import-function
  * type synthesis. `what` names the call site in diagnostics. Returns the
  * interned type index. The caller-provided buffer `params` (length `cap`) is
  * filled from index 0; *nparams_out is the count written. */
 static u32 abi_to_wasm_func_type(WTarget* t, const ABIFuncInfo* abi,
                                  WasmValType* params, u32 cap, u32* nparams_out,
                                  WasmValType* result_out, u32* nresults_out,
                                  const char* what) {
   WasmFunc tmp;
   /* results is a pointer now (multi-value); back it with a small stack buffer
    * just like the caller-supplied params buffer, so tmp.results[0] = ... below
    * is not a NULL deref. A direct return lowers to at most one result. */
   WasmValType results[4];
   memset(&tmp, 0, sizeof tmp);
   tmp.params = params;
   tmp.cap_params = cap;
   tmp.results = results;
   tmp.cap_results = 4u;
   if (abi->has_sret) {
     if (tmp.nparams >= cap) wfail(t, "wasm: %s has too many params", what);
     params[tmp.nparams++] = WASM_VAL_I32;
   }
   for (u32 i = 0; i < abi->nparams; ++i) {
     const ABIArgInfo* ai = &abi->params[i];
     if (ai->kind == ABI_ARG_IGNORE) continue;
     if (tmp.nparams >= cap) wfail(t, "wasm: %s has too many params", what);
     if (ai->kind == ABI_ARG_INDIRECT) {
       params[tmp.nparams++] = WASM_VAL_I32;
     } else {
       if (ai->nparts != 1)
         wfail(t, "wasm: %s has multi-part DIRECT param (unsupported)", what);
       const ABIArgPart* p = &ai->parts[0];
       params[tmp.nparams++] = valtype_for_size_kind(
           t, p->size, p->cls == ABI_CLASS_FP ? ABI_SC_FLOAT : ABI_SC_INT);
     }
   }
   /* Variadic functions take one hidden trailing i32 va_ptr — the address of
    * the caller-packed varargs buffer in linear memory. See wasm_call's
    * variadic packing and wasm_va_start in this file. */
   if (abi->variadic) {
     if (tmp.nparams >= cap) wfail(t, "wasm: %s has too many params", what);
     params[tmp.nparams++] = WASM_VAL_I32;
   }
   tmp.nresults = 0;
   if (!abi->has_sret && abi->ret.kind == ABI_ARG_DIRECT &&
       abi->ret.nparts == 1) {
     const ABIArgPart* p = &abi->ret.parts[0];
     tmp.results[0] = valtype_for_size_kind(
         t, p->size, p->cls == ABI_CLASS_FP ? ABI_SC_FLOAT : ABI_SC_INT);
     tmp.nresults = 1;
   }
   if (nparams_out) *nparams_out = tmp.nparams;
   if (result_out && tmp.nresults) *result_out = tmp.results[0];
   if (nresults_out) *nresults_out = tmp.nresults;
   return wasm_intern_func_type(t->c, t->module, &tmp);
 }
 
 static u32 intern_indirect_signature(WTarget* t, const ABIFuncInfo* abi) {
   WasmValType params[64];
   return abi_to_wasm_func_type(t, abi, params, 64u, NULL, NULL, NULL,
                                "indirect call");
 }
 
 /* Promote `f` (already allocated for `sym` via sym_to_wasm_func) into a wasm
  * `(import "<module>" "<field>" (func ...))` entry. The signature is
  * synthesized from the supplied ABIFuncInfo, mirroring the layout the
  * caller-side WIR_CALL pushes onto the stack: hidden i32 sret-pointer when
  * has_sret, followed by lowered params, with a single i32/i64/f32/f64
  * result for direct, non-sret returns. Module/field default to "env" / the
  * symbol's name; either may be overridden by
  * `__attribute__((import_module/import_name))`. */
 static void promote_import_func(WTarget* t, ObjSymId sym, WasmFunc* f,
                                 const ABIFuncInfo* abi) {
   Heap* h = t->c->ctx->heap;
   const ObjSym* os;
   const char* sym_name;
   size_t sym_name_len = 0;
   Sym attr_module = 0;
   Sym attr_name = 0;
   const char* mod_str = "env";
   size_t mod_len = sizeof("env") - 1u;
   if (!t->module) return;
   if (f->is_import) return;
   os = obj_symbol_get(t->obj, sym);
   if (!os) return;
   if (os->section_id != OBJ_SEC_NONE) return; /* already defined locally */
   if (os->kind != SK_UNDEF && os->kind != SK_FUNC) return;
   if (!abi)
     wfail(t,
           "wasm: cannot synthesize import signature for '%s' "
           "(missing ABI info)",
           pool_sym_cstr(t->c->global, os->name, NULL));
   if (f->ninsns != 0)
     wfail(t, "wasm: cannot promote function with emitted body to import");
   /* Synthesize the wasm type from the ABI. Diagnoses unsupported shapes
    * (varargs => multi-part DIRECT or extra parts; by-value aggregates that
    * the ABI didn't already lower to ABI_ARG_INDIRECT) by naming the import
    * symbol so the error points at the C declaration. */
   WasmValType params[64];
   u32 nparams = 0;
   WasmValType result_vt = 0;
   u32 nresults = 0;
   char what[160];
   sym_name = pool_sym_cstr(t->c->global, os->name, &sym_name_len);
   if (!sym_name) sym_name = "(anonymous)";
   /* Snprintf-free: build a short context string by hand to avoid pulling in
    * stdio. The buffer is large enough for any plausible C identifier. */
   {
     const char* prefix = "import '";
     const char* suffix = "'";
     size_t plen = 8u; /* strlen(prefix) */
     size_t slen = 1u; /* strlen(suffix) */
     size_t nlen = sym_name_len ? sym_name_len : strlen(sym_name);
     if (nlen > sizeof(what) - plen - slen - 1u)
       nlen = sizeof(what) - plen - slen - 1u;
     memcpy(what, prefix, plen);
     memcpy(what + plen, sym_name, nlen);
     memcpy(what + plen + nlen, suffix, slen);
     what[plen + nlen + slen] = 0;
   }
   f->typeidx = abi_to_wasm_func_type(t, abi, params, 64u, &nparams, &result_vt,
                                      &nresults, what);
   f->has_typeidx = 1;
   /* Mirror the synthesized params/results onto the WasmFunc so the import
    * encoder writes the matching signature. */
   wasm_func_set_params(t->c, t->module, f, params, nparams);
   wasm_func_set_results(t->c, t->module, f, &result_vt, nresults);
   /* Resolve module/name overrides set via __attribute__((import_module/
    * import_name)) on the C declaration. */
   (void)wasm_imports_get(t->obj, os->name, &attr_module, &attr_name);
   if (attr_module) mod_str = pool_sym_cstr(t->c->global, attr_module, &mod_len);
   const char* name_str =
       attr_name ? pool_sym_cstr(t->c->global, attr_name, &sym_name_len)
                 : sym_name;
   size_t name_len = attr_name
                         ? sym_name_len
                         : (sym_name_len ? sym_name_len : strlen(name_str));
   f->is_import = 1;
   {
     char* m = (char*)h->alloc(h, mod_len + 1u, 1);
     if (!m) wfail(t, "wasm: out of memory");
     memcpy(m, mod_str, mod_len);
     m[mod_len] = 0;
     f->import_module = m;
   }
   {
     char* n = (char*)h->alloc(h, name_len + 1u, 1);
     if (!n) wfail(t, "wasm: out of memory");
     memcpy(n, name_str, name_len);
     n[name_len] = 0;
     f->import_name = n;
   }
 }
 
 void wasm_call(CGTarget* tg, const CGCallDesc* d) {
   WTarget* t = (WTarget*)tg;
   if (t->dead) return;
   int is_indirect = (d->callee.kind != OPK_GLOBAL);
   if (is_indirect && d->callee.kind != OPK_REG)
     wfail(t, "wasm: indirect call callee must be a register (got opkind %u)",
           (unsigned)d->callee.kind);
   Heap* h = t->c->ctx->heap;
   int callee_has_sret = (d->abi && d->abi->has_sret) ? 1 : 0;
   int callee_variadic = (d->abi && d->abi->variadic) ? 1 : 0;
   int is_tail = (d->flags & CG_CALL_TAIL) ? 1 : 0;
   if (is_tail) {
     /* Realizability is decided by CG via wasm_ir_tail_call_unrealizable_reason
      * (target.c) before CG_CALL_TAIL is set: variadic tails are rejected there,
      * and sret tails forward the incoming sret pointer (handled in WIR emit).
      */
     ensure_module(t);
     t->module->features |= WASM_FEATURE_TAIL_CALLS;
   }
   u32 nfixed = (u32)d->nargs;
   u32 nvar = 0u;
   if (callee_variadic) {
     if (d->nargs < d->abi->nparams)
       wfail(t, "wasm: variadic call has fewer args (%u) than fixed params (%u)",
             (unsigned)d->nargs, (unsigned)d->abi->nparams);
     nfixed = d->abi->nparams;
     nvar = (u32)d->nargs - nfixed;
   }
   WIR* w = wir_push(t);
   if (is_indirect) {
     if (!d->abi) wfail(t, "wasm: indirect call without ABIFuncInfo");
     ensure_module(t);
     w->op = WIR_CALL_INDIRECT;
     w->a = d->callee.v.reg;
     w->imm = (i64)intern_indirect_signature(t, d->abi);
   } else {
     w->op = WIR_CALL;
     w->call_sym = d->callee.v.global.sym;
     /* Direct calls into externally-defined functions become wasm imports.
      * Synthesize the import signature now while the ABI is available — the
      * WIR emit loop only has the symbol index. The C frontend mints SK_FUNC
      * for `extern foo(...)` declarations; the "undefined" signal is
      * `section_id == OBJ_SEC_NONE`. SK_UNDEF can appear when a symbol's
      * kind hasn't been pinned down yet. */
     {
       const ObjSym* os = obj_symbol_get(t->obj, d->callee.v.global.sym);
       if (os && os->section_id == OBJ_SEC_NONE &&
           (os->kind == SK_UNDEF || os->kind == SK_FUNC)) {
         WasmFunc* f;
         ensure_module(t);
         (void)sym_to_wasm_func(t, d->callee.v.global.sym, &f);
         if (!f->is_import)
           promote_import_func(t, d->callee.v.global.sym, f, d->abi);
       }
     }
   }
   w->call_narg = nfixed;
   w->type = d->ret.type;
   w->call_has_sret = (u8)callee_has_sret;
   w->call_variadic = (u8)callee_variadic;
   w->call_tail = (u8)is_tail;
   w->call_nvar = nvar;
   if (callee_variadic) ensure_linear_memory(t);
   if (callee_has_sret) {
     /* Caller allocated a frame slot for the aggregate result via
      * api_alloc_call_ret_storage; pass its address as the prepended i32. */
     w->call_sret_addr = d->ret.storage;
     ensure_linear_memory(t);
   }
   if (nfixed) {
     w->call_args = (Reg*)h->alloc(h, sizeof(Reg) * nfixed, _Alignof(Reg));
     w->call_arg_imms = (i64*)h->alloc(h, sizeof(i64) * nfixed, _Alignof(i64));
     w->call_arg_kinds = (u8*)h->alloc(h, nfixed, 1);
     w->call_arg_types = (KitCgTypeId*)h->alloc(h, sizeof(KitCgTypeId) * nfixed,
                                                _Alignof(KitCgTypeId));
     w->call_arg_addrs =
         (Operand*)h->alloc(h, sizeof(Operand) * nfixed, _Alignof(Operand));
     memset(w->call_arg_addrs, 0, sizeof(Operand) * nfixed);
     for (u32 i = 0; i < nfixed; ++i) {
       const CGABIValue* av = &d->args[i];
       w->call_arg_types[i] = av->type;
       int is_indirect = (av->abi && av->abi->kind == ABI_ARG_INDIRECT);
       if (is_indirect) {
         if (av->storage.kind != OPK_LOCAL && av->storage.kind != OPK_INDIRECT &&
             av->storage.kind != OPK_GLOBAL) {
           wfail(t, "wasm: byval call arg %u storage kind %u must be an address",
                 i, (unsigned)av->storage.kind);
         }
         w->call_arg_kinds[i] = WOP_ADDR;
         w->call_args[i] = REG_NONE;
         w->call_arg_imms[i] = 0;
         w->call_arg_addrs[i] = av->storage;
         ensure_linear_memory(t);
       } else if (av->storage.kind == OPK_REG) {
         w->call_arg_kinds[i] = 0;
         w->call_args[i] = av->storage.v.reg;
         w->call_arg_imms[i] = 0;
       } else if (av->storage.kind == OPK_IMM) {
         w->call_arg_kinds[i] = 1;
         w->call_args[i] = REG_NONE;
         w->call_arg_imms[i] = av->storage.v.imm;
       } else {
         wfail(t,
               "wasm: call arg %u has unsupported operand kind %u (only "
               "REG/IMM scalar args are supported in v1)",
               i, (unsigned)av->storage.kind);
       }
     }
   }
   if (nvar) {
     w->call_var_regs = (Reg*)h->alloc(h, sizeof(Reg) * nvar, _Alignof(Reg));
     w->call_var_imms = (i64*)h->alloc(h, sizeof(i64) * nvar, _Alignof(i64));
     w->call_var_kinds = (u8*)h->alloc(h, nvar, 1);
     w->call_var_types = (KitCgTypeId*)h->alloc(h, sizeof(KitCgTypeId) * nvar,
                                                _Alignof(KitCgTypeId));
     for (u32 i = 0; i < nvar; ++i) {
       const CGABIValue* av = &d->args[nfixed + i];
       const CgType* aty = av->type ? cg_type_get(t->c, av->type) : NULL;
       w->call_var_types[i] = av->type;
       if (aty &&
           (aty->kind == KIT_CG_TYPE_RECORD || aty->kind == KIT_CG_TYPE_ARRAY)) {
         wfail(t, "wasm target: aggregate variadic arg %u not yet supported", i);
       }
       if (av->storage.kind == OPK_REG) {
         w->call_var_kinds[i] = WOP_REG;
         w->call_var_regs[i] = av->storage.v.reg;
         w->call_var_imms[i] = 0;
       } else if (av->storage.kind == OPK_IMM) {
         w->call_var_kinds[i] = WOP_IMM;
         w->call_var_regs[i] = REG_NONE;
         w->call_var_imms[i] = av->storage.v.imm;
       } else {
         wfail(t,
               "wasm target: variadic arg %u has unsupported operand kind %u "
               "(only REG/IMM scalar args supported in v1)",
               i, (unsigned)av->storage.kind);
       }
     }
   }
   if (callee_has_sret) {
     /* The call has no wasm result; the buffer pointed to by the sret arg
      * holds the aggregate. */
     w->dst = REG_NONE;
   } else if (d->ret.storage.kind == OPK_REG &&
              d->ret.storage.v.reg != REG_NONE) {
     w->dst = d->ret.storage.v.reg;
   } else {
     w->dst = REG_NONE;
   }
 }
 
 void wasm_ret(CGTarget* tg, const CGABIValue* v) {
   WTarget* t = (WTarget*)tg;
   if (t->dead) return;
   WIR* w = wir_push(t);
   w->op = WIR_RET;
   if (t->cur_has_sret && v && v->abi && v->abi->kind == ABI_ARG_INDIRECT) {
     /* Aggregate sret return: emit a memcpy from av.storage to the buffer
      * pointed to by the hidden sret parameter, then a void return. */
     w->addr = v->storage;
     w->type = v->type;
     w->agg.size = (u32)abi_cg_sizeof(t->c->abi, v->type);
     w->agg.align = 1u;
     w->cgop = 1; /* tag: sret copy */
     w->dst = REG_NONE;
   } else if (v && v->storage.kind == OPK_REG && v->storage.v.reg != REG_NONE) {
     w->dst = v->storage.v.reg;
     w->type = v->type;
   } else if (v && v->storage.kind == OPK_IMM) {
     w->imm_kind = 1;
     w->imm_a = v->storage.v.imm;
     w->type = v->type;
     w->dst = REG_NONE;
   } else {
     w->dst = REG_NONE;
   }
   t->dead = 1;
 }
 
 void wasm_load(CGTarget* tg, Operand dst, Operand addr, MemAccess mem) {
   WTarget* t = (WTarget*)tg;
   if (t->dead) return;
   if (dst.kind != OPK_REG) wfail(t, "wasm: load dst must be REG");
   if (addr.kind != OPK_LOCAL ||
       slot_for(t, addr.v.frame_slot)->kind == W_SLOT_STACK)
     ensure_linear_memory(t);
   WIR* w = wir_push(t);
   w->op = (addr.kind == OPK_LOCAL &&
            slot_for(t, addr.v.frame_slot)->kind == W_SLOT_LOCAL)
               ? WIR_LOAD_LOCAL
               : WIR_LOAD_MEM;
   w->dst = dst.v.reg;
   w->addr = addr;
   if (addr.kind == OPK_LOCAL) {
     WSlot* s = slot_for(t, addr.v.frame_slot);
     w->imm =
         (w->op == WIR_LOAD_LOCAL) ? (i64)s->wasm_local : (i64)addr.v.frame_slot;
   }
   w->mem = mem;
   w->type = dst.type;
   w->cls = dst.cls;
 }
 
 void wasm_store(CGTarget* tg, Operand addr, Operand src, MemAccess mem) {
   WTarget* t = (WTarget*)tg;
   if (t->dead) return;
   if (addr.kind != OPK_LOCAL ||
       slot_for(t, addr.v.frame_slot)->kind == W_SLOT_STACK)
     ensure_linear_memory(t);
   WIR* w = wir_push(t);
   w->op = (addr.kind == OPK_LOCAL &&
            slot_for(t, addr.v.frame_slot)->kind == W_SLOT_LOCAL)
               ? WIR_STORE_LOCAL
               : WIR_STORE_MEM;
   w->addr = addr;
   if (addr.kind == OPK_LOCAL) {
     WSlot* s = slot_for(t, addr.v.frame_slot);
     w->imm = (w->op == WIR_STORE_LOCAL) ? (i64)s->wasm_local
                                         : (i64)addr.v.frame_slot;
   }
   wir_capture_operand(w, 0, src);
   w->mem = mem;
   /* The store's value type is the accessed type. When storing through a
    * pointer register, addr.type is the (possibly void) pointer rvalue type,
    * not the pointee — so prefer the MemAccess type, which always describes
    * the element being written, before falling back to the operands. */
   w->type = mem.type ? mem.type : (addr.type ? addr.type : src.type);
 }
 
 /* Variadic CG hooks. va_list on wasm32 is a single i32 pointer into a
  * caller-packed buffer of 8-byte slots (see wasm_call's variadic packing).
  * va_start writes the hidden va_ptr param into *ap; va_arg loads T from
  * *ap and advances *ap by 8; va_end is a no-op; va_copy copies the i32. */
 void wasm_va_start(CGTarget* tg, Operand ap_addr) {
   WTarget* t = (WTarget*)tg;
   if (t->dead) return;
   if (!t->cur_is_variadic || t->va_ptr_param_local == 0xffffffffu)
     wfail(t, "wasm: va_start in non-variadic function");
   ensure_linear_memory(t);
   WIR* w = wir_push(t);
   w->op = WIR_VA_START;
   w->addr = ap_addr;
 }
 
 void wasm_va_arg(CGTarget* tg, Operand dst, Operand ap_addr, KitCgTypeId type) {
   WTarget* t = (WTarget*)tg;
   if (t->dead) return;
   if (dst.kind != OPK_REG) wfail(t, "wasm: va_arg dst must be REG");
   const CgType* aty = type ? cg_type_get(t->c, type) : NULL;
   if (aty &&
       (aty->kind == KIT_CG_TYPE_RECORD || aty->kind == KIT_CG_TYPE_ARRAY)) {
     wfail(t, "wasm target: va_arg of aggregate type not yet supported");
   }
   ensure_linear_memory(t);
   WIR* w = wir_push(t);
   w->op = WIR_VA_ARG;
   w->dst = dst.v.reg;
   w->addr = ap_addr;
   w->type = type;
   w->cls = dst.cls;
 }
 
 void wasm_va_end(CGTarget* tg, Operand ap_addr) {
   WTarget* t = (WTarget*)tg;
   (void)ap_addr;
   if (t->dead) return;
   /* No-op: nothing to release. */
 }
 
 void wasm_va_copy(CGTarget* tg, Operand dst_ap_addr, Operand src_ap_addr) {
   WTarget* t = (WTarget*)tg;
   if (t->dead) return;
   ensure_linear_memory(t);
   WIR* w = wir_push(t);
   w->op = WIR_VA_COPY;
   w->addr = dst_ap_addr;
   w->call_sret_addr = src_ap_addr; /* reused slot — see WIR comment */
 }
 
 void wasm_addr_of(CGTarget* tg, Operand dst, Operand lv) {
   WTarget* t = (WTarget*)tg;
   if (t->dead) return;
   if (dst.kind != OPK_REG) wfail(t, "wasm: addr_of dst must be REG");
   if (lv.kind == OPK_LOCAL) {
     WSlot* s = slot_for(t, lv.v.frame_slot);
     if (s->kind == W_SLOT_LOCAL) {
       u32 old_local = s->wasm_local;
       promote_slot_to_stack(t, s);
       WIR* st = wir_push(t);
       st->op = WIR_STORE_MEM;
       st->addr = lv;
       st->type = s->type;
       st->mem.type = s->type;
       st->mem.size = s->size;
       st->mem.align = s->align;
       st->imm_kind = WOP_WASM_LOCAL;
       st->imm_a = old_local;
     }
   } else {
     ensure_linear_memory(t);
   }
   WIR* w = wir_push(t);
   w->op = WIR_ADDR_OF;
   w->dst = dst.v.reg;
   w->addr = lv;
   w->type = dst.type;
   w->cls = dst.cls;
 }
 
 void wasm_alloca(CGTarget* tg, Operand dst, Operand size, u32 align) {
   WTarget* t = (WTarget*)tg;
   if (t->dead) return;
   ensure_linear_memory(t);
   if (dst.kind != OPK_REG) wfail(t, "wasm: alloca dst must be REG");
   ensure_stack_pointer(t);
   t->has_stack_frame = 1;
   WIR* w = wir_push(t);
   w->op = WIR_ALLOCA;
   w->dst = dst.v.reg;
   wir_capture_operand(w, 0, size);
   w->type = dst.type;
   w->type2 = size.type;
   w->cls = dst.cls;
   w->imm = align ? align : 16u;
 }
 
 void wasm_copy_bytes(CGTarget* tg, Operand dst, Operand src,
                      AggregateAccess a) {
   WTarget* t = (WTarget*)tg;
   if (t->dead) return;
   ensure_linear_memory(t);
   WIR* w = wir_push(t);
   w->op = WIR_COPY_BYTES;
   w->addr = dst;
   wir_capture_operand(w, 0, src);
   w->agg = a;
 }
 
 void wasm_set_bytes(CGTarget* tg, Operand dst, Operand byte,
                     AggregateAccess a) {
   WTarget* t = (WTarget*)tg;
   if (t->dead) return;
   WIR* w = wir_push(t);
   w->op = WIR_SET_BYTES;
   w->addr = dst;
   wir_capture_operand(w, 0, byte);
   w->agg = a;
 }
 
 /* Atomic ops. CG forces `addr` to a REG and accepts reg-or-imm for value
  * operands. Wasm only models seq_cst; the KitCgMemOrder argument is captured
  * but not encoded — every emitted atomic op is sequentially consistent. The
  * caller-provided MemAccess carries the type and natural alignment we need
  * for the memarg width. */
 static void atomic_require_addr_reg(WTarget* t, Operand addr,
                                     const char* what) {
   if (addr.kind != OPK_REG)
     wfail(t, "wasm: %s address must be in a register (got opkind %u)", what,
           (unsigned)addr.kind);
 }
 
 void wasm_atomic_load(CGTarget* tg, Operand dst, Operand addr, MemAccess mem,
                       KitCgMemOrder mo) {
   WTarget* t = (WTarget*)tg;
   (void)mo;
   if (t->dead) return;
   if (dst.kind != OPK_REG) wfail(t, "wasm: atomic_load dst must be REG");
   atomic_require_addr_reg(t, addr, "atomic_load");
   ensure_shared_memory(t);
   WIR* w = wir_push(t);
   w->op = WIR_ATOMIC_LOAD;
   w->dst = dst.v.reg;
   w->a = addr.v.reg;
   w->mem = mem;
   w->type = dst.type ? dst.type : mem.type;
   w->cls = dst.cls;
 }
 
 void wasm_atomic_store(CGTarget* tg, Operand addr, Operand src, MemAccess mem,
                        KitCgMemOrder mo) {
   WTarget* t = (WTarget*)tg;
   (void)mo;
   if (t->dead) return;
   atomic_require_addr_reg(t, addr, "atomic_store");
   if (src.kind != OPK_REG && src.kind != OPK_IMM)
     wfail(t, "wasm: atomic_store value must be REG or IMM");
   ensure_shared_memory(t);
   WIR* w = wir_push(t);
   w->op = WIR_ATOMIC_STORE;
   w->a = addr.v.reg;
   wir_capture_operand(w, 1, src);
   w->mem = mem;
   w->type = mem.type ? mem.type : src.type;
 }
 
 void wasm_atomic_rmw(CGTarget* tg, KitCgAtomicOp op, Operand dst, Operand addr,
                      Operand val, MemAccess mem, KitCgMemOrder mo) {
   WTarget* t = (WTarget*)tg;
   (void)mo;
   if (t->dead) return;
   if (dst.kind != OPK_REG) wfail(t, "wasm: atomic_rmw dst must be REG");
   atomic_require_addr_reg(t, addr, "atomic_rmw");
   if (val.kind != OPK_REG && val.kind != OPK_IMM)
     wfail(t, "wasm: atomic_rmw value must be REG or IMM");
   /* KIT_CG_ATOMIC_NAND has no native wasm-threads opcode; the linearizer
    * expands it into an atomic cmpxchg retry loop (see WIR_ATOMIC_RMW). */
   ensure_shared_memory(t);
   WIR* w = wir_push(t);
   w->op = WIR_ATOMIC_RMW;
   w->cgop = (u8)op;
   w->dst = dst.v.reg;
   w->a = addr.v.reg;
   wir_capture_operand(w, 1, val);
   w->mem = mem;
   w->type = dst.type ? dst.type : mem.type;
   w->cls = dst.cls;
 }
 
 void wasm_atomic_cas(CGTarget* tg, Operand prior, Operand ok, Operand addr,
                      Operand expected, Operand desired, MemAccess mem,
                      KitCgMemOrder success, KitCgMemOrder failure) {
   WTarget* t = (WTarget*)tg;
   (void)success;
   (void)failure;
   if (t->dead) return;
   if (prior.kind != OPK_REG) wfail(t, "wasm: atomic_cas prior must be REG");
   if (ok.kind != OPK_REG) wfail(t, "wasm: atomic_cas ok must be REG");
   atomic_require_addr_reg(t, addr, "atomic_cas");
   if (expected.kind != OPK_REG && expected.kind != OPK_IMM)
     wfail(t, "wasm: atomic_cas expected must be REG or IMM");
   if (desired.kind != OPK_REG && desired.kind != OPK_IMM)
     wfail(t, "wasm: atomic_cas desired must be REG or IMM");
   ensure_shared_memory(t);
   WIR* w = wir_push(t);
   w->op = WIR_ATOMIC_CAS;
   w->dst = prior.v.reg;
   w->dst2 = ok.v.reg;
   w->a = addr.v.reg;
   wir_capture_operand(w, 1, expected);
   /* Capture desired into op_c/imm_kind_c/imm_c (third operand slot). */
   if (desired.kind == OPK_REG) {
     w->imm_kind_c = WOP_REG;
     w->op_c = desired.v.reg;
   } else {
     w->imm_kind_c = WOP_IMM;
     w->imm_c = desired.v.imm;
   }
   w->mem = mem;
   w->type = prior.type ? prior.type : mem.type;
   w->cls = prior.cls;
   w->type2 = ok.type;
 }
 
 void wasm_fence(CGTarget* tg, KitCgMemOrder mo) {
   WTarget* t = (WTarget*)tg;
   (void)mo;
   if (t->dead) return;
   /* Wasm atomic.fence does not require a memory to exist, but in practice it
    * is meaningful only inside a module that has shared memory. We don't
    * force-create memory here to avoid producing a bogus memory for fence-only
    * modules. */
   WIR* w = wir_push(t);
   w->op = WIR_FENCE;
 }
 
 /* Forward decls: defined further down. */
 static WasmValType type_valtype(WTarget* t, KitCgTypeId ty);
 void wasm_emit_unreachable(WTarget* t);
 
 /* Per-intrinsic-name diagnostic text. Used both by the recorder for
  * SETJMP/LONGJMP (which we still reject) and for the fallback panic so
  * users see "wasm target: __builtin_clz ..." instead of a numeric kind. */
 static const char* intrin_name(IntrinKind k) {
   switch (k) {
     case INTRIN_NONE:
       return "<none>";
     case INTRIN_POPCOUNT:
       return "__builtin_popcount";
     case INTRIN_CTZ:
       return "__builtin_ctz";
     case INTRIN_CLZ:
       return "__builtin_clz";
     case INTRIN_BSWAP:
       return "__builtin_bswap";
     case INTRIN_SMUL_HIGH:
       return "smul_high";
     case INTRIN_UMUL_HIGH:
       return "umul_high";
     case INTRIN_MEMMOVE:
       return "memmove";
     case INTRIN_PREFETCH:
       return "__builtin_prefetch";
     case INTRIN_ASSUME_ALIGNED:
       return "__builtin_assume_aligned";
     case INTRIN_EXPECT:
       return "__builtin_expect";
     case INTRIN_TRAP:
       return "__builtin_trap";
     case INTRIN_SYSCALL:
       return "__kit_syscall";
     case INTRIN_SETJMP:
       return "setjmp";
     case INTRIN_LONGJMP:
       return "longjmp";
     case INTRIN_SADD_OVERFLOW:
       return "__builtin_sadd_overflow";
     case INTRIN_UADD_OVERFLOW:
       return "__builtin_uadd_overflow";
     case INTRIN_SSUB_OVERFLOW:
       return "__builtin_ssub_overflow";
     case INTRIN_USUB_OVERFLOW:
       return "__builtin_usub_overflow";
     case INTRIN_SMUL_OVERFLOW:
       return "__builtin_smul_overflow";
     case INTRIN_UMUL_OVERFLOW:
       return "__builtin_umul_overflow";
     case INTRIN_CPU_NOP:
       return "cpu_nop";
     case INTRIN_CPU_YIELD:
       return "cpu_yield";
     case INTRIN_WFI:
       return "wfi";
     case INTRIN_WFE:
       return "wfe";
     case INTRIN_SEV:
       return "sev";
     case INTRIN_ISB:
       return "isb";
     case INTRIN_DMB:
       return "dmb";
     case INTRIN_DSB:
       return "dsb";
     case INTRIN_IRQ_SAVE:
       return "irq_save";
     case INTRIN_IRQ_RESTORE:
       return "irq_restore";
     case INTRIN_IRQ_ENABLE:
       return "irq_enable";
     case INTRIN_IRQ_DISABLE:
       return "irq_disable";
     case INTRIN_FRAME_ADDRESS:
       return "frame_address";
     case INTRIN_RETURN_ADDRESS:
       return "return_address";
     case INTRIN_READCYCLECOUNTER:
       return "readcyclecounter";
     case INTRIN_STACK_GUARD:
       return "stack_guard";
   }
   return "<unknown>";
 }
 
 void wasm_intrinsic(CGTarget* tg, IntrinKind k, Operand* dst, u32 ndst,
                     const Operand* args, u32 nargs) {
   WTarget* t = (WTarget*)tg;
   if (t->dead) return;
 
   switch (k) {
     case INTRIN_TRAP:
       wasm_emit_unreachable(t);
       return;
 
     case INTRIN_PREFETCH:
       /* No-op hint. */
       return;
 
     case INTRIN_CPU_YIELD:
       /* Portable spin-loop hint; wasm has no corresponding core instruction. */
       return;
 
     case INTRIN_EXPECT:
     case INTRIN_ASSUME_ALIGNED:
       /* Pass-through hint: result = first argument. CG always allocates a
        * dst reg for these; copy arg[0] there so downstream uses see the
        * expected value. CG keeps the first arg as OPK_IMM when it was a
        * literal so the constant flows through unchanged. */
       if (ndst == 1 && nargs >= 1) {
         if (args[0].kind == OPK_IMM) {
           wasm_load_imm(tg, dst[0], args[0].v.imm);
         } else {
           wasm_copy(tg, dst[0], args[0]);
         }
       }
       return;
 
     case INTRIN_MEMMOVE: {
       /* memmove lowers to memory.copy, which is spec-defined to handle overlap
        * correctly. CG forces (dst, src) to REG and passes size as OPK_IMM
        * (kit_cg_memmove). */
       if (nargs != 3 || args[0].kind != OPK_REG || args[1].kind != OPK_REG) {
         compiler_panic(t->c, cur_loc(t),
                        "wasm target: %s requires register pointers",
                        intrin_name(k));
         return;
       }
       if (args[2].kind != OPK_IMM) {
         compiler_panic(t->c, cur_loc(t),
                        "wasm target: %s with non-constant size is not yet "
                        "supported",
                        intrin_name(k));
         return;
       }
       ensure_linear_memory(t);
       AggregateAccess a;
       memset(&a, 0, sizeof a);
       a.size = (u32)args[2].v.imm;
       a.align = 1;
       WIR* w = wir_push(t);
       w->op = WIR_COPY_BYTES;
       w->addr = args[0];
       wir_capture_operand(w, 0, args[1]);
       w->agg = a;
       return;
     }
 
     case INTRIN_CLZ:
     case INTRIN_CTZ:
     case INTRIN_POPCOUNT:
     case INTRIN_BSWAP: {
       if (ndst != 1 || nargs != 1 || dst[0].kind != OPK_REG ||
           args[0].kind != OPK_REG) {
         compiler_panic(t->c, cur_loc(t),
                        "wasm target: %s requires single REG operand",
                        intrin_name(k));
         return;
       }
       WIR* w = wir_push(t);
       w->op = WIR_INTRINSIC;
       w->cgop = (u8)k;
       w->dst = dst[0].v.reg;
       w->a = args[0].v.reg;
       w->type = dst[0].type;
       /* clz/ctz/popcount return int (i32) but operate at the operand's width
        * (e.g. __builtin_ctzl over an i64). The wasm op width must follow the
        * operand, with a wrap to the i32 dst afterward. type2 carries it. */
       w->type2 = args[0].type;
       w->cls = dst[0].cls;
       return;
     }
 
     case INTRIN_SADD_OVERFLOW:
     case INTRIN_UADD_OVERFLOW:
     case INTRIN_SSUB_OVERFLOW:
     case INTRIN_USUB_OVERFLOW:
     case INTRIN_SMUL_OVERFLOW:
     case INTRIN_UMUL_OVERFLOW: {
       if (ndst != 2 || nargs != 2 || dst[0].kind != OPK_REG ||
           dst[1].kind != OPK_REG) {
         compiler_panic(t->c, cur_loc(t),
                        "wasm target: %s requires 2 args + 2 result regs",
                        intrin_name(k));
         return;
       }
       /* Reject i64 mul-overflow for now: wasm core has no widening 64x64
        * multiply, so the standard expansion would need partial-product
        * synthesis. 32-bit (the common shape on wasm32) is supported. */
       WasmValType vt = type_valtype(t, dst[0].type);
       if (vt == WASM_VAL_I64 &&
           (k == INTRIN_SMUL_OVERFLOW || k == INTRIN_UMUL_OVERFLOW)) {
         compiler_panic(t->c, cur_loc(t),
                        "wasm target: 64-bit checked-overflow multiply is "
                        "not yet supported");
         return;
       }
       WIR* w = wir_push(t);
       w->op = WIR_INTRINSIC;
       w->cgop = (u8)k;
       w->dst = dst[0].v.reg;
       w->dst2 = dst[1].v.reg;
       w->type = dst[0].type;
       w->cls = dst[0].cls;
       wir_capture_operand(w, 0, args[0]);
       wir_capture_operand(w, 1, args[1]);
       return;
     }
 
     case INTRIN_SMUL_HIGH:
     case INTRIN_UMUL_HIGH: {
       if (ndst != 1 || nargs != 2 || dst[0].kind != OPK_REG) {
         compiler_panic(t->c, cur_loc(t),
                        "wasm target: %s requires 2 args + 1 result reg",
                        intrin_name(k));
         return;
       }
       WIR* w = wir_push(t);
       w->op = WIR_INTRINSIC;
       w->cgop = (u8)k;
       w->dst = dst[0].v.reg;
       w->type = dst[0].type;
       w->cls = dst[0].cls;
       wir_capture_operand(w, 0, args[0]);
       wir_capture_operand(w, 1, args[1]);
       return;
     }
 
     case INTRIN_SETJMP:
     case INTRIN_LONGJMP:
       compiler_panic(t->c, cur_loc(t),
                      "wasm target: %s is not yet supported (no exception/"
                      "stack-unwind runtime)",
                      intrin_name(k));
       return;
 
     /* Baremetal/CPU-control intrinsics have no wasm lowering;
      * kit_cg_target_supports_intrinsic reports them false so frontends
      * diagnose before reaching here. Fall through to the generic panic. */
     case INTRIN_CPU_NOP:
     case INTRIN_WFI:
     case INTRIN_WFE:
     case INTRIN_SEV:
     case INTRIN_ISB:
     case INTRIN_DMB:
     case INTRIN_DSB:
     case INTRIN_IRQ_SAVE:
     case INTRIN_IRQ_RESTORE:
     case INTRIN_IRQ_ENABLE:
     case INTRIN_IRQ_DISABLE:
     case INTRIN_SYSCALL:
     case INTRIN_READCYCLECOUNTER:
     case INTRIN_STACK_GUARD:
     /* No frame-pointer chain in wasm; reported unsupported up front. */
     case INTRIN_FRAME_ADDRESS:
     case INTRIN_RETURN_ADDRESS:
     case INTRIN_NONE:
       break;
   }
   compiler_panic(t->c, cur_loc(t),
                  "wasm target: intrinsic %s not yet implemented",
                  intrin_name(k));
 }
 
 /* Inline asm v1 — see doc/WASM.md "Inline asm" for the surface contract.
  *
  * Template syntax: WAT instruction sequence. Inputs pre-pushed to the value
  * stack (via local.get on synthetic input locals); outputs popped from the
  * stack into synthetic output locals after the body. The snippet's
  * local.get/set/tee with index < nin refers to the i-th input.
  *
  * Constraints: "r" → wasm local; "i" → const-folded; "m" → i32 address.
  * Numeric tie-back constraints ("0","1",...) reuse the referenced output's
  * slot (cg/asm.c expands them into duplicate input entries at the end).
  *
  * Disallowed in v1: escaping br/br_if/br_table, return/return_call*,
  * call_indirect, snippet-internal locals, output count > 1, register
  * clobbers (only `memory` is accepted). */
 static WasmValType wasm_asm_operand_vt(WTarget* t, KitCgTypeId ty,
                                        const char* what, SrcLoc loc) {
   WasmValType vt;
   if (!ty) wfail_at(t, loc, "wasm target: asm %s with no CG type", what);
   vt = valtype_for_type(t, ty);
   if (vt != WASM_VAL_I32 && vt != WASM_VAL_I64 && vt != WASM_VAL_F32 &&
       vt != WASM_VAL_F64)
     wfail_at(t, loc, "wasm target: asm %s of non-scalar type not supported",
              what);
   return vt;
 }
 
 void wasm_asm_block(CGTarget* tg, const char* tmpl, const AsmConstraint* outs,
                     u32 nout, Operand* out_ops, const AsmConstraint* ins,
                     u32 nin, const Operand* in_ops, const Sym* clob,
                     u32 nclob) {
   WTarget* t = (WTarget*)tg;
   Heap* h = t->c->ctx->heap;
   Sym sym_memory;
   WasmFunc scratch;
   SrcLoc loc = cur_loc(t);
   u32 depth;
   u32 i;
 
   if (t->dead) return;
 
   /* Clobber policy: only `memory` is meaningful on wasm (effective no-op
    * because cg/asm.c already spilled live SSA values). Reject named-register
    * clobbers explicitly. */
   sym_memory = pool_intern_slice(t->c->global, SLICE_LIT("memory"));
   for (i = 0; i < nclob; ++i) {
     if (clob[i] != sym_memory)
       wfail_at(t, loc, "wasm target: asm register clobbers not yet supported");
   }
   for (i = 0; i < nout; ++i) {
     if (outs[i].reg)
       wfail_at(t, loc, "wasm target: asm hard-register operands not supported");
   }
   for (i = 0; i < nin; ++i) {
     if (ins[i].reg)
       wfail_at(t, loc, "wasm target: asm hard-register operands not supported");
   }
 
   /* Build a scratch WasmFunc with the synthetic signature. Layout is:
    *   params  = input types (indices 0 .. nin-1)
    *   locals  = output types (indices nin .. nin+nout-1)
    *   results = empty
    * Snippets use local.get/set/tee N to read/write either side. The author
    * is responsible for writing each output via local.set N (>= nin); empty
    * snippets paired with `+r` / numeric tieback constraints get identity
    * behavior because the input and output share a wasm local at lowering. */
   memset(&scratch, 0, sizeof scratch);
   for (i = 0; i < nin; ++i) {
     WasmValType vt = wasm_asm_operand_vt(t, ins[i].type, "input operand", loc);
     /* Constraint-specific checks: "i" requires an immediate operand; "m"
      * requires an indirect (i32 address). */
     if (ins[i].str && ins[i].str[0] == 'i' && in_ops[i].kind != OPK_IMM)
       wfail_at(t, loc, "wasm target: asm 'i' input must be an immediate");
     if (ins[i].str && ins[i].str[0] == 'm') {
       if (in_ops[i].kind != OPK_INDIRECT)
         wfail_at(t, loc, "wasm target: asm 'm' input must be indirect");
       vt = WASM_VAL_I32;
     }
     wasm_func_push_param(t->c, t->module, &scratch, vt);
   }
   for (i = 0; i < nout; ++i) {
     WasmValType vt =
         wasm_asm_operand_vt(t, outs[i].type, "output operand", loc);
     wasm_func_push_local(t->c, t->module, &scratch, vt);
   }
   /* No declared result — outputs are read from locals at the end of the
    * lowering, not popped from the value stack. */
 
   /* Parse the template into scratch.insns. */
   wasm_parse_wat_body(t->c, t->module, &scratch, tmpl, strlen(tmpl), loc);
 
   /* Walk the parsed body to reject constructs that escape or aren't
    * supported in v1. Track control depth so br/br_if/br_table with imm >=
    * depth (i.e. would branch out of the snippet) are rejected. */
   depth = 0;
   for (i = 0; i < scratch.ninsns; ++i) {
     WasmInsn* in = &scratch.insns[i];
     switch (in->kind) {
       case WASM_INSN_BLOCK:
       case WASM_INSN_LOOP:
       case WASM_INSN_IF:
         depth++;
         break;
       case WASM_INSN_END:
         if (depth) depth--;
         break;
       case WASM_INSN_BR:
       case WASM_INSN_BR_IF:
         if (in->imm < 0 || (u64)in->imm >= depth)
           wfail_at(t, in->loc,
                    "wasm target: asm template branch escapes snippet");
         break;
       case WASM_INSN_BR_TABLE: {
         u32 k;
         for (k = 0; k < in->ntargets; ++k)
           if (in->targets[k] >= depth)
             wfail_at(t, in->loc,
                      "wasm target: asm template br_table escapes snippet");
         break;
       }
       case WASM_INSN_RETURN:
       case WASM_INSN_RETURN_CALL:
       case WASM_INSN_RETURN_CALL_INDIRECT:
       case WASM_INSN_RETURN_CALL_REF:
         wfail_at(t, in->loc,
                  "wasm target: return/tail-call in asm template not "
                  "supported");
         break;
       case WASM_INSN_LOCAL_GET:
       case WASM_INSN_LOCAL_SET:
       case WASM_INSN_LOCAL_TEE:
         if (in->imm < 0 || (u64)in->imm >= (u64)(nin + nout))
           wfail_at(t, in->loc,
                    "wasm target: asm template references local beyond "
                    "declared operands (snippet-internal locals not "
                    "supported)");
         break;
       default:
         break;
     }
   }
 
   /* Validate the body against the synthetic signature. */
   wasm_validate_func(t->c, t->module, &scratch);
 
   {
     /* Build the WIR_ASM_BLOCK payload. raw_insns is owned by the WIR; the
      * other arrays are too. Sizes are zero-when-empty per the WIR teardown
      * conventions. */
     WIR* w = wir_push(t);
     w->op = WIR_ASM_BLOCK;
     w->raw_ninsns = scratch.ninsns;
     if (scratch.ninsns) {
       w->raw_insns = (WasmInsn*)h->alloc(h, sizeof(WasmInsn) * scratch.ninsns,
                                          _Alignof(WasmInsn));
       if (!w->raw_insns) wfail(t, "wasm: out of memory");
       memcpy(w->raw_insns, scratch.insns, sizeof(WasmInsn) * scratch.ninsns);
     }
     w->asm_nin = nin;
     w->asm_nout = nout;
     if (nin) {
       w->asm_in_kinds = (u8*)h->alloc(h, nin, 1);
       w->asm_in_imms = (i64*)h->alloc(h, sizeof(i64) * nin, _Alignof(i64));
       w->asm_in_regs = (Reg*)h->alloc(h, sizeof(Reg) * nin, _Alignof(Reg));
       w->asm_in_types = (KitCgTypeId*)h->alloc(h, sizeof(KitCgTypeId) * nin,
                                                _Alignof(KitCgTypeId));
       w->asm_in_share_out = (i32*)h->alloc(h, sizeof(i32) * nin, _Alignof(i32));
       if (!w->asm_in_kinds || !w->asm_in_imms || !w->asm_in_regs ||
           !w->asm_in_types || !w->asm_in_share_out)
         wfail(t, "wasm: out of memory");
       for (i = 0; i < nin; ++i) w->asm_in_share_out[i] = -1;
       for (i = 0; i < nin; ++i) {
         Operand op = in_ops[i];
         /* Numeric tieback constraints ("0".."9") share the matching
          * output's wasm local. cg/asm.c also rewrites +r inout outputs
          * into duplicate inputs at the tail of ins[], using the same
          * numeric encoding. */
         const char* s = ins[i].str;
         if (s && s[0] >= '0' && s[0] <= '9' && s[1] == '\0') {
           int idx = s[0] - '0';
           if ((u32)idx < nout) w->asm_in_share_out[i] = idx;
         }
         switch (op.kind) {
           case OPK_REG:
             w->asm_in_kinds[i] = WOP_REG;
             w->asm_in_regs[i] = op.v.reg;
             w->asm_in_imms[i] = 0;
             w->asm_in_types[i] = op.type ? op.type : ins[i].type;
             break;
           case OPK_IMM:
             w->asm_in_kinds[i] = WOP_IMM;
             w->asm_in_regs[i] = REG_NONE;
             w->asm_in_imms[i] = op.v.imm;
             w->asm_in_types[i] = op.type ? op.type : ins[i].type;
             break;
           case OPK_INDIRECT:
             /* For "m" constraint: the input local holds the i32 address
              * `base + ofs` of the lvalue. We re-use asm_in_imms (unused
              * for WOP_REG operands) to carry the displacement so the
              * linearizer can splice in `i32.const ofs; i32.add` after
              * pushing the base local. */
             w->asm_in_kinds[i] = WOP_REG;
             w->asm_in_regs[i] = op.v.ind.base;
             w->asm_in_imms[i] = (i64)op.v.ind.ofs;
             w->asm_in_types[i] = builtin_id(KIT_CG_BUILTIN_I32);
             break;
           default:
             wfail_at(t, loc, "wasm target: unsupported asm input operand kind");
         }
       }
     }
     if (nout) {
       w->asm_out_regs = (Reg*)h->alloc(h, sizeof(Reg) * nout, _Alignof(Reg));
       w->asm_out_types = (KitCgTypeId*)h->alloc(h, sizeof(KitCgTypeId) * nout,
                                                 _Alignof(KitCgTypeId));
       if (!w->asm_out_regs || !w->asm_out_types)
         wfail(t, "wasm: out of memory");
       for (i = 0; i < nout; ++i) {
         if (out_ops[i].kind != OPK_REG)
           wfail_at(t, loc, "wasm target: asm output must be a register");
         w->asm_out_regs[i] = out_ops[i].v.reg;
         w->asm_out_types[i] = out_ops[i].type ? out_ops[i].type : outs[i].type;
       }
     }
   }
 
   /* Free scratch func storage. The parsed insns have been copied into the
    * WIR payload. */
   if (scratch.params)
     h->free(h, scratch.params, sizeof(WasmValType) * scratch.cap_params);
   if (scratch.locals)
     h->free(h, scratch.locals, sizeof(WasmValType) * scratch.cap_locals);
   if (scratch.insns)
     h->free(h, scratch.insns, sizeof(WasmInsn) * scratch.cap_insns);
 }
 
 void wasm_file_scope_asm(CGTarget* tg, const char* src, size_t len) {
   WTarget* t = (WTarget*)tg;
   (void)src;
   (void)len;
   compiler_panic(t->c, cur_loc(t),
                  "wasm target: file-scope asm not yet supported");
 }
 
 void wasm_set_loc(CGTarget* tg, SrcLoc loc) {
   WTarget* t = (WTarget*)tg;
   /* No debug info in v1, but we stash the most recent loc so cur_loc /
    * diagnostics attribute to the actual statement rather than the
    * function-definition location. */
   t->cur_stmt_loc = loc;
 }
 
 void wasm_emit_unreachable(WTarget* t) {
   if (t->dead) return;
   WIR* w = wir_push(t);
   w->op = WIR_UNREACHABLE;
   t->dead = 1;
 }
 
 /* Control terminator (the C __builtin_unreachable point): emit the Wasm
  * `unreachable` opcode, which traps if reached. Ends the current block. */
 void wasm_unreachable(CGTarget* tg) { wasm_emit_unreachable((WTarget*)tg); }
 
 /* -----------------------------------------------------------------
  * WIR -> WasmFunc lowering
  * ----------------------------------------------------------------- */
 
 static void emit_insn(WTarget* t, WasmInsnKind k, i64 imm) {
   wasm_func_add_insn(t->c, t->module, t->cur_func, k, imm);
 }
 static void emit_fp(WTarget* t, WasmInsnKind k, double v) {
   wasm_func_add_fp_insn(t->c, t->module, t->cur_func, k, v);
 }
 
 /* Push an operand onto the wasm value stack. */
 static void emit_push_operand_reg(WTarget* t, Reg r) {
   if (r == REG_NONE) wfail(t, "wasm: push of REG_NONE");
   /* The reg must already have a local. */
   if (r >= t->reg_cap || t->reg_to_local[r] == 0xffffffffu) {
     wfail(t, "wasm: reg %u used before being defined", (unsigned)r);
   }
   emit_insn(t, WASM_INSN_LOCAL_GET, (i64)t->reg_to_local[r]);
 }
 
 static WasmValType type_valtype(WTarget* t, KitCgTypeId ty) {
   return valtype_for_type(t, ty);
 }
 
 static void emit_push_imm(WTarget* t, WasmValType vt, i64 imm) {
   WasmInsnKind k =
       (vt == WASM_VAL_I64) ? WASM_INSN_I64_CONST : WASM_INSN_I32_CONST;
   emit_insn(t, k, imm);
 }
 
 static u32 memarg_align_log2(u32 align, u32 width);
 static WasmInsnKind load_kind_for(WTarget* t, KitCgTypeId ty, MemAccess ma);
 
 static void emit_push_operand(WTarget* t, u32 kind, i64 imm, Reg r,
                               KitCgTypeId ty) {
   if (kind == WOP_IMM) {
     WasmValType vt = type_valtype(t, ty);
     if (vt == WASM_VAL_F32 || vt == WASM_VAL_F64) {
       wfail(t, "wasm: float immediate operand not supported");
     }
     emit_push_imm(t, vt, imm);
   } else if (kind == WOP_LOCAL) {
     FrameSlot fs = (FrameSlot)imm;
     WSlot* s = slot_for(t, fs);
     if (s->kind == W_SLOT_LOCAL) {
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)s->wasm_local);
     } else {
       MemAccess ma;
       memset(&ma, 0, sizeof ma);
       ma.type = ty;
       ma.size = (u32)abi_cg_sizeof(t->c->abi, ty);
       ma.align = s->align;
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)t->frame_base_local);
       WasmInsnKind k = load_kind_for(t, ty, ma);
       wasm_func_add_mem_insn(
           t->c, t->module, t->cur_func, k,
           memarg_align_log2(ma.align, wasm_mem_width((uint8_t)k)),
           s->frame_offset, 0);
     }
   } else {
     emit_push_operand_reg(t, r);
   }
 }
 
 static void emit_local_set(WTarget* t, Reg dst, KitCgTypeId ty, RegClass cls) {
   u32 idx = reg_local(t, dst, ty, cls);
   emit_insn(t, WASM_INSN_LOCAL_SET, (i64)idx);
 }
 
 static u32 memarg_align_log2(u32 align, u32 width) {
   u32 a = align ? align : width;
   u32 lg = 0;
   if (a > width) a = width;
   while (a > 1u) {
     a >>= 1u;
     lg++;
   }
   return lg;
 }
 
 static WasmInsnKind load_kind_for(WTarget* t, KitCgTypeId ty, MemAccess ma) {
   WasmValType vt = type_valtype(t, ty);
   u32 size = ma.size ? ma.size : (u32)abi_cg_sizeof(t->c->abi, ty);
   if (vt == WASM_VAL_F32) return WASM_INSN_F32_LOAD;
   if (vt == WASM_VAL_F64) return WASM_INSN_F64_LOAD;
   if (vt == WASM_VAL_I64) {
     if (size == 1) return WASM_INSN_I64_LOAD8_U;
     if (size == 2) return WASM_INSN_I64_LOAD16_U;
     if (size == 4) return WASM_INSN_I64_LOAD32_U;
     return WASM_INSN_I64_LOAD;
   }
   if (size == 1) return WASM_INSN_I32_LOAD8_U;
   if (size == 2) return WASM_INSN_I32_LOAD16_U;
   return WASM_INSN_I32_LOAD;
 }
 
 static WasmInsnKind store_kind_for(WTarget* t, KitCgTypeId ty, MemAccess ma) {
   WasmValType vt = type_valtype(t, ty);
   u32 size = ma.size ? ma.size : (u32)abi_cg_sizeof(t->c->abi, ty);
   if (vt == WASM_VAL_F32) return WASM_INSN_F32_STORE;
   if (vt == WASM_VAL_F64) return WASM_INSN_F64_STORE;
   if (vt == WASM_VAL_I64) {
     if (size == 1) return WASM_INSN_I64_STORE8;
     if (size == 2) return WASM_INSN_I64_STORE16;
     if (size == 4) return WASM_INSN_I64_STORE32;
     return WASM_INSN_I64_STORE;
   }
   if (size == 1) return WASM_INSN_I32_STORE8;
   if (size == 2) return WASM_INSN_I32_STORE16;
   return WASM_INSN_I32_STORE;
 }
 
 /* Atomic op selection. Wasm threads gives natural-width atomic load/store for
  * i32/i64 (with 8/16/32 subword variants) but only full-width (i32/i64) RMW
  * and cmpxchg in kit's wasm core. Sub-word RMW/cmpxchg therefore diagnose
  * rather than silently widening. */
 static WasmInsnKind atomic_load_kind_for(WTarget* t, KitCgTypeId ty,
                                          MemAccess ma) {
   WasmValType vt = type_valtype(t, ty);
   u32 size = ma.size ? ma.size : (u32)abi_cg_sizeof(t->c->abi, ty);
   if (vt == WASM_VAL_F32 || vt == WASM_VAL_F64)
     wfail(t,
           "wasm target: atomic load of floating-point value is not "
           "representable in wasm threads");
   if (vt == WASM_VAL_I64) {
     if (size == 1) return WASM_INSN_I64_ATOMIC_LOAD8_U;
     if (size == 2) return WASM_INSN_I64_ATOMIC_LOAD16_U;
     if (size == 4) return WASM_INSN_I64_ATOMIC_LOAD32_U;
     if (size == 8) return WASM_INSN_I64_ATOMIC_LOAD;
     wfail(t, "wasm: atomic load i64 size %u not supported", size);
   }
   if (size == 1) return WASM_INSN_I32_ATOMIC_LOAD8_U;
   if (size == 2) return WASM_INSN_I32_ATOMIC_LOAD16_U;
   if (size == 4) return WASM_INSN_I32_ATOMIC_LOAD;
   wfail(t, "wasm: atomic load i32 size %u not supported", size);
 }
 
 static WasmInsnKind atomic_store_kind_for(WTarget* t, KitCgTypeId ty,
                                           MemAccess ma) {
   WasmValType vt = type_valtype(t, ty);
   u32 size = ma.size ? ma.size : (u32)abi_cg_sizeof(t->c->abi, ty);
   if (vt == WASM_VAL_F32 || vt == WASM_VAL_F64)
     wfail(t,
           "wasm target: atomic store of floating-point value is not "
           "representable in wasm threads");
   if (vt == WASM_VAL_I64) {
     if (size == 1) return WASM_INSN_I64_ATOMIC_STORE8;
     if (size == 2) return WASM_INSN_I64_ATOMIC_STORE16;
     if (size == 4) return WASM_INSN_I64_ATOMIC_STORE32;
     if (size == 8) return WASM_INSN_I64_ATOMIC_STORE;
     wfail(t, "wasm: atomic store i64 size %u not supported", size);
   }
   if (size == 1) return WASM_INSN_I32_ATOMIC_STORE8;
   if (size == 2) return WASM_INSN_I32_ATOMIC_STORE16;
   if (size == 4) return WASM_INSN_I32_ATOMIC_STORE;
   wfail(t, "wasm: atomic store i32 size %u not supported", size);
 }
 
 static WasmInsnKind atomic_rmw_kind_for(WTarget* t, KitCgAtomicOp op,
                                         KitCgTypeId ty, MemAccess ma) {
   WasmValType vt = type_valtype(t, ty);
   u32 size = ma.size ? ma.size : (u32)abi_cg_sizeof(t->c->abi, ty);
   int is64 = (vt == WASM_VAL_I64);
   if (vt == WASM_VAL_F32 || vt == WASM_VAL_F64)
     wfail(t,
           "wasm target: atomic RMW on floating-point value is not "
           "representable in wasm threads");
   if (!(size == 4 || size == 8) || (is64 && size != 8) ||
       (!is64 && size != 4)) {
     wfail(t,
           "wasm target: atomic RMW size %u not yet supported (only "
           "full-width i32 and i64 atomic RMW are wired)",
           size);
   }
   switch (op) {
     case KIT_CG_ATOMIC_ADD:
       return is64 ? WASM_INSN_I64_ATOMIC_RMW_ADD : WASM_INSN_I32_ATOMIC_RMW_ADD;
     case KIT_CG_ATOMIC_SUB:
       return is64 ? WASM_INSN_I64_ATOMIC_RMW_SUB : WASM_INSN_I32_ATOMIC_RMW_SUB;
     case KIT_CG_ATOMIC_AND:
       return is64 ? WASM_INSN_I64_ATOMIC_RMW_AND : WASM_INSN_I32_ATOMIC_RMW_AND;
     case KIT_CG_ATOMIC_OR:
       return is64 ? WASM_INSN_I64_ATOMIC_RMW_OR : WASM_INSN_I32_ATOMIC_RMW_OR;
     case KIT_CG_ATOMIC_XOR:
       return is64 ? WASM_INSN_I64_ATOMIC_RMW_XOR : WASM_INSN_I32_ATOMIC_RMW_XOR;
     case KIT_CG_ATOMIC_XCHG:
       return is64 ? WASM_INSN_I64_ATOMIC_RMW_XCHG
                   : WASM_INSN_I32_ATOMIC_RMW_XCHG;
     case KIT_CG_ATOMIC_NAND:
       wfail(t, "wasm target: atomic NAND has no native wasm-threads opcode");
   }
   wfail(t, "wasm: unsupported atomic RMW op %d", (int)op);
 }
 
 static WasmInsnKind atomic_cmpxchg_kind_for(WTarget* t, KitCgTypeId ty,
                                             MemAccess ma) {
   WasmValType vt = type_valtype(t, ty);
   u32 size = ma.size ? ma.size : (u32)abi_cg_sizeof(t->c->abi, ty);
   if (vt == WASM_VAL_F32 || vt == WASM_VAL_F64)
     wfail(t,
           "wasm target: atomic cmpxchg on floating-point value is not "
           "representable in wasm threads");
   if (vt == WASM_VAL_I64) {
     if (size != 8)
       wfail(t, "wasm target: atomic cmpxchg i64 size %u not yet supported",
             size);
     return WASM_INSN_I64_ATOMIC_RMW_CMPXCHG;
   }
   if (size != 4)
     wfail(t, "wasm target: atomic cmpxchg i32 size %u not yet supported", size);
   return WASM_INSN_I32_ATOMIC_RMW_CMPXCHG;
 }
 
 /* Look up (or assign) `sym`'s slot in the funcref table. Returned index is
  * the wasm table index (>= 1, with 0 reserved as the null/trap slot). */
 static u32 func_table_index_for(WTarget* t, ObjSymId sym) {
   Heap* h = t->c->ctx->heap;
   for (u32 i = 0; i < t->func_table_count; ++i) {
     if (t->func_table[i] == sym) return i + 1u;
   }
   if (t->func_table_count == t->func_table_cap) {
     u32 nc = t->func_table_cap ? t->func_table_cap * 2u : 8u;
     void* p = h->realloc(h, t->func_table, sizeof(ObjSymId) * t->func_table_cap,
                          sizeof(ObjSymId) * nc, _Alignof(ObjSymId));
     if (!p) wfail(t, "wasm: out of memory");
     t->func_table = (ObjSymId*)p;
     t->func_table_cap = nc;
   }
   t->func_table[t->func_table_count] = sym;
   t->has_func_table = 1;
   return ++t->func_table_count; /* slot 0 reserved; first sym -> index 1 */
 }
 
 /* Defer function-pointer materialization to wasm_materialize_functable.
  * Emits `i32.const 0` and queues a WFuncTableFixup keyed by the placeholder's
  * (cur_func_idx, ninsns-1). */
 static void queue_func_table_fixup(WTarget* t, ObjSymId sym) {
   Heap* h = t->c->ctx->heap;
   if (!t->cur_func) wfail(t, "wasm: function address outside a function");
   /* Ensure the function gets a slot and force the WasmFunc shell to exist so
    * the table's element segment can resolve its wasm-func index later. */
   (void)func_table_index_for(t, sym);
   (void)sym_to_wasm_func(t, sym, NULL);
   emit_insn(t, WASM_INSN_I32_CONST, 0);
   if (t->func_table_fixups_count == t->func_table_fixups_cap) {
     u32 nc = t->func_table_fixups_cap ? t->func_table_fixups_cap * 2u : 16u;
     void* p =
         h->realloc(h, t->func_table_fixups,
                    sizeof(WFuncTableFixup) * t->func_table_fixups_cap,
                    sizeof(WFuncTableFixup) * nc, _Alignof(WFuncTableFixup));
     if (!p) wfail(t, "wasm: out of memory");
     t->func_table_fixups = (WFuncTableFixup*)p;
     t->func_table_fixups_cap = nc;
   }
   WFuncTableFixup* fx = &t->func_table_fixups[t->func_table_fixups_count++];
   fx->wasm_func_idx = t->cur_func_idx;
   fx->insn_idx = t->cur_func->ninsns - 1u;
   fx->sym = sym;
 }
 
 /* Defer symbol-address resolution to wasm_materialize_data. Emits an
  * i32.const placeholder into the current function and queues a WSymFixup
  * keyed by (cur_func_idx, ninsns-1). The compact section layout is only
  * known once every section's final size is settled, which doesn't happen
  * until finalize. */
 static void queue_symbol_addr_fixup(WTarget* t, ObjSymId sym, i64 addend) {
   Heap* h = t->c->ctx->heap;
   const ObjSym* os = obj_symbol_get(t->obj, sym);
   /* Function-symbol addresses route through the funcref table, not linear
    * memory. CG occasionally takes the address of an extern function before
    * the function body is seen (forward declarations, indirect-call
    * setup); the SK_FUNC kind is set by the frontend at sym creation. */
   if (os && os->kind == SK_FUNC) {
     if (addend != 0)
       wfail(t, "wasm: nonzero addend on function-pointer reference");
     queue_func_table_fixup(t, sym);
     return;
   }
   if (!os)
     wfail(t, "wasm target: address of unresolved symbol not yet implemented");
   if (os->section_id == OBJ_SEC_NONE && os->kind != SK_COMMON)
     wfail(t, "wasm target: address of undefined symbol not yet implemented");
   /* SK_COMMON falls through: apply_sym_fixups allocates a BSS-style base for
    * it in wasm_materialize_data and patches the i32.const at finalize. */
   if (addend < INT32_MIN || addend > INT32_MAX)
     wfail(t, "wasm: symbol addend out of range");
   if (!t->cur_func) wfail(t, "wasm: symbol address outside a function");
   emit_insn(t, WASM_INSN_I32_CONST, 0);
   if (t->sym_fixups_count == t->sym_fixups_cap) {
     u32 nc = t->sym_fixups_cap ? t->sym_fixups_cap * 2u : 16u;
     void* p =
         h->realloc(h, t->sym_fixups, sizeof(WSymFixup) * t->sym_fixups_cap,
                    sizeof(WSymFixup) * nc, _Alignof(WSymFixup));
     if (!p) wfail(t, "wasm: out of memory");
     t->sym_fixups = (WSymFixup*)p;
     t->sym_fixups_cap = nc;
   }
   WSymFixup* fx = &t->sym_fixups[t->sym_fixups_count++];
   fx->wasm_func_idx = t->cur_func_idx;
   fx->insn_idx = t->cur_func->ninsns - 1u;
   fx->sym = sym;
   fx->addend = addend;
 }
 
 /* Push the value of an OPK_INDIRECT base/index component. The CG defers loading
  * the pointer value of an address-taken (frame-resident) pointer local to the
  * backend: the deref of such a local arrives as an OPK_INDIRECT whose base
  * names the local itself, not a materialized register (see
  * fold_ea_into_operand, and native_direct_target's nd_cache_reg_for, which
  * loads it from the home). In the wasm backend each id is either a register
  * (reg_to_local set) or a frame slot, never both — so dispatch on that: a
  * register is fetched directly; a frame-resident local is read from its home
  * like any other WOP_LOCAL operand. */
 static void emit_push_addr_component(WTarget* t, Reg id) {
   if (id < t->reg_cap && t->reg_to_local[id] != 0xffffffffu) {
     emit_push_operand_reg(t, id);
   } else {
     WSlot* s = slot_for(t, id);
     emit_push_operand(t, WOP_LOCAL, (i64)id, REG_NONE, s->type);
   }
 }
 
 /* Value type of an indirect component, whether it lives in a register or a
  * frame slot (used to decide i64->i32 address narrowing). */
 static WasmValType addr_component_valtype(WTarget* t, Reg id) {
   if (id < t->reg_cap && t->reg_to_local[id] != 0xffffffffu && t->reg_type[id])
     return type_valtype(t, t->reg_type[id]);
   return type_valtype(t, slot_for(t, id)->type);
 }
 
 static void emit_addr_operand(WTarget* t, Operand addr, uint64_t* offset_out) {
   *offset_out = 0;
   if (addr.kind == OPK_LOCAL) {
     WSlot* s = slot_for(t, addr.v.frame_slot);
     if (s->kind != W_SLOT_STACK)
       wfail(t, "wasm: address of non-addressable local");
     emit_insn(t, WASM_INSN_LOCAL_GET, (i64)t->frame_base_local);
     *offset_out = s->frame_offset;
     return;
   }
   if (addr.kind == OPK_INDIRECT) {
     emit_push_addr_component(t, addr.v.ind.base);
     if (addr.v.ind.index != REG_NONE) {
       emit_push_addr_component(t, addr.v.ind.index);
       if (addr_component_valtype(t, addr.v.ind.index) == WASM_VAL_I64) {
         emit_insn(t, WASM_INSN_I32_WRAP_I64, 0);
       }
       if (addr.v.ind.log2_scale != 0) {
         emit_insn(t, WASM_INSN_I32_CONST, (i64)addr.v.ind.log2_scale);
         emit_insn(t, WASM_INSN_I32_SHL, 0);
       }
       emit_insn(t, WASM_INSN_I32_ADD, 0);
     }
     if (addr.v.ind.ofs >= 0) {
       *offset_out = (uint32_t)addr.v.ind.ofs;
     } else {
       emit_insn(t, WASM_INSN_I32_CONST, (i64)addr.v.ind.ofs);
       emit_insn(t, WASM_INSN_I32_ADD, 0);
     }
     return;
   }
   if (addr.kind == OPK_GLOBAL) {
     queue_symbol_addr_fixup(t, addr.v.global.sym, addr.v.global.addend);
     return;
   }
   if (addr.kind == OPK_REG) {
     /* An i32 address already materialized in a register: just push it. */
     emit_push_operand_reg(t, addr.v.reg);
     return;
   }
   wfail(t, "wasm: unsupported address operand kind %u", (unsigned)addr.kind);
 }
 
 /* Push a complete i32 address value (folding any positive offset into the base
  * via i32.add). Used by bulk-memory ops (memory.copy / memory.fill) which take
  * the address as a stack operand and carry no memarg offset. */
 static void emit_push_addr_value(WTarget* t, Operand addr) {
   uint64_t off;
   emit_addr_operand(t, addr, &off);
   if (off != 0) {
     emit_insn(t, WASM_INSN_I32_CONST, (i64)(uint32_t)off);
     emit_insn(t, WASM_INSN_I32_ADD, 0);
   }
 }
 
 static void emit_load_addr(WTarget* t, Operand addr, KitCgTypeId ty,
                            MemAccess ma) {
   uint64_t offset;
   WasmInsnKind k = load_kind_for(t, ty, ma);
   u32 width = wasm_mem_width((uint8_t)k);
   emit_addr_operand(t, addr, &offset);
   wasm_func_add_mem_insn(t->c, t->module, t->cur_func, k,
                          memarg_align_log2(ma.align, width), offset, 0);
 }
 
 static void emit_store_addr(WTarget* t, Operand addr, KitCgTypeId ty,
                             Operand src, MemAccess ma, u32 src_kind,
                             i64 src_imm, Reg src_reg) {
   uint64_t offset;
   WasmInsnKind k = store_kind_for(t, ty, ma);
   u32 width = wasm_mem_width((uint8_t)k);
   emit_addr_operand(t, addr, &offset);
   if (src_kind == WOP_IMM) {
     emit_push_imm(t, type_valtype(t, ty), src_imm);
   } else if (src_kind == WOP_WASM_LOCAL) {
     emit_insn(t, WASM_INSN_LOCAL_GET, src_imm);
   } else if (src.kind == OPK_IMM) {
     emit_push_imm(t, type_valtype(t, ty), src.v.imm);
   } else if (src_kind == WOP_LOCAL) {
     emit_push_operand(t, WOP_LOCAL, src_imm, REG_NONE, ty);
   } else {
     emit_push_operand_reg(t, src_reg);
   }
   wasm_func_add_mem_insn(t->c, t->module, t->cur_func, k,
                          memarg_align_log2(ma.align, width), offset, 0);
 }
 
 /* Map (BinOp, valtype) to wasm opcode. */
 static WasmInsnKind binop_kind(WTarget* t, BinOp op, WasmValType vt) {
   switch (op) {
     case BO_IADD:
       return vt == WASM_VAL_I64 ? WASM_INSN_I64_ADD : WASM_INSN_I32_ADD;
     case BO_ISUB:
       return vt == WASM_VAL_I64 ? WASM_INSN_I64_SUB : WASM_INSN_I32_SUB;
     case BO_IMUL:
       return vt == WASM_VAL_I64 ? WASM_INSN_I64_MUL : WASM_INSN_I32_MUL;
     case BO_SDIV:
       return vt == WASM_VAL_I64 ? WASM_INSN_I64_DIV_S : WASM_INSN_I32_DIV_S;
     case BO_UDIV:
       return vt == WASM_VAL_I64 ? WASM_INSN_I64_DIV_U : WASM_INSN_I32_DIV_U;
     case BO_SREM:
       return vt == WASM_VAL_I64 ? WASM_INSN_I64_REM_S : WASM_INSN_I32_REM_S;
     case BO_UREM:
       return vt == WASM_VAL_I64 ? WASM_INSN_I64_REM_U : WASM_INSN_I32_REM_U;
     case BO_AND:
       return vt == WASM_VAL_I64 ? WASM_INSN_I64_AND : WASM_INSN_I32_AND;
     case BO_OR:
       return vt == WASM_VAL_I64 ? WASM_INSN_I64_OR : WASM_INSN_I32_OR;
     case BO_XOR:
       return vt == WASM_VAL_I64 ? WASM_INSN_I64_XOR : WASM_INSN_I32_XOR;
     case BO_SHL:
       return vt == WASM_VAL_I64 ? WASM_INSN_I64_SHL : WASM_INSN_I32_SHL;
     case BO_SHR_S:
       return vt == WASM_VAL_I64 ? WASM_INSN_I64_SHR_S : WASM_INSN_I32_SHR_S;
     case BO_SHR_U:
       return vt == WASM_VAL_I64 ? WASM_INSN_I64_SHR_U : WASM_INSN_I32_SHR_U;
     case BO_FADD:
       return vt == WASM_VAL_F64 ? WASM_INSN_F64_ADD : WASM_INSN_F32_ADD;
     case BO_FSUB:
       return vt == WASM_VAL_F64 ? WASM_INSN_F64_SUB : WASM_INSN_F32_SUB;
     case BO_FMUL:
       return vt == WASM_VAL_F64 ? WASM_INSN_F64_MUL : WASM_INSN_F32_MUL;
     case BO_FDIV:
       return vt == WASM_VAL_F64 ? WASM_INSN_F64_DIV : WASM_INSN_F32_DIV;
   }
   wfail(t, "wasm: unsupported binop %d", (int)op);
 }
 
 static WasmInsnKind cmp_kind(WTarget* t, CmpOp op, WasmValType vt) {
   int is64 = (vt == WASM_VAL_I64);
   switch (op) {
     case CMP_EQ:
       return is64 ? WASM_INSN_I64_EQ : WASM_INSN_I32_EQ;
     case CMP_NE:
       return is64 ? WASM_INSN_I64_NE : WASM_INSN_I32_NE;
     case CMP_LT_S:
       return is64 ? WASM_INSN_I64_LT_S : WASM_INSN_I32_LT_S;
     case CMP_LE_S:
       return is64 ? WASM_INSN_I64_LE_S : WASM_INSN_I32_LE_S;
     case CMP_GT_S:
       return is64 ? WASM_INSN_I64_GT_S : WASM_INSN_I32_GT_S;
     case CMP_GE_S:
       return is64 ? WASM_INSN_I64_GE_S : WASM_INSN_I32_GE_S;
     case CMP_LT_U:
       return is64 ? WASM_INSN_I64_LT_U : WASM_INSN_I32_LT_U;
     case CMP_LE_U:
       return is64 ? WASM_INSN_I64_LE_U : WASM_INSN_I32_LE_U;
     case CMP_GT_U:
       return is64 ? WASM_INSN_I64_GT_U : WASM_INSN_I32_GT_U;
     case CMP_GE_U:
       return is64 ? WASM_INSN_I64_GE_U : WASM_INSN_I32_GE_U;
     /* FP compares are lowered by emit_fp_cmp (they may need multiple wasm
      * instructions) and never reach cmp_kind. Listed so -Wswitch stays useful.
      */
     case CMP_OEQ_F:
     case CMP_ONE_F:
     case CMP_OLT_F:
     case CMP_OLE_F:
     case CMP_OGT_F:
     case CMP_OGE_F:
     case CMP_UEQ_F:
     case CMP_UNE_F:
     case CMP_ULT_F:
     case CMP_ULE_F:
     case CMP_UGT_F:
     case CMP_UGE_F:
       break;
   }
   wfail(t, "wasm: unsupported cmp %d", (int)op);
 }
 
 /* Push both compare operands (a then b) onto the wasm stack. */
 static void push_cmp_operands(WTarget* t, WIR* w, KitCgTypeId opty) {
   emit_push_operand(t, w->imm_kind, w->imm_a, w->a, opty);
   emit_push_operand(t, w->imm_kind_b, w->imm_b, w->b, opty);
 }
 
 /* Lower an FP compare to the wasm stack, leaving an i32 0/1 result. wasm's
  * f.eq/f.lt/f.le/f.gt/f.ge are ordered (false on NaN) and f.ne is unordered
  * (true on NaN), so the 12 IEEE predicates compose from those plus i32.eqz /
  * i32.or, using unordered-R == !(ordered-not-R). ONE/UEQ need both operands
  * twice, so push_cmp_operands runs again for the second relation. */
 static void emit_fp_cmp(WTarget* t, CmpOp op, WIR* w, KitCgTypeId opty) {
   int d = (type_valtype(t, opty) == WASM_VAL_F64);
   WasmInsnKind EQ = d ? WASM_INSN_F64_EQ : WASM_INSN_F32_EQ;
   WasmInsnKind NE = d ? WASM_INSN_F64_NE : WASM_INSN_F32_NE;
   WasmInsnKind LT = d ? WASM_INSN_F64_LT : WASM_INSN_F32_LT;
   WasmInsnKind LE = d ? WASM_INSN_F64_LE : WASM_INSN_F32_LE;
   WasmInsnKind GT = d ? WASM_INSN_F64_GT : WASM_INSN_F32_GT;
   WasmInsnKind GE = d ? WASM_INSN_F64_GE : WASM_INSN_F32_GE;
   switch (op) {
     case CMP_OEQ_F:
       push_cmp_operands(t, w, opty);
       emit_insn(t, EQ, 0);
       return;
     case CMP_UNE_F:
       push_cmp_operands(t, w, opty);
       emit_insn(t, NE, 0);
       return;
     case CMP_OLT_F:
       push_cmp_operands(t, w, opty);
       emit_insn(t, LT, 0);
       return;
     case CMP_OLE_F:
       push_cmp_operands(t, w, opty);
       emit_insn(t, LE, 0);
       return;
     case CMP_OGT_F:
       push_cmp_operands(t, w, opty);
       emit_insn(t, GT, 0);
       return;
     case CMP_OGE_F:
       push_cmp_operands(t, w, opty);
       emit_insn(t, GE, 0);
       return;
     case CMP_UGE_F: /* !(OLT) */
       push_cmp_operands(t, w, opty);
       emit_insn(t, LT, 0);
       emit_insn(t, WASM_INSN_I32_EQZ, 0);
       return;
     case CMP_UGT_F: /* !(OLE) */
       push_cmp_operands(t, w, opty);
       emit_insn(t, LE, 0);
       emit_insn(t, WASM_INSN_I32_EQZ, 0);
       return;
     case CMP_ULE_F: /* !(OGT) */
       push_cmp_operands(t, w, opty);
       emit_insn(t, GT, 0);
       emit_insn(t, WASM_INSN_I32_EQZ, 0);
       return;
     case CMP_ULT_F: /* !(OGE) */
       push_cmp_operands(t, w, opty);
       emit_insn(t, GE, 0);
       emit_insn(t, WASM_INSN_I32_EQZ, 0);
       return;
     case CMP_ONE_F: /* ordered & !=: (a<b) | (a>b) */
       push_cmp_operands(t, w, opty);
       emit_insn(t, LT, 0);
       push_cmp_operands(t, w, opty);
       emit_insn(t, GT, 0);
       emit_insn(t, WASM_INSN_I32_OR, 0);
       return;
     case CMP_UEQ_F: /* unordered | ==: !((a<b) | (a>b)) */
       push_cmp_operands(t, w, opty);
       emit_insn(t, LT, 0);
       push_cmp_operands(t, w, opty);
       emit_insn(t, GT, 0);
       emit_insn(t, WASM_INSN_I32_OR, 0);
       emit_insn(t, WASM_INSN_I32_EQZ, 0);
       return;
     default:
       wfail(t, "wasm: unsupported fp cmp %d", (int)op);
   }
 }
 
 static void emit_convert(WTarget* t, ConvKind ck, WasmValType src,
                          WasmValType dst, u32 sw, u32 dw) {
   (void)dw;
   /* Integer sign/zero extension. Sub-i32 logical widths (i8/i16) share the i32
    * valtype, so a "same valtype" SEXT/ZEXT is NOT a no-op — the high bits must
    * be filled per the source's logical width (sw). The CG IR keeps narrow
    * immediates as truncated bit patterns, so without this an i8 value like
    * (signed char)-128 reads back as 128. */
   if (ck == CV_SEXT && src != WASM_VAL_F32 && src != WASM_VAL_F64) {
     if (src == WASM_VAL_I32) {
       if (sw == 8u)
         emit_insn(t, WASM_INSN_I32_EXTEND8_S, 0);
       else if (sw == 16u)
         emit_insn(t, WASM_INSN_I32_EXTEND16_S, 0);
     } else {
       if (sw == 8u)
         emit_insn(t, WASM_INSN_I64_EXTEND8_S, 0);
       else if (sw == 16u)
         emit_insn(t, WASM_INSN_I64_EXTEND16_S, 0);
       else if (sw == 32u)
         emit_insn(t, WASM_INSN_I64_EXTEND32_S, 0);
     }
     if (src == WASM_VAL_I32 && dst == WASM_VAL_I64)
       emit_insn(t, WASM_INSN_I64_EXTEND_I32_S, 0);
     else if (src == WASM_VAL_I64 && dst == WASM_VAL_I32)
       emit_insn(t, WASM_INSN_I32_WRAP_I64, 0);
     return;
   }
   if (ck == CV_ZEXT && src != WASM_VAL_F32 && src != WASM_VAL_F64) {
     if (src == WASM_VAL_I32) {
       if (sw > 0u && sw < 32u) {
         emit_insn(t, WASM_INSN_I32_CONST, (i64)(((u32)1 << sw) - 1u));
         emit_insn(t, WASM_INSN_I32_AND, 0);
       }
       if (dst == WASM_VAL_I64) emit_insn(t, WASM_INSN_I64_EXTEND_I32_U, 0);
     } else {
       if (sw > 0u && sw < 64u) {
         emit_push_imm(t, WASM_VAL_I64, (i64)(((u64)1 << sw) - 1u));
         emit_insn(t, WASM_INSN_I64_AND, 0);
       }
       if (dst == WASM_VAL_I32) emit_insn(t, WASM_INSN_I32_WRAP_I64, 0);
     }
     return;
   }
   if (src == dst && (ck == CV_BITCAST || ck == CV_TRUNC)) {
     /* No-op conversion. */
     return;
   }
   if (ck == CV_BITCAST) {
     if (src == WASM_VAL_I32 && dst == WASM_VAL_F32) {
       emit_insn(t, WASM_INSN_F32_REINTERPRET_I32, 0);
       return;
     }
     if (src == WASM_VAL_F32 && dst == WASM_VAL_I32) {
       emit_insn(t, WASM_INSN_I32_REINTERPRET_F32, 0);
       return;
     }
     if (src == WASM_VAL_I64 && dst == WASM_VAL_F64) {
       emit_insn(t, WASM_INSN_F64_REINTERPRET_I64, 0);
       return;
     }
     if (src == WASM_VAL_F64 && dst == WASM_VAL_I64) {
       emit_insn(t, WASM_INSN_I64_REINTERPRET_F64, 0);
       return;
     }
     /* Width-changing ptr/int bitcasts: kit_cg_ptr_to_int and
      * kit_cg_int_to_ptr route through CV_BITCAST, and on wasm32 a pointer
      * is i32 while the frontend integer side may be i64. Lower as
      * wrap/extend (zero-extend; pointers are non-negative addresses). */
     if (src == WASM_VAL_I64 && dst == WASM_VAL_I32) {
       emit_insn(t, WASM_INSN_I32_WRAP_I64, 0);
       return;
     }
     if (src == WASM_VAL_I32 && dst == WASM_VAL_I64) {
       emit_insn(t, WASM_INSN_I64_EXTEND_I32_U, 0);
       return;
     }
     wfail(t, "wasm: unsupported bitcast");
   }
   if (ck == CV_TRUNC) {
     if (src == WASM_VAL_I64 && dst == WASM_VAL_I32) {
       emit_insn(t, WASM_INSN_I32_WRAP_I64, 0);
       return;
     }
   }
   if (ck == CV_FEXT && src == WASM_VAL_F32 && dst == WASM_VAL_F64) {
     emit_insn(t, WASM_INSN_F64_PROMOTE_F32, 0);
     return;
   }
   if (ck == CV_FTRUNC && src == WASM_VAL_F64 && dst == WASM_VAL_F32) {
     emit_insn(t, WASM_INSN_F32_DEMOTE_F64, 0);
     return;
   }
   if (ck == CV_ITOF_S) {
     if (src == WASM_VAL_I32 && dst == WASM_VAL_F32) {
       emit_insn(t, WASM_INSN_F32_CONVERT_I32_S, 0);
       return;
     }
     if (src == WASM_VAL_I32 && dst == WASM_VAL_F64) {
       emit_insn(t, WASM_INSN_F64_CONVERT_I32_S, 0);
       return;
     }
     if (src == WASM_VAL_I64 && dst == WASM_VAL_F32) {
       emit_insn(t, WASM_INSN_F32_CONVERT_I64_S, 0);
       return;
     }
     if (src == WASM_VAL_I64 && dst == WASM_VAL_F64) {
       emit_insn(t, WASM_INSN_F64_CONVERT_I64_S, 0);
       return;
     }
   }
   if (ck == CV_ITOF_U) {
     if (src == WASM_VAL_I32 && dst == WASM_VAL_F32) {
       emit_insn(t, WASM_INSN_F32_CONVERT_I32_U, 0);
       return;
     }
     if (src == WASM_VAL_I32 && dst == WASM_VAL_F64) {
       emit_insn(t, WASM_INSN_F64_CONVERT_I32_U, 0);
       return;
     }
     if (src == WASM_VAL_I64 && dst == WASM_VAL_F32) {
       emit_insn(t, WASM_INSN_F32_CONVERT_I64_U, 0);
       return;
     }
     if (src == WASM_VAL_I64 && dst == WASM_VAL_F64) {
       emit_insn(t, WASM_INSN_F64_CONVERT_I64_U, 0);
       return;
     }
   }
   if (ck == CV_FTOI_S) {
     if (src == WASM_VAL_F32 && dst == WASM_VAL_I32) {
       emit_insn(t, WASM_INSN_I32_TRUNC_F32_S, 0);
       return;
     }
     if (src == WASM_VAL_F64 && dst == WASM_VAL_I32) {
       emit_insn(t, WASM_INSN_I32_TRUNC_F64_S, 0);
       return;
     }
     if (src == WASM_VAL_F32 && dst == WASM_VAL_I64) {
       emit_insn(t, WASM_INSN_I64_TRUNC_F32_S, 0);
       return;
     }
     if (src == WASM_VAL_F64 && dst == WASM_VAL_I64) {
       emit_insn(t, WASM_INSN_I64_TRUNC_F64_S, 0);
       return;
     }
   }
   if (ck == CV_FTOI_U) {
     if (src == WASM_VAL_F32 && dst == WASM_VAL_I32) {
       emit_insn(t, WASM_INSN_I32_TRUNC_F32_U, 0);
       return;
     }
     if (src == WASM_VAL_F64 && dst == WASM_VAL_I32) {
       emit_insn(t, WASM_INSN_I32_TRUNC_F64_U, 0);
       return;
     }
     if (src == WASM_VAL_F32 && dst == WASM_VAL_I64) {
       emit_insn(t, WASM_INSN_I64_TRUNC_F32_U, 0);
       return;
     }
     if (src == WASM_VAL_F64 && dst == WASM_VAL_I64) {
       emit_insn(t, WASM_INSN_I64_TRUNC_F64_U, 0);
       return;
     }
   }
   wfail(t, "wasm: unsupported convert kind %d (%d -> %d)", (int)ck, (int)src,
         (int)dst);
 }
 
 /* During lowering we keep a running active-scope stack so we can compute
  * br depths. */
 typedef struct LoweringScope {
   u32 id;
   u8 kind;
   /* Depth at which break/continue targets are reached. */
   u32 break_depth;
   u32 cont_depth;
 } LoweringScope;
 
 typedef struct LoweringState {
   WTarget* t;
   /* Bounded by the deepest synthetic + CG scope nesting we'll emit.
    * Switch islands wrap one block per case, so the limit is roughly
    * (max cases + max user nesting). 1024 leaves room for very wide
    * switches without forcing future per-case-count caps. */
   LoweringScope stack[1024];
   u32 nstack;
   u32 cur_depth;
 } LoweringState;
 
 static u32 br_to_label(LoweringState* L, Label l) {
   WLabel* lbl = lookup_label(L->t, l);
   if (!lbl) wfail(L->t, "wasm: br to unknown label");
   if (lbl->kind == WLBL_SCOPE_BREAK) {
     for (u32 i = L->nstack; i > 0; --i) {
       if (L->stack[i - 1u].id == lbl->scope_id) {
         return L->cur_depth - L->stack[i - 1u].break_depth;
       }
     }
     wfail(L->t, "wasm: br to break label of inactive scope");
   }
   if (lbl->kind == WLBL_SCOPE_CONT) {
     for (u32 i = L->nstack; i > 0; --i) {
       if (L->stack[i - 1u].id == lbl->scope_id) {
         return L->cur_depth - L->stack[i - 1u].cont_depth;
       }
     }
     wfail(L->t, "wasm: br to continue label of inactive scope");
   }
   /* wasm_structurize wraps every reachable forward label in a synthetic
    * SCOPE_BLOCK (forward goto) or SCOPE_LOOP (backward goto), and
    * unroll_switch_islands reorders the WIR so switch case labels are
    * forward refs from WIR_SWITCH. Arriving here means the structurer
    * missed a shape — a bug, not a feature gap. */
   wfail(L->t,
         "wasm: br to free label whose synthetic scope was not "
         "emitted; structurer bug");
 }
 
 static i64 wasm_switch_sign_extend(u64 v, u32 width) {
   if (width == 0u || width >= 64u) return (i64)v;
   {
     u64 bit = 1ull << (width - 1u);
     u64 mask = (1ull << width) - 1u;
     v &= mask;
     return (i64)((v ^ bit) - bit);
   }
 }
 
 static int wasm_switch_extents(WTarget* t, const WIR* w, i64* out_vmin,
                                u64* out_span) {
   u32 width;
   i64 vmin = INT64_MAX;
   i64 vmax = INT64_MIN;
   if (w->switch_ncases == 0) return 0;
   width = kit_cg_type_int_width((KitCompiler*)t->c, w->type);
   if (!width || width > 64u) return 0;
   for (u32 i = 0; i < w->switch_ncases; ++i) {
     i64 vi = wasm_switch_sign_extend(w->switch_cases[i].value, width);
     if (vi < vmin) vmin = vi;
     if (vi > vmax) vmax = vi;
   }
   if (vmax < vmin) return 0;
   {
     u64 delta = (u64)vmax - (u64)vmin;
     if (delta == UINT64_MAX) return 0;
     *out_span = delta + 1u;
   }
   *out_vmin = vmin;
   return 1;
 }
 
 static void emit_br_table(WTarget* t, const u32* targets, u32 ntargets) {
   WasmInsn* in;
   if (ntargets == 0)
     wfail(t, "wasm: br_table needs at least the default target");
   wasm_func_add_insn(t->c, t->module, t->cur_func, WASM_INSN_BR_TABLE, 0);
   in = &t->cur_func->insns[t->cur_func->ninsns - 1u];
   wasm_insn_set_targets(t->c, t->module, in, targets, ntargets);
 }
 
 /* A switch lowers to a dense br_table when its case values span a range that
  * isn't pathologically sparse relative to the number of cases; otherwise an
  * `eq`/`br_if` comparison chain. Small ranges always take the table (cheap
  * either way); larger ranges only when at least ~half the table slots carry a
  * real case, so a sparse switch (e.g. `case 0`, `case 1000000`) doesn't
  * materialize a giant mostly-default table. There is no range-splitting yet,
  * so a switch that fails this test is a linear scan. */
 static int switch_use_br_table(const WIR* w, u64 span) {
   if (span <= 64u) return 1;
   return span <= (u64)w->switch_ncases * 2u;
 }
 
 static void emit_switch_br_table(WTarget* t, LoweringState* L, const WIR* w) {
   i64 vmin;
   u64 span;
   u32* targets;
   Label* labels;
   u32 ntargets;
   WasmValType vt;
   Heap* h = t->c->ctx->heap;
 
   if (w->switch_ncases == 0) {
     emit_insn(t, WASM_INSN_BR, (i64)br_to_label(L, w->labels[0]));
     return;
   }
   if (!wasm_switch_extents(t, w, &vmin, &span))
     wfail(t, "wasm: unsupported switch selector type");
   vt = type_valtype(t, w->type);
   if (vt != WASM_VAL_I32 && vt != WASM_VAL_I64)
     wfail(t, "wasm: switch selector must be integer");
   if (!switch_use_br_table(w, span)) {
     for (u32 i = 0; i < w->switch_ncases; ++i) {
       u32 width = kit_cg_type_int_width((KitCompiler*)t->c, w->type);
       i64 vi = wasm_switch_sign_extend(w->switch_cases[i].value, width);
       emit_push_operand(t, w->imm_kind, w->imm_a, w->a, w->type);
       emit_push_imm(t, vt, vi);
       emit_insn(t, vt == WASM_VAL_I64 ? WASM_INSN_I64_EQ : WASM_INSN_I32_EQ, 0);
       emit_insn(t, WASM_INSN_BR_IF,
                 (i64)br_to_label(L, w->switch_cases[i].label));
     }
     emit_insn(t, WASM_INSN_BR, (i64)br_to_label(L, w->labels[0]));
     return;
   }
 
   /* Dense table: one slot per value in [vmin, vmin+span), default-filled, with
    * the default appended as the trailing out-of-range target. */
   ntargets = (u32)span + 1u;
   labels = (Label*)h->alloc(h, sizeof(Label) * span, _Alignof(Label));
   targets = (u32*)h->alloc(h, sizeof(u32) * ntargets, _Alignof(u32));
   if (!labels || !targets) wfail(t, "wasm: out of memory for switch table");
   for (u64 i = 0; i < span; ++i) labels[i] = w->labels[0];
   for (u32 i = 0; i < w->switch_ncases; ++i) {
     u32 width = kit_cg_type_int_width((KitCompiler*)t->c, w->type);
     i64 vi = wasm_switch_sign_extend(w->switch_cases[i].value, width);
     u64 slot = (u64)(vi - vmin);
     if (slot >= span) wfail(t, "wasm: switch case outside span");
     labels[slot] = w->switch_cases[i].label;
   }
 
   emit_push_operand(t, w->imm_kind, w->imm_a, w->a, w->type);
   if (vmin != 0) {
     emit_push_imm(t, vt, vmin);
     emit_insn(t, vt == WASM_VAL_I64 ? WASM_INSN_I64_SUB : WASM_INSN_I32_SUB, 0);
   }
   if (vt == WASM_VAL_I64) emit_insn(t, WASM_INSN_I32_WRAP_I64, 0);
 
   for (u32 i = 0; i < (u32)span; ++i) targets[i] = br_to_label(L, labels[i]);
   targets[ntargets - 1u] = br_to_label(L, w->labels[0]);
   emit_br_table(t, targets, ntargets);
   h->free(h, targets, sizeof(u32) * ntargets);
   h->free(h, labels, sizeof(Label) * span);
 }
 
 /* -----------------------------------------------------------------
  * Intrinsics (bit ops / bswap / overflow arith)
  *
  * MEMCPY/MEMMOVE/MEMSET don't appear here: the recorder funnels them
  * into WIR_COPY_BYTES / WIR_SET_BYTES which already lower to
  * memory.copy / memory.fill. Hints (PREFETCH/EXPECT/ASSUME_ALIGNED)
  * also don't reach the linearizer — the recorder either drops them or
  * emits a plain copy. ----------------------------------------------- */
 
 static void emit_intrinsic_bit_op(WTarget* t, const WIR* w) {
   /* clz/ctz/popcount instruction width follows the operand (type2), not the
    * i32 result. i64 forms produce an i64 count that we wrap to the i32 dst. */
   WasmValType vt = type_valtype(t, w->type2 ? w->type2 : w->type);
   WasmValType dvt = type_valtype(t, w->type);
   WasmInsnKind op;
   switch ((IntrinKind)w->cgop) {
     case INTRIN_CLZ:
       op = (vt == WASM_VAL_I64) ? WASM_INSN_I64_CLZ : WASM_INSN_I32_CLZ;
       break;
     case INTRIN_CTZ:
       op = (vt == WASM_VAL_I64) ? WASM_INSN_I64_CTZ : WASM_INSN_I32_CTZ;
       break;
     case INTRIN_POPCOUNT:
       op = (vt == WASM_VAL_I64) ? WASM_INSN_I64_POPCNT : WASM_INSN_I32_POPCNT;
       break;
     default:
       wfail(t, "wasm: unexpected bit-op intrinsic %d", (int)w->cgop);
       return;
   }
   emit_push_operand_reg(t, w->a);
   emit_insn(t, op, 0);
   if (vt == WASM_VAL_I64 && dvt == WASM_VAL_I32)
     emit_insn(t, WASM_INSN_I32_WRAP_I64, 0);
   emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
 }
 
 static void emit_intrinsic_bswap(WTarget* t, const WIR* w) {
   /* Width-by-type: the recorded result type fixes the byte width. */
   u32 width = (u32)abi_cg_sizeof(t->c->abi, w->type);
   if (width <= 4) {
     /* Both 16- and 32-bit forms operate over i32. The 16-bit form only
      * touches the low 16 bits; any extra high bits in the input are
      * discarded by the AND mask. */
     u32 tmp = add_wasm_local(t, WASM_VAL_I32);
     emit_push_operand_reg(t, w->a);
     emit_insn(t, WASM_INSN_LOCAL_SET, (i64)tmp);
     if (width <= 2) {
       /* (x & 0xff) << 8 */
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)tmp);
       emit_insn(t, WASM_INSN_I32_CONST, 0xff);
       emit_insn(t, WASM_INSN_I32_AND, 0);
       emit_insn(t, WASM_INSN_I32_CONST, 8);
       emit_insn(t, WASM_INSN_I32_SHL, 0);
       /* (x >> 8) & 0xff */
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)tmp);
       emit_insn(t, WASM_INSN_I32_CONST, 8);
       emit_insn(t, WASM_INSN_I32_SHR_U, 0);
       emit_insn(t, WASM_INSN_I32_CONST, 0xff);
       emit_insn(t, WASM_INSN_I32_AND, 0);
       emit_insn(t, WASM_INSN_I32_OR, 0);
     } else {
       /* Four-byte shuffle. */
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)tmp);
       emit_insn(t, WASM_INSN_I32_CONST, 24);
       emit_insn(t, WASM_INSN_I32_SHR_U, 0);
       emit_insn(t, WASM_INSN_I32_CONST, 0xff);
       emit_insn(t, WASM_INSN_I32_AND, 0);
 
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)tmp);
       emit_insn(t, WASM_INSN_I32_CONST, 8);
       emit_insn(t, WASM_INSN_I32_SHR_U, 0);
       emit_insn(t, WASM_INSN_I32_CONST, 0xff00);
       emit_insn(t, WASM_INSN_I32_AND, 0);
       emit_insn(t, WASM_INSN_I32_OR, 0);
 
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)tmp);
       emit_insn(t, WASM_INSN_I32_CONST, 8);
       emit_insn(t, WASM_INSN_I32_SHL, 0);
       emit_insn(t, WASM_INSN_I32_CONST, 0xff0000);
       emit_insn(t, WASM_INSN_I32_AND, 0);
       emit_insn(t, WASM_INSN_I32_OR, 0);
 
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)tmp);
       emit_insn(t, WASM_INSN_I32_CONST, 24);
       emit_insn(t, WASM_INSN_I32_SHL, 0);
       emit_insn(t, WASM_INSN_I32_OR, 0);
     }
     emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
     return;
   }
   /* 8-byte form: byte reverse over i64. */
   u32 tmp = add_wasm_local(t, WASM_VAL_I64);
   emit_push_operand_reg(t, w->a);
   emit_insn(t, WASM_INSN_LOCAL_SET, (i64)tmp);
   for (int i = 0; i < 8; ++i) {
     emit_insn(t, WASM_INSN_LOCAL_GET, (i64)tmp);
     if (i > 0) {
       emit_insn(t, WASM_INSN_I64_CONST, (i64)(i * 8));
       emit_insn(t, WASM_INSN_I64_SHR_U, 0);
     }
     emit_insn(t, WASM_INSN_I64_CONST, 0xff);
     emit_insn(t, WASM_INSN_I64_AND, 0);
     int shift = (7 - i) * 8;
     if (shift > 0) {
       emit_insn(t, WASM_INSN_I64_CONST, (i64)shift);
       emit_insn(t, WASM_INSN_I64_SHL, 0);
     }
     if (i > 0) emit_insn(t, WASM_INSN_I64_OR, 0);
   }
   emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
 }
 
 static void emit_intrinsic_mul_high(WTarget* t, const WIR* w) {
   IntrinKind k = (IntrinKind)w->cgop;
   WasmValType vt = type_valtype(t, w->type);
   u32 a_loc = add_wasm_local(t, vt);
   u32 b_loc = add_wasm_local(t, vt);
   emit_push_operand(t, w->imm_kind, w->imm_a, w->a, w->type);
   emit_insn(t, WASM_INSN_LOCAL_SET, (i64)a_loc);
   emit_push_operand(t, w->imm_kind_b, w->imm_b, w->b, w->type);
   emit_insn(t, WASM_INSN_LOCAL_SET, (i64)b_loc);
 
   if (vt == WASM_VAL_I32) {
     emit_insn(t, WASM_INSN_LOCAL_GET, (i64)a_loc);
     emit_insn(t, k == INTRIN_SMUL_HIGH ? WASM_INSN_I64_EXTEND_I32_S
                                        : WASM_INSN_I64_EXTEND_I32_U,
               0);
     emit_insn(t, WASM_INSN_LOCAL_GET, (i64)b_loc);
     emit_insn(t, k == INTRIN_SMUL_HIGH ? WASM_INSN_I64_EXTEND_I32_S
                                        : WASM_INSN_I64_EXTEND_I32_U,
               0);
     emit_insn(t, WASM_INSN_I64_MUL, 0);
     emit_insn(t, WASM_INSN_I64_CONST, 32);
     emit_insn(t, k == INTRIN_SMUL_HIGH ? WASM_INSN_I64_SHR_S
                                        : WASM_INSN_I64_SHR_U,
               0);
     emit_insn(t, WASM_INSN_I32_WRAP_I64, 0);
     emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
     return;
   }
 
   /* Core wasm has no widening i64 multiply. Split into 32-bit limbs:
    * high = p11 + (p01>>32) + (p10>>32) +
    *        (((p00>>32) + lo32(p01) + lo32(p10)) >> 32). */
   {
     u32 p00 = add_wasm_local(t, WASM_VAL_I64);
     u32 p01 = add_wasm_local(t, WASM_VAL_I64);
     u32 p10 = add_wasm_local(t, WASM_VAL_I64);
     u32 p11 = add_wasm_local(t, WASM_VAL_I64);
     u32 middle = add_wasm_local(t, WASM_VAL_I64);
     u32 high = add_wasm_local(t, WASM_VAL_I64);
 
     emit_insn(t, WASM_INSN_LOCAL_GET, (i64)a_loc);
     emit_insn(t, WASM_INSN_I64_CONST, 0xffffffffull);
     emit_insn(t, WASM_INSN_I64_AND, 0);
     emit_insn(t, WASM_INSN_LOCAL_GET, (i64)b_loc);
     emit_insn(t, WASM_INSN_I64_CONST, 0xffffffffull);
     emit_insn(t, WASM_INSN_I64_AND, 0);
     emit_insn(t, WASM_INSN_I64_MUL, 0);
     emit_insn(t, WASM_INSN_LOCAL_SET, (i64)p00);
 
     emit_insn(t, WASM_INSN_LOCAL_GET, (i64)a_loc);
     emit_insn(t, WASM_INSN_I64_CONST, 0xffffffffull);
     emit_insn(t, WASM_INSN_I64_AND, 0);
     emit_insn(t, WASM_INSN_LOCAL_GET, (i64)b_loc);
     emit_insn(t, WASM_INSN_I64_CONST, 32);
     emit_insn(t, WASM_INSN_I64_SHR_U, 0);
     emit_insn(t, WASM_INSN_I64_MUL, 0);
     emit_insn(t, WASM_INSN_LOCAL_SET, (i64)p01);
 
     emit_insn(t, WASM_INSN_LOCAL_GET, (i64)a_loc);
     emit_insn(t, WASM_INSN_I64_CONST, 32);
     emit_insn(t, WASM_INSN_I64_SHR_U, 0);
     emit_insn(t, WASM_INSN_LOCAL_GET, (i64)b_loc);
     emit_insn(t, WASM_INSN_I64_CONST, 0xffffffffull);
     emit_insn(t, WASM_INSN_I64_AND, 0);
     emit_insn(t, WASM_INSN_I64_MUL, 0);
     emit_insn(t, WASM_INSN_LOCAL_SET, (i64)p10);
 
     emit_insn(t, WASM_INSN_LOCAL_GET, (i64)a_loc);
     emit_insn(t, WASM_INSN_I64_CONST, 32);
     emit_insn(t, WASM_INSN_I64_SHR_U, 0);
     emit_insn(t, WASM_INSN_LOCAL_GET, (i64)b_loc);
     emit_insn(t, WASM_INSN_I64_CONST, 32);
     emit_insn(t, WASM_INSN_I64_SHR_U, 0);
     emit_insn(t, WASM_INSN_I64_MUL, 0);
     emit_insn(t, WASM_INSN_LOCAL_SET, (i64)p11);
 
     emit_insn(t, WASM_INSN_LOCAL_GET, (i64)p00);
     emit_insn(t, WASM_INSN_I64_CONST, 32);
     emit_insn(t, WASM_INSN_I64_SHR_U, 0);
     emit_insn(t, WASM_INSN_LOCAL_GET, (i64)p01);
     emit_insn(t, WASM_INSN_I64_CONST, 0xffffffffull);
     emit_insn(t, WASM_INSN_I64_AND, 0);
     emit_insn(t, WASM_INSN_I64_ADD, 0);
     emit_insn(t, WASM_INSN_LOCAL_GET, (i64)p10);
     emit_insn(t, WASM_INSN_I64_CONST, 0xffffffffull);
     emit_insn(t, WASM_INSN_I64_AND, 0);
     emit_insn(t, WASM_INSN_I64_ADD, 0);
     emit_insn(t, WASM_INSN_LOCAL_SET, (i64)middle);
 
     emit_insn(t, WASM_INSN_LOCAL_GET, (i64)p11);
     emit_insn(t, WASM_INSN_LOCAL_GET, (i64)p01);
     emit_insn(t, WASM_INSN_I64_CONST, 32);
     emit_insn(t, WASM_INSN_I64_SHR_U, 0);
     emit_insn(t, WASM_INSN_I64_ADD, 0);
     emit_insn(t, WASM_INSN_LOCAL_GET, (i64)p10);
     emit_insn(t, WASM_INSN_I64_CONST, 32);
     emit_insn(t, WASM_INSN_I64_SHR_U, 0);
     emit_insn(t, WASM_INSN_I64_ADD, 0);
     emit_insn(t, WASM_INSN_LOCAL_GET, (i64)middle);
     emit_insn(t, WASM_INSN_I64_CONST, 32);
     emit_insn(t, WASM_INSN_I64_SHR_U, 0);
     emit_insn(t, WASM_INSN_I64_ADD, 0);
     emit_insn(t, WASM_INSN_LOCAL_SET, (i64)high);
 
     emit_insn(t, WASM_INSN_LOCAL_GET, (i64)high);
     if (k == INTRIN_SMUL_HIGH) {
       /* signed_high(a,b) = unsigned_high(a,b) - (a<0 ? b : 0)
        *                                      - (b<0 ? a : 0). */
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)b_loc);
       emit_insn(t, WASM_INSN_I64_CONST, 0);
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)a_loc);
       emit_insn(t, WASM_INSN_I64_CONST, 0);
       emit_insn(t, WASM_INSN_I64_LT_S, 0);
       emit_insn(t, WASM_INSN_SELECT, 0);
       emit_insn(t, WASM_INSN_I64_SUB, 0);
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)a_loc);
       emit_insn(t, WASM_INSN_I64_CONST, 0);
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)b_loc);
       emit_insn(t, WASM_INSN_I64_CONST, 0);
       emit_insn(t, WASM_INSN_I64_LT_S, 0);
       emit_insn(t, WASM_INSN_SELECT, 0);
       emit_insn(t, WASM_INSN_I64_SUB, 0);
     }
     emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
   }
 }
 
 static void emit_intrinsic_overflow(WTarget* t, const WIR* w) {
   IntrinKind k = (IntrinKind)w->cgop;
   WasmValType vt = type_valtype(t, w->type);
   WasmInsnKind k_add =
       (vt == WASM_VAL_I64) ? WASM_INSN_I64_ADD : WASM_INSN_I32_ADD;
   WasmInsnKind k_sub =
       (vt == WASM_VAL_I64) ? WASM_INSN_I64_SUB : WASM_INSN_I32_SUB;
   WasmInsnKind k_and =
       (vt == WASM_VAL_I64) ? WASM_INSN_I64_AND : WASM_INSN_I32_AND;
   WasmInsnKind k_xor =
       (vt == WASM_VAL_I64) ? WASM_INSN_I64_XOR : WASM_INSN_I32_XOR;
   WasmInsnKind k_shr_u =
       (vt == WASM_VAL_I64) ? WASM_INSN_I64_SHR_U : WASM_INSN_I32_SHR_U;
   WasmInsnKind k_lt_u =
       (vt == WASM_VAL_I64) ? WASM_INSN_I64_LT_U : WASM_INSN_I32_LT_U;
   WasmInsnKind k_const =
       (vt == WASM_VAL_I64) ? WASM_INSN_I64_CONST : WASM_INSN_I32_CONST;
   KitCgTypeId bool_ty = builtin_id(KIT_CG_BUILTIN_BOOL);
 
   /* Stash both operands in scratch locals so each side of the expansion can
    * re-load them without re-evaluating immediates or relying on the wasm
    * value stack shape. */
   u32 a_loc = add_wasm_local(t, vt);
   u32 b_loc = add_wasm_local(t, vt);
   emit_push_operand(t, w->imm_kind, w->imm_a, w->a, w->type);
   emit_insn(t, WASM_INSN_LOCAL_SET, (i64)a_loc);
   emit_push_operand(t, w->imm_kind_b, w->imm_b, w->b, w->type);
   emit_insn(t, WASM_INSN_LOCAL_SET, (i64)b_loc);
 
   switch (k) {
     case INTRIN_UADD_OVERFLOW:
       /* r = a + b; ovf = (r <u a) */
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)a_loc);
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)b_loc);
       emit_insn(t, k_add, 0);
       emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
       emit_push_operand_reg(t, w->dst);
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)a_loc);
       emit_insn(t, k_lt_u, 0);
       emit_local_set(t, w->dst2, bool_ty, RC_INT);
       break;
     case INTRIN_USUB_OVERFLOW:
       /* r = a - b; ovf = (a <u b) */
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)a_loc);
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)b_loc);
       emit_insn(t, k_sub, 0);
       emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)a_loc);
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)b_loc);
       emit_insn(t, k_lt_u, 0);
       emit_local_set(t, w->dst2, bool_ty, RC_INT);
       break;
     case INTRIN_SADD_OVERFLOW:
       /* r = a + b; ovf = ((r ^ a) & (r ^ b)) >>u (W-1) */
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)a_loc);
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)b_loc);
       emit_insn(t, k_add, 0);
       emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
       emit_push_operand_reg(t, w->dst);
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)a_loc);
       emit_insn(t, k_xor, 0);
       emit_push_operand_reg(t, w->dst);
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)b_loc);
       emit_insn(t, k_xor, 0);
       emit_insn(t, k_and, 0);
       emit_insn(t, k_const, (i64)(vt == WASM_VAL_I64 ? 63 : 31));
       emit_insn(t, k_shr_u, 0);
       if (vt == WASM_VAL_I64) emit_insn(t, WASM_INSN_I32_WRAP_I64, 0);
       emit_local_set(t, w->dst2, bool_ty, RC_INT);
       break;
     case INTRIN_SSUB_OVERFLOW:
       /* r = a - b; ovf = ((a ^ b) & (a ^ r)) >>u (W-1) */
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)a_loc);
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)b_loc);
       emit_insn(t, k_sub, 0);
       emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)a_loc);
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)b_loc);
       emit_insn(t, k_xor, 0);
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)a_loc);
       emit_push_operand_reg(t, w->dst);
       emit_insn(t, k_xor, 0);
       emit_insn(t, k_and, 0);
       emit_insn(t, k_const, (i64)(vt == WASM_VAL_I64 ? 63 : 31));
       emit_insn(t, k_shr_u, 0);
       if (vt == WASM_VAL_I64) emit_insn(t, WASM_INSN_I32_WRAP_I64, 0);
       emit_local_set(t, w->dst2, bool_ty, RC_INT);
       break;
     case INTRIN_UMUL_OVERFLOW: {
       /* i32 only (i64 rejected in recorder). Widen to i64, multiply,
        * low 32 = result, ovf = (wide >> 32) != 0. */
       u32 wide = add_wasm_local(t, WASM_VAL_I64);
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)a_loc);
       emit_insn(t, WASM_INSN_I64_EXTEND_I32_U, 0);
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)b_loc);
       emit_insn(t, WASM_INSN_I64_EXTEND_I32_U, 0);
       emit_insn(t, WASM_INSN_I64_MUL, 0);
       emit_insn(t, WASM_INSN_LOCAL_SET, (i64)wide);
 
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)wide);
       emit_insn(t, WASM_INSN_I32_WRAP_I64, 0);
       emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
 
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)wide);
       emit_insn(t, WASM_INSN_I64_CONST, 32);
       emit_insn(t, WASM_INSN_I64_SHR_U, 0);
       emit_insn(t, WASM_INSN_I64_CONST, 0);
       emit_insn(t, WASM_INSN_I64_NE, 0);
       emit_local_set(t, w->dst2, bool_ty, RC_INT);
       break;
     }
     case INTRIN_SMUL_OVERFLOW: {
       /* i32 only. Sign-extend, multiply, low 32 = result, ovf if
        * sext(result) != wide product. */
       u32 wide = add_wasm_local(t, WASM_VAL_I64);
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)a_loc);
       emit_insn(t, WASM_INSN_I64_EXTEND_I32_S, 0);
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)b_loc);
       emit_insn(t, WASM_INSN_I64_EXTEND_I32_S, 0);
       emit_insn(t, WASM_INSN_I64_MUL, 0);
       emit_insn(t, WASM_INSN_LOCAL_SET, (i64)wide);
 
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)wide);
       emit_insn(t, WASM_INSN_I32_WRAP_I64, 0);
       emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
 
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)wide);
       emit_push_operand_reg(t, w->dst);
       emit_insn(t, WASM_INSN_I64_EXTEND_I32_S, 0);
       emit_insn(t, WASM_INSN_I64_NE, 0);
       emit_local_set(t, w->dst2, bool_ty, RC_INT);
       break;
     }
     default:
       wfail(t, "wasm: overflow intrinsic dispatch reached default (%d)",
             (int)k);
   }
 }
 
 static void emit_intrinsic(WTarget* t, const WIR* w) {
   IntrinKind k = (IntrinKind)w->cgop;
   switch (k) {
     case INTRIN_CLZ:
     case INTRIN_CTZ:
     case INTRIN_POPCOUNT:
       emit_intrinsic_bit_op(t, w);
       return;
     case INTRIN_BSWAP:
       emit_intrinsic_bswap(t, w);
       return;
     case INTRIN_SMUL_HIGH:
     case INTRIN_UMUL_HIGH:
       emit_intrinsic_mul_high(t, w);
       return;
     case INTRIN_SADD_OVERFLOW:
     case INTRIN_UADD_OVERFLOW:
     case INTRIN_SSUB_OVERFLOW:
     case INTRIN_USUB_OVERFLOW:
     case INTRIN_SMUL_OVERFLOW:
     case INTRIN_UMUL_OVERFLOW:
       emit_intrinsic_overflow(t, w);
       return;
     default:
       wfail(t, "wasm: unexpected intrinsic kind %d in linearizer", (int)k);
   }
 }
 
 static void linearize_range(WTarget* t, LoweringState* L, u32 start, u32 end) {
   for (u32 i = start; i < end; ++i) {
     WIR* w = &t->wir[i];
     switch (w->op) {
       case WIR_LOAD_IMM: {
         WasmValType vt = type_valtype(t, w->type);
         emit_push_imm(t, vt, w->imm);
         emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
         break;
       }
       case WIR_LOAD_CONST_F: {
         WasmValType vt = type_valtype(t, w->type);
         emit_fp(t,
                 vt == WASM_VAL_F64 ? WASM_INSN_F64_CONST : WASM_INSN_F32_CONST,
                 w->fp_imm);
         emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
         break;
       }
       case WIR_COPY: {
         emit_push_operand_reg(t, w->a);
         emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
         break;
       }
       case WIR_BINOP: {
         emit_push_operand(t, w->imm_kind, w->imm_a, w->a, w->type);
         emit_push_operand(t, w->imm_kind_b, w->imm_b, w->b, w->type);
         WasmValType vt = type_valtype(t, w->type);
         emit_insn(t, binop_kind(t, (BinOp)w->cgop, vt), 0);
         emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
         break;
       }
       case WIR_UNOP: {
         WasmValType vt = type_valtype(t, w->type);
         switch ((UnOp)w->cgop) {
           case UO_NEG: {
             /* 0 - a */
             emit_push_imm(t, vt, 0);
             emit_push_operand(t, w->imm_kind, w->imm_a, w->a, w->type);
             emit_insn(
                 t, vt == WASM_VAL_I64 ? WASM_INSN_I64_SUB : WASM_INSN_I32_SUB,
                 0);
             break;
           }
           case UO_FNEG: {
             emit_push_operand(t, w->imm_kind, w->imm_a, w->a, w->type);
             emit_insn(
                 t, vt == WASM_VAL_F64 ? WASM_INSN_F64_NEG : WASM_INSN_F32_NEG,
                 0);
             break;
           }
           case UO_BNOT: {
             /* a XOR -1 */
             emit_push_operand(t, w->imm_kind, w->imm_a, w->a, w->type);
             emit_push_imm(t, vt, -1);
             emit_insn(
                 t, vt == WASM_VAL_I64 ? WASM_INSN_I64_XOR : WASM_INSN_I32_XOR,
                 0);
             break;
           }
           case UO_NOT: {
             /* a == 0 — i{32,64}.eqz always produces an i32 0/1. When the CG
              * destination is i64 (e.g. !x where x was zext'd to i64 before the
              * negation), widen the i32 boolean back to i64 so the following
              * local.set is well-typed. */
             emit_push_operand(t, w->imm_kind, w->imm_a, w->a, w->type);
             emit_insn(
                 t, vt == WASM_VAL_I64 ? WASM_INSN_I64_EQZ : WASM_INSN_I32_EQZ,
                 0);
             if (vt == WASM_VAL_I64) emit_insn(t, WASM_INSN_I64_EXTEND_I32_U, 0);
             break;
           }
         }
         emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
         break;
       }
       case WIR_CMP: {
         CmpOp cop = (CmpOp)w->cgop;
         if (cop >= CMP_OEQ_F) {
           emit_fp_cmp(t, cop, w, w->type2);
         } else {
           push_cmp_operands(t, w, w->type2);
           emit_insn(t, cmp_kind(t, cop, type_valtype(t, w->type2)), 0);
         }
         /* cmp result is i32 (0/1). dst type may be wider — but cg generally
          * stores cmp results into i32. */
         emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
         break;
       }
       case WIR_CONVERT: {
         WasmValType src = type_valtype(t, w->type2);
         WasmValType dst = type_valtype(t, w->type);
         u32 sw = kit_cg_type_int_width((KitCompiler*)t->c, w->type2);
         u32 dw = kit_cg_type_int_width((KitCompiler*)t->c, w->type);
         emit_push_operand(t, w->imm_kind, w->imm_a, w->a, w->type2);
         emit_convert(t, (ConvKind)w->cgop, src, dst, sw, dw);
         emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
         break;
       }
       case WIR_CALL:
       case WIR_CALL_INDIRECT: {
         /* Tail calls tear down the caller's wasm frame (return_call /
          * return_call_indirect have polymorphic-unreachable type after the
          * call). Mirror the WIR_RET linear-stack epilogue before pushing
          * args so the linear-memory stack frame is released. Operands
          * below come from wasm locals (incoming params or reg-locals),
          * not from the linear stack we just freed. Variadic tail calls are
          * rejected upstream; sret tail calls forward the incoming pointer. */
         if (w->call_tail && t->has_stack_frame) {
           emit_insn(t, WASM_INSN_LOCAL_GET, (i64)t->frame_saved_sp_local);
           emit_insn(t, WASM_INSN_GLOBAL_SET, (i64)t->stack_pointer_global);
         }
         if (w->call_has_sret) {
           if (w->call_tail) {
             /* Forward this function's own incoming sret pointer: the callee
              * writes the same buffer (in our caller's frame, which outlives
              * the sibling call) and return_calls back. The pointer is a wasm
              * local, unaffected by the linear-frame teardown above. */
             if (t->sret_param_local == 0xffffffffu)
               wfail(t, "wasm: sret tail call without an incoming sret pointer");
             emit_insn(t, WASM_INSN_LOCAL_GET, (i64)t->sret_param_local);
           } else {
             /* Push sret pointer (address of caller-allocated buffer). */
             uint64_t off;
             emit_addr_operand(t, w->call_sret_addr, &off);
             if (off) {
               emit_insn(t, WASM_INSN_I32_CONST, (i64)off);
               emit_insn(t, WASM_INSN_I32_ADD, 0);
             }
           }
         }
         for (u32 a = 0; a < w->call_narg; ++a) {
           if (w->call_arg_kinds[a] == WOP_REG) {
             emit_push_operand_reg(t, w->call_args[a]);
           } else if (w->call_arg_kinds[a] == WOP_IMM) {
             WasmValType vt = type_valtype(t, w->call_arg_types[a]);
             emit_push_imm(t, vt, w->call_arg_imms[a]);
           } else if (w->call_arg_kinds[a] == WOP_ADDR) {
             uint64_t off;
             emit_addr_operand(t, w->call_arg_addrs[a], &off);
             if (off) {
               emit_insn(t, WASM_INSN_I32_CONST, (i64)off);
               emit_insn(t, WASM_INSN_I32_ADD, 0);
             }
           } else {
             wfail(t, "wasm: bad call-arg kind %u", w->call_arg_kinds[a]);
           }
         }
         /* Variadic packing. Each variadic arg occupies an 8-byte slot in a
          * caller-allocated linear-memory buffer; the buffer's address is
          * passed as the hidden trailing i32. We save __stack_pointer to a
          * scratch local before allocating the buffer and restore it after
          * the call returns, so a variadic call in a loop doesn't grow the
          * linear stack. See wasm_va_start / wasm_va_arg for the callee side.
          */
         if (w->call_variadic) {
           if (w->call_nvar == 0u) {
             /* No varargs: still pass a hidden i32. NULL is fine — the callee
              * must not deref va_list without a matching @va_arg, which a
              * well-typed program won't do. */
             emit_insn(t, WASM_INSN_I32_CONST, 0);
           } else {
             ensure_stack_pointer(t);
             if (t->varcall_saved_sp_local == 0xffffffffu)
               t->varcall_saved_sp_local = add_wasm_local(t, WASM_VAL_I32);
             if (t->varcall_buf_local == 0xffffffffu)
               t->varcall_buf_local = add_wasm_local(t, WASM_VAL_I32);
             u32 buf_size = w->call_nvar * 8u;
             /* Save SP, allocate aligned buffer, set SP = buf. */
             emit_insn(t, WASM_INSN_GLOBAL_GET, (i64)t->stack_pointer_global);
             emit_insn(t, WASM_INSN_LOCAL_TEE, (i64)t->varcall_saved_sp_local);
             emit_insn(t, WASM_INSN_I32_CONST, (i64)buf_size);
             emit_insn(t, WASM_INSN_I32_SUB, 0);
             emit_insn(t, WASM_INSN_I32_CONST, -(i64)8);
             emit_insn(t, WASM_INSN_I32_AND, 0);
             emit_insn(t, WASM_INSN_LOCAL_TEE, (i64)t->varcall_buf_local);
             emit_insn(t, WASM_INSN_GLOBAL_SET, (i64)t->stack_pointer_global);
             /* Pack each variadic arg at offset i*8. Store width is the
              * value's natural width (i32/i64/f32/f64); the unused high
              * bytes of i32/f32 slots are left as whatever __stack_pointer
              * pointed at, which @va_arg won't read for those slots. */
             for (u32 v = 0; v < w->call_nvar; ++v) {
               KitCgTypeId vty = w->call_var_types[v];
               WasmValType vvt = type_valtype(t, vty);
               WasmInsnKind store_op;
               u32 width;
               if (vvt == WASM_VAL_I64) {
                 store_op = WASM_INSN_I64_STORE;
                 width = 8u;
               } else if (vvt == WASM_VAL_F32) {
                 store_op = WASM_INSN_F32_STORE;
                 width = 4u;
               } else if (vvt == WASM_VAL_F64) {
                 store_op = WASM_INSN_F64_STORE;
                 width = 8u;
               } else {
                 store_op = WASM_INSN_I32_STORE;
                 width = 4u;
               }
               emit_insn(t, WASM_INSN_LOCAL_GET, (i64)t->varcall_buf_local);
               if (w->call_var_kinds[v] == WOP_REG) {
                 emit_push_operand_reg(t, w->call_var_regs[v]);
               } else if (w->call_var_kinds[v] == WOP_IMM) {
                 if (vvt == WASM_VAL_F32 || vvt == WASM_VAL_F64)
                   wfail(t, "wasm: float immediate variadic arg unsupported");
                 emit_push_imm(t, vvt, w->call_var_imms[v]);
               } else {
                 wfail(t, "wasm: bad variadic-arg kind %u",
                       w->call_var_kinds[v]);
               }
               wasm_func_add_mem_insn(t->c, t->module, t->cur_func, store_op,
                                      memarg_align_log2(width, width),
                                      (u64)(v * 8u), 0u);
             }
             /* Push buf addr as hidden last arg. */
             emit_insn(t, WASM_INSN_LOCAL_GET, (i64)t->varcall_buf_local);
           }
         }
         if (w->op == WIR_CALL_INDIRECT) {
           /* Callee: push the i32 table index. */
           emit_push_operand_reg(t, w->a);
           /* call_indirect / return_call_indirect both encode
            * (typeidx, tableidx). The encoder reads `imm` as typeidx and
            * `align` as tableidx. */
           wasm_func_add_insn(t->c, t->module, t->cur_func,
                              w->call_tail ? WASM_INSN_RETURN_CALL_INDIRECT
                                           : WASM_INSN_CALL_INDIRECT,
                              w->imm);
           t->cur_func->insns[t->cur_func->ninsns - 1u].align = 0u;
         } else {
           u32 idx = sym_to_wasm_func(t, w->call_sym, NULL);
           emit_insn(t, w->call_tail ? WASM_INSN_RETURN_CALL : WASM_INSN_CALL,
                     (i64)idx);
         }
         /* Tail calls never return to this function: the operand stack is
          * polymorphic-unreachable after return_call*, so writing dst or
          * restoring the variadic stack pointer would be dead and would
          * also corrupt stack typing. (Variadic tail calls are rejected
          * upstream, so the variadic SP-restore guard is defensive.) */
         if (!w->call_tail) {
           if (w->dst != REG_NONE) {
             emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
           }
           /* Restore SP after variadic call so loop-resident variadic calls
            * don't accumulate stack usage. Done after stashing the return
            * value into its local. */
           if (w->call_variadic && w->call_nvar) {
             emit_insn(t, WASM_INSN_LOCAL_GET, (i64)t->varcall_saved_sp_local);
             emit_insn(t, WASM_INSN_GLOBAL_SET, (i64)t->stack_pointer_global);
           }
         }
         break;
       }
       case WIR_RET: {
         if (w->cgop == 1) {
           /* Aggregate sret return: memcpy w->addr -> *sret_param, then
            * void return. The sret pointer was the hidden first wasm param.
            * NOTE: this still uses a byte loop rather than memory.copy so
            * the produced module remains loadable by the kit runtime
            * before the wasm-core default-feature change lands. The path
            * will collapse to memory.copy once the core's default feature
            * set includes WASM_FEATURE_BULK_MEMORY (subagent A). */
           if (t->sret_param_local == 0xffffffffu)
             wfail(t, "wasm: sret return without hidden sret param");
           for (u32 n = 0; n < w->agg.size; ++n) {
             /* Push destination address (sret_ptr) onto stack. The memarg
              * offset on the i32.store8 carries the per-byte offset. */
             emit_insn(t, WASM_INSN_LOCAL_GET, (i64)t->sret_param_local);
             /* Load src byte at (w->addr + n). */
             Operand src = w->addr;
             if (src.kind == OPK_INDIRECT)
               src.v.ind.ofs += (i32)n;
             else if (src.kind == OPK_GLOBAL)
               src.v.global.addend += n;
             uint64_t src_off;
             emit_addr_operand(t, src, &src_off);
             if (src.kind == OPK_LOCAL) src_off += n;
             wasm_func_add_mem_insn(t->c, t->module, t->cur_func,
                                    WASM_INSN_I32_LOAD8_U, 0, src_off, 0);
             wasm_func_add_mem_insn(t->c, t->module, t->cur_func,
                                    WASM_INSN_I32_STORE8, 0, n, 0);
           }
         } else if (w->dst != REG_NONE)
           emit_push_operand_reg(t, w->dst);
         else if (w->imm_kind == 1) {
           WasmValType vt = type_valtype(t, w->type);
           emit_push_imm(t, vt, w->imm_a);
         }
         if (t->has_stack_frame) {
           emit_insn(t, WASM_INSN_LOCAL_GET, (i64)t->frame_saved_sp_local);
           emit_insn(t, WASM_INSN_GLOBAL_SET, (i64)t->stack_pointer_global);
         }
         emit_insn(t, WASM_INSN_RETURN, 0);
         break;
       }
       case WIR_UNREACHABLE: {
         emit_insn(t, WASM_INSN_UNREACHABLE, 0);
         break;
       }
       case WIR_LOAD_LOCAL: {
         u32 wli = (u32)w->imm;
         emit_insn(t, WASM_INSN_LOCAL_GET, (i64)wli);
         emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
         break;
       }
       case WIR_STORE_LOCAL: {
         u32 wli = (u32)w->imm;
         if (w->imm_kind == 1) {
           WasmValType vt = type_valtype(t, w->type);
           emit_push_imm(t, vt, w->imm_a);
         } else {
           emit_push_operand_reg(t, w->a);
         }
         emit_insn(t, WASM_INSN_LOCAL_SET, (i64)wli);
         break;
       }
       case WIR_LOAD_MEM: {
         emit_load_addr(t, w->addr, w->type, w->mem);
         emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
         break;
       }
       case WIR_STORE_MEM: {
         Operand src;
         memset(&src, 0, sizeof src);
         src.kind = w->imm_kind == WOP_IMM ? OPK_IMM : OPK_REG;
         src.type = w->type;
         if (src.kind == OPK_IMM)
           src.v.imm = w->imm_a;
         else
           src.v.reg = w->a;
         emit_store_addr(t, w->addr, w->type, src, w->mem, w->imm_kind, w->imm_a,
                         w->a);
         break;
       }
       case WIR_ADDR_OF: {
         uint64_t offset;
         emit_addr_operand(t, w->addr, &offset);
         if (offset) {
           emit_insn(t, WASM_INSN_I32_CONST, (i64)offset);
           emit_insn(t, WASM_INSN_I32_ADD, 0);
         }
         emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
         break;
       }
       case WIR_ALLOCA: {
         u32 align = (u32)w->imm;
         emit_insn(t, WASM_INSN_GLOBAL_GET, (i64)t->stack_pointer_global);
         emit_push_operand(t, w->imm_kind, w->imm_a, w->a,
                           w->type2 ? w->type2 : builtin_id(KIT_CG_BUILTIN_I32));
         if (w->type2 && type_valtype(t, w->type2) == WASM_VAL_I64)
           emit_insn(t, WASM_INSN_I32_WRAP_I64, 0);
         emit_insn(t, WASM_INSN_I32_SUB, 0);
         if (align > 1u) {
           emit_insn(t, WASM_INSN_I32_CONST, -(i64)align);
           emit_insn(t, WASM_INSN_I32_AND, 0);
         }
         emit_insn(t, WASM_INSN_LOCAL_TEE,
                   (i64)reg_local(t, w->dst, w->type, (RegClass)w->cls));
         emit_insn(t, WASM_INSN_GLOBAL_SET, (i64)t->stack_pointer_global);
         break;
       }
       case WIR_COPY_BYTES: {
         /* memory.copy: stack = dst_addr, src_addr, n; both memidx fields = 0.
          */
         Operand src_addr;
         if (w->imm_kind != WOP_REG)
           wfail(t, "wasm: copy_bytes source must be a register pointer");
         memset(&src_addr, 0, sizeof src_addr);
         src_addr.kind = OPK_INDIRECT;
         src_addr.type = w->addr.type;
         src_addr.v.ind.base = w->a;
         src_addr.v.ind.index = REG_NONE;
         src_addr.v.ind.log2_scale = 0;
         src_addr.v.ind.ofs = 0;
         if (w->agg.size == 0) break;
         emit_push_addr_value(t, w->addr);
         emit_push_addr_value(t, src_addr);
         emit_insn(t, WASM_INSN_I32_CONST, (i64)(uint32_t)w->agg.size);
         wasm_func_add_insn(t->c, t->module, t->cur_func, WASM_INSN_MEMORY_COPY,
                            0);
         /* dst memidx = 0, src memidx = 0 (kit-cc single-memory module). */
         t->cur_func->insns[t->cur_func->ninsns - 1u].memidx = 0;
         t->cur_func->insns[t->cur_func->ninsns - 1u].aux_idx = 0;
         break;
       }
       case WIR_SET_BYTES: {
         /* memory.fill: stack = dst_addr, value_i32, n; memidx = 0. */
         if (w->imm_kind != WOP_IMM)
           wfail(t, "wasm: set_bytes value must be immediate in v1");
         if (w->agg.size == 0) break;
         emit_push_addr_value(t, w->addr);
         emit_insn(t, WASM_INSN_I32_CONST, (i64)(w->imm_a & 0xff));
         emit_insn(t, WASM_INSN_I32_CONST, (i64)(uint32_t)w->agg.size);
         wasm_func_add_insn(t->c, t->module, t->cur_func, WASM_INSN_MEMORY_FILL,
                            0);
         t->cur_func->insns[t->cur_func->ninsns - 1u].memidx = 0;
         break;
       }
       case WIR_ATOMIC_LOAD: {
         WasmInsnKind k = atomic_load_kind_for(t, w->type, w->mem);
         u32 width = wasm_mem_width((uint8_t)k);
         emit_push_operand_reg(t, w->a);
         wasm_func_add_mem_insn(t->c, t->module, t->cur_func, k,
                                memarg_align_log2(w->mem.align, width), 0, 0);
         emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
         break;
       }
       case WIR_ATOMIC_STORE: {
         WasmInsnKind k = atomic_store_kind_for(t, w->type, w->mem);
         u32 width = wasm_mem_width((uint8_t)k);
         emit_push_operand_reg(t, w->a);
         emit_push_operand(t, w->imm_kind_b, w->imm_b, w->b, w->type);
         wasm_func_add_mem_insn(t->c, t->module, t->cur_func, k,
                                memarg_align_log2(w->mem.align, width), 0, 0);
         break;
       }
       case WIR_ATOMIC_RMW: {
         if ((KitCgAtomicOp)w->cgop == KIT_CG_ATOMIC_NAND) {
           /* wasm-threads has no atomic.rmw.nand. Expand to a cmpxchg retry
            * loop computing desired = ~(old & val):
            *   loop
            *     old = atomic.load(addr)            ; tee into old_local
            *     desired = (old & val) ^ -1
            *     got = atomic.rmw.cmpxchg(addr, old, desired)
            *     br_if loop  (got != old)           ; lost the race, retry
            *   end
            *   dst = old_local                      ; fetch returns prior value
            */
           WasmValType vt = type_valtype(t, w->type);
           WasmInsnKind load_k = atomic_load_kind_for(t, w->type, w->mem);
           WasmInsnKind cas_k = atomic_cmpxchg_kind_for(t, w->type, w->mem);
           u32 load_w = wasm_mem_width((uint8_t)load_k);
           u32 cas_w = wasm_mem_width((uint8_t)cas_k);
           int is64 = (vt == WASM_VAL_I64);
           u32 old_local = add_wasm_local(t, vt);
           emit_insn(t, WASM_INSN_LOOP, 0);
           /* addr (cmpxchg arg0) */
           emit_push_operand_reg(t, w->a);
           /* expected = atomic.load(addr), tee into old_local */
           emit_push_operand_reg(t, w->a);
           wasm_func_add_mem_insn(t->c, t->module, t->cur_func, load_k,
                                  memarg_align_log2(w->mem.align, load_w), 0, 0);
           emit_insn(t, WASM_INSN_LOCAL_TEE, (i64)old_local);
           /* desired = (old & val) ^ -1 */
           emit_insn(t, WASM_INSN_LOCAL_GET, (i64)old_local);
           emit_push_operand(t, w->imm_kind_b, w->imm_b, w->b, w->type);
           emit_insn(t, is64 ? WASM_INSN_I64_AND : WASM_INSN_I32_AND, 0);
           emit_push_imm(t, vt, -1);
           emit_insn(t, is64 ? WASM_INSN_I64_XOR : WASM_INSN_I32_XOR, 0);
           /* cmpxchg -> value previously in memory */
           wasm_func_add_mem_insn(t->c, t->module, t->cur_func, cas_k,
                                  memarg_align_log2(w->mem.align, cas_w), 0, 0);
           /* retry if memory had changed (got != expected) */
           emit_insn(t, WASM_INSN_LOCAL_GET, (i64)old_local);
           emit_insn(t, is64 ? WASM_INSN_I64_NE : WASM_INSN_I32_NE, 0);
           emit_insn(t, WASM_INSN_BR_IF, 0); /* 0 = innermost loop */
           emit_insn(t, WASM_INSN_END, 0);
           emit_insn(t, WASM_INSN_LOCAL_GET, (i64)old_local);
           emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
           break;
         }
         WasmInsnKind k =
             atomic_rmw_kind_for(t, (KitCgAtomicOp)w->cgop, w->type, w->mem);
         u32 width = wasm_mem_width((uint8_t)k);
         emit_push_operand_reg(t, w->a);
         emit_push_operand(t, w->imm_kind_b, w->imm_b, w->b, w->type);
         wasm_func_add_mem_insn(t->c, t->module, t->cur_func, k,
                                memarg_align_log2(w->mem.align, width), 0, 0);
         emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
         break;
       }
       case WIR_ATOMIC_CAS: {
         WasmInsnKind k = atomic_cmpxchg_kind_for(t, w->type, w->mem);
         WasmValType vt = type_valtype(t, w->type);
         u32 width = wasm_mem_width((uint8_t)k);
         /* Save expected into a fresh wasm local before consuming inputs. CG
          * may reuse one of (addr, expected, desired) regs for prior or ok;
          * reg_local() for w->dst/w->dst2 would then rebind that reg's local
          * mid-stream, and re-pushing expected via the (now-stale) mapping
          * would read an uninitialized local. The temp sidesteps that. */
         u32 saved_expected = add_wasm_local(t, vt);
         /* push addr; expected (tee into saved-expected, leaves on stack);
          * desired; cmpxchg -> prior on stack. */
         emit_push_operand_reg(t, w->a);
         emit_push_operand(t, w->imm_kind_b, w->imm_b, w->b, w->type);
         emit_insn(t, WASM_INSN_LOCAL_TEE, (i64)saved_expected);
         emit_push_operand(t, w->imm_kind_c, w->imm_c, w->op_c, w->type);
         wasm_func_add_mem_insn(t->c, t->module, t->cur_func, k,
                                memarg_align_log2(w->mem.align, width), 0, 0);
         /* All input regs have been consumed; safe to rebind. */
         u32 prior_local = reg_local(t, w->dst, w->type, (RegClass)w->cls);
         emit_insn(t, WASM_INSN_LOCAL_TEE, (i64)prior_local);
         emit_insn(t, WASM_INSN_LOCAL_GET, (i64)saved_expected);
         emit_insn(t, vt == WASM_VAL_I64 ? WASM_INSN_I64_EQ : WASM_INSN_I32_EQ,
                   0);
         emit_local_set(t, w->dst2,
                        w->type2 ? w->type2 : builtin_id(KIT_CG_BUILTIN_BOOL),
                        RC_INT);
         break;
       }
       case WIR_FENCE: {
         emit_insn(t, WASM_INSN_ATOMIC_FENCE, 0);
         break;
       }
       case WIR_JUMP: {
         u32 d = br_to_label(L, w->labels[0]);
         emit_insn(t, WASM_INSN_BR, (i64)d);
         break;
       }
       case WIR_CMP_BRANCH: {
         CmpOp cop = (CmpOp)w->cgop;
         if (cop >= CMP_OEQ_F) {
           emit_fp_cmp(t, cop, w, w->type);
         } else {
           push_cmp_operands(t, w, w->type);
           emit_insn(t, cmp_kind(t, cop, type_valtype(t, w->type)), 0);
         }
         u32 d = br_to_label(L, w->labels[0]);
         emit_insn(t, WASM_INSN_BR_IF, (i64)d);
         break;
       }
       case WIR_SWITCH: {
         emit_switch_br_table(t, L, w);
         break;
       }
       case WIR_SCOPE_OPEN: {
         if (L->nstack >= 1024u)
           wfail(t, "wasm: scope nesting too deep (max 1024)");
         LoweringScope* s = &L->stack[L->nstack++];
         s->id = w->scope_id;
         s->kind = w->cgop;
         if (w->cgop == SCOPE_LOOP) {
           /* (block (loop ...)); inside the body:
            *   br to loop top (cur_depth+1) = continue
            *   br to past block (cur_depth) = break (one more level out) */
           emit_insn(t, WASM_INSN_BLOCK, 0);
           L->cur_depth++;
           s->break_depth = L->cur_depth; /* `br N` lands AFTER block */
           emit_insn(t, WASM_INSN_LOOP, 0);
           L->cur_depth++;
           s->cont_depth = L->cur_depth; /* `br N` lands at LOOP top */
         } else if (w->cgop == SCOPE_BLOCK) {
           emit_insn(t, WASM_INSN_BLOCK, 0);
           L->cur_depth++;
           s->break_depth = L->cur_depth;
           s->cont_depth = L->cur_depth; /* unused */
         } else {
           wfail(t, "wasm: unknown scope kind %d", (int)w->cgop);
         }
         break;
       }
       case WIR_SCOPE_CLOSE: {
         if (L->nstack == 0) wfail(t, "wasm: scope_close without open scope");
         LoweringScope* s = &L->stack[L->nstack - 1u];
         if (s->kind == SCOPE_LOOP) {
           emit_insn(t, WASM_INSN_END, 0); /* close loop */
           L->cur_depth--;
           emit_insn(t, WASM_INSN_END, 0); /* close outer block */
           L->cur_depth--;
         } else {
           emit_insn(t, WASM_INSN_END, 0);
           L->cur_depth--;
         }
         L->nstack--;
         break;
       }
       case WIR_VA_START: {
         /* *ap_addr = va_ptr_param_local */
         emit_push_addr_value(t, w->addr);
         emit_insn(t, WASM_INSN_LOCAL_GET, (i64)t->va_ptr_param_local);
         wasm_func_add_mem_insn(t->c, t->module, t->cur_func,
                                WASM_INSN_I32_STORE, 2u, 0u, 0u);
         break;
       }
       case WIR_VA_ARG: {
         if (t->va_arg_tmp_addr_local == 0xffffffffu)
           t->va_arg_tmp_addr_local = add_wasm_local(t, WASM_VAL_I32);
         WasmValType vt = type_valtype(t, w->type);
         WasmInsnKind load_op;
         u32 width;
         if (vt == WASM_VAL_I64) {
           load_op = WASM_INSN_I64_LOAD;
           width = 8u;
         } else if (vt == WASM_VAL_F32) {
           load_op = WASM_INSN_F32_LOAD;
           width = 4u;
         } else if (vt == WASM_VAL_F64) {
           load_op = WASM_INSN_F64_LOAD;
           width = 8u;
         } else {
           load_op = WASM_INSN_I32_LOAD;
           width = 4u;
         }
         /* Load T from current *ap and stash into dst. */
         emit_push_addr_value(t, w->addr);
         emit_insn(t, WASM_INSN_LOCAL_TEE, (i64)t->va_arg_tmp_addr_local);
         wasm_func_add_mem_insn(t->c, t->module, t->cur_func, WASM_INSN_I32_LOAD,
                                2u, 0u, 0u);
         wasm_func_add_mem_insn(t->c, t->module, t->cur_func, load_op,
                                memarg_align_log2(width, width), 0u, 0u);
         emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
         /* Advance: *ap = *ap + 8. */
         emit_insn(t, WASM_INSN_LOCAL_GET, (i64)t->va_arg_tmp_addr_local);
         emit_insn(t, WASM_INSN_LOCAL_GET, (i64)t->va_arg_tmp_addr_local);
         wasm_func_add_mem_insn(t->c, t->module, t->cur_func, WASM_INSN_I32_LOAD,
                                2u, 0u, 0u);
         emit_insn(t, WASM_INSN_I32_CONST, (i64)8);
         emit_insn(t, WASM_INSN_I32_ADD, 0);
         wasm_func_add_mem_insn(t->c, t->module, t->cur_func,
                                WASM_INSN_I32_STORE, 2u, 0u, 0u);
         break;
       }
       case WIR_VA_COPY: {
         /* *dst_ap = *src_ap (single i32). */
         emit_push_addr_value(t, w->addr);
         emit_push_addr_value(t, w->call_sret_addr);
         wasm_func_add_mem_insn(t->c, t->module, t->cur_func, WASM_INSN_I32_LOAD,
                                2u, 0u, 0u);
         wasm_func_add_mem_insn(t->c, t->module, t->cur_func,
                                WASM_INSN_I32_STORE, 2u, 0u, 0u);
         break;
       }
       case WIR_INTRINSIC: {
         emit_intrinsic(t, w);
         break;
       }
       case WIR_ASM_BLOCK: {
         Heap* h_blk = t->c->ctx->heap;
         u32 nin = w->asm_nin;
         u32 nout = w->asm_nout;
         u32* in_locals = NULL;
         u32* out_locals = NULL;
         if (nin) {
           in_locals =
               (u32*)h_blk->alloc(h_blk, sizeof(u32) * nin, _Alignof(u32));
           if (!in_locals) wfail(t, "wasm: out of memory");
           /* defer per-input allocation until after output locals are known
            * so numeric tieback ("+r", "0".."9") can share. */
         }
         if (nout) {
           out_locals =
               (u32*)h_blk->alloc(h_blk, sizeof(u32) * nout, _Alignof(u32));
           if (!out_locals) wfail(t, "wasm: out of memory");
           for (u32 i = 0; i < nout; ++i)
             out_locals[i] =
                 add_wasm_local(t, valtype_for_type(t, w->asm_out_types[i]));
         }
         if (nin) {
           for (u32 i = 0; i < nin; ++i) {
             i32 share = w->asm_in_share_out[i];
             if (share >= 0 && (u32)share < nout) {
               in_locals[i] = out_locals[share];
             } else {
               in_locals[i] =
                   add_wasm_local(t, valtype_for_type(t, w->asm_in_types[i]));
             }
           }
         }
         /* Input materialization: push source operand, then for OPK_INDIRECT
          * inputs ("m" constraint with displacement) splice in
          * `i32.const ofs; i32.add` so the input local holds base+ofs.
          * Finally local.set into the input's local (which may be a shared
          * output local). */
         for (u32 i = 0; i < nin; ++i) {
           emit_push_operand(t, w->asm_in_kinds[i], w->asm_in_imms[i],
                             w->asm_in_regs[i], w->asm_in_types[i]);
           if (w->asm_in_kinds[i] == WOP_REG && w->asm_in_imms[i] != 0) {
             emit_push_imm(t, WASM_VAL_I32, w->asm_in_imms[i]);
             emit_insn(t, WASM_INSN_I32_ADD, 0);
           }
           emit_insn(t, WASM_INSN_LOCAL_SET, (i64)in_locals[i]);
         }
         /* Splice body, remapping local indices < nin+nout to the actual
          * wasm local table. */
         for (u32 i = 0; i < w->raw_ninsns; ++i) {
           WasmInsn in = w->raw_insns[i];
           if (in.kind == WASM_INSN_LOCAL_GET ||
               in.kind == WASM_INSN_LOCAL_SET ||
               in.kind == WASM_INSN_LOCAL_TEE) {
             if (in.imm >= 0 && (u64)in.imm < (u64)nin)
               in.imm = (i64)in_locals[in.imm];
             else if (in.imm >= (i64)nin && (u64)in.imm < (u64)(nin + nout))
               in.imm = (i64)out_locals[in.imm - (i64)nin];
           }
           t->module->current_loc = in.loc;
           wasm_func_add_insn(t->c, t->module, t->cur_func,
                              (WasmInsnKind)in.kind, in.imm);
           t->cur_func->insns[t->cur_func->ninsns - 1u] = in;
         }
         /* Output extraction: copy each output local into the destination
          * Reg's wasm local. */
         for (u32 i = 0; i < nout; ++i) {
           WasmValType ovt = valtype_for_type(t, w->asm_out_types[i]);
           RegClass cls =
               (ovt == WASM_VAL_F32 || ovt == WASM_VAL_F64) ? RC_FP : RC_INT;
           emit_insn(t, WASM_INSN_LOCAL_GET, (i64)out_locals[i]);
           emit_local_set(t, w->asm_out_regs[i], w->asm_out_types[i], cls);
         }
         if (in_locals) h_blk->free(h_blk, in_locals, sizeof(u32) * nin);
         if (out_locals) h_blk->free(h_blk, out_locals, sizeof(u32) * nout);
         break;
       }
       case WIR_LABEL: {
         break;
       }
     }
   }
 }
 
 static void linearize(WTarget* t) {
   LoweringState L;
   /* Rewrite WIR so every free label is bound to a synthetic SCOPE_BLOCK
    * (forward goto) or SCOPE_LOOP (backward goto). Switch islands are
    * pre-reordered into selector + WIR_SWITCH + case bodies, turning their
    * case labels into forward refs the same structuring covers. After this,
    * br_to_label resolves every jump through scope-bound machinery. */
   wasm_structurize(t);
   memset(&L, 0, sizeof L);
   L.t = t;
 
   if (t->has_stack_frame) {
     t->frame_saved_sp_local = add_wasm_local(t, WASM_VAL_I32);
     t->frame_base_local = add_wasm_local(t, WASM_VAL_I32);
     emit_insn(t, WASM_INSN_GLOBAL_GET, (i64)t->stack_pointer_global);
     emit_insn(t, WASM_INSN_LOCAL_TEE, (i64)t->frame_saved_sp_local);
     if (t->frame_size) {
       emit_insn(t, WASM_INSN_I32_CONST,
                 (i64)align_to_u32(t->frame_size, t->frame_align));
       emit_insn(t, WASM_INSN_I32_SUB, 0);
     }
     emit_insn(t, WASM_INSN_LOCAL_TEE, (i64)t->frame_base_local);
     emit_insn(t, WASM_INSN_GLOBAL_SET, (i64)t->stack_pointer_global);
   }
 
   /* Byval copy-in: for each ABI_ARG_INDIRECT param, copy the aggregate from
    * the caller's pointer into the callee's stack-frame buffer so callee
    * mutations are isolated (wasm32 BasicCABI). Byte-by-byte for v1; can be
    * promoted to wider chunks later. */
   for (u32 i = 0; i < t->nbyval_copies; ++i) {
     const WByvalCopy* bc = &t->byval_copies[i];
     const WSlot* s = &t->slots[bc->dst_slot_id];
     for (u32 n = 0; n < s->size; ++n) {
       /* dst: frame_base */
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)t->frame_base_local);
       /* src byte: i32.load8_u (ptr_local) offset=n */
       emit_insn(t, WASM_INSN_LOCAL_GET, (i64)bc->ptr_wasm_local);
       wasm_func_add_mem_insn(t->c, t->module, t->cur_func,
                              WASM_INSN_I32_LOAD8_U, 0, n, 0);
       wasm_func_add_mem_insn(t->c, t->module, t->cur_func, WASM_INSN_I32_STORE8,
                              0, s->frame_offset + n, 0);
     }
   }
 
   linearize_range(t, &L, 0, t->nwir);
   if (L.nstack != 0)
     wfail(t, "wasm: function ended with %u open scopes", L.nstack);
   /* If the body's last real WIR is a terminator (return / br / switch /
    * unreachable / tail call) buried inside nested blocks, kit's wasm
    * validator does not propagate the unreachable flag across enclosing
    * ENDs and would complain about a missing result at the implicit
    * function exit. Emit an explicit trailing `unreachable` so control[0]
    * is marked unreachable independent of the validator's propagation
    * rules. Also: when the terminator is a tail call we've already
    * emitted the linear-stack SP restore inline (see WIR_CALL handler),
    * and the function never reaches the post-body epilogue at runtime —
    * skip it to avoid emitting dead GLOBAL_GET/GLOBAL_SET pairs after the
    * return_call. */
   int last_is_tail_call = 0;
   {
     int needs_unreachable = 0;
     for (u32 i = t->nwir; i > 0; --i) {
       WIR* w = &t->wir[i - 1u];
       if (w->op == WIR_LABEL || w->op == WIR_SCOPE_OPEN ||
           w->op == WIR_SCOPE_CLOSE)
         continue;
       if (w->op == WIR_RET || w->op == WIR_JUMP || w->op == WIR_SWITCH ||
           w->op == WIR_UNREACHABLE) {
         needs_unreachable = 1;
       } else if ((w->op == WIR_CALL || w->op == WIR_CALL_INDIRECT) &&
                  w->call_tail) {
         needs_unreachable = 1;
         last_is_tail_call = 1;
       }
       break;
     }
     if (needs_unreachable) emit_insn(t, WASM_INSN_UNREACHABLE, 0);
   }
   if (t->has_stack_frame && !t->dead && !last_is_tail_call) {
     emit_insn(t, WASM_INSN_LOCAL_GET, (i64)t->frame_saved_sp_local);
     emit_insn(t, WASM_INSN_GLOBAL_SET, (i64)t->stack_pointer_global);
   }
 }
 
 void wasm_func_end(CGTarget* tg) {
   WTarget* t = (WTarget*)tg;
   if (!t->cur_func) return;
   /* Linearize WIR into the WasmFunc body. */
   linearize(t);
   t->cur_fn_desc = NULL;
   t->cur_func = NULL;
   /* Free per-function WIR arg arrays. */
   Heap* h = t->c->ctx->heap;
   for (u32 i = 0; i < t->nwir; ++i) {
     WIR* w = &t->wir[i];
     if (w->call_args) {
       h->free(h, w->call_args, sizeof(Reg) * w->call_narg);
       h->free(h, w->call_arg_imms, sizeof(i64) * w->call_narg);
       h->free(h, w->call_arg_kinds, w->call_narg);
       h->free(h, w->call_arg_types, sizeof(KitCgTypeId) * w->call_narg);
       if (w->call_arg_addrs)
         h->free(h, w->call_arg_addrs, sizeof(Operand) * w->call_narg);
       w->call_args = NULL;
       w->call_arg_imms = NULL;
       w->call_arg_kinds = NULL;
       w->call_arg_types = NULL;
       w->call_arg_addrs = NULL;
     }
     if (w->switch_cases) {
       h->free(h, w->switch_cases, sizeof(CGSwitchCase) * w->switch_ncases);
       w->switch_cases = NULL;
       w->switch_ncases = 0;
     }
     if (w->raw_insns) {
       h->free(h, w->raw_insns, sizeof(WasmInsn) * w->raw_ninsns);
       w->raw_insns = NULL;
       w->raw_ninsns = 0;
     }
     if (w->asm_in_kinds) {
       h->free(h, w->asm_in_kinds, w->asm_nin);
       h->free(h, w->asm_in_imms, sizeof(i64) * w->asm_nin);
       h->free(h, w->asm_in_regs, sizeof(Reg) * w->asm_nin);
       h->free(h, w->asm_in_types, sizeof(KitCgTypeId) * w->asm_nin);
       h->free(h, w->asm_in_share_out, sizeof(i32) * w->asm_nin);
       w->asm_in_kinds = NULL;
       w->asm_in_imms = NULL;
       w->asm_in_regs = NULL;
       w->asm_in_types = NULL;
       w->asm_in_share_out = NULL;
       w->asm_nin = 0;
     }
     if (w->asm_out_regs) {
       h->free(h, w->asm_out_regs, sizeof(Reg) * w->asm_nout);
       h->free(h, w->asm_out_types, sizeof(KitCgTypeId) * w->asm_nout);
       w->asm_out_regs = NULL;
       w->asm_out_types = NULL;
       w->asm_nout = 0;
     }
   }
   t->nwir = 0;
 }
 
 /* CGTarget alias hook. cg/session.c has already shared (section_id, value)
  * between alias_sym and target_sym at the ObjBuilder layer, which covers
  * data aliases (apply_sym_fixups reads section_id/value directly off the
  * ObjSym). Function aliases need extra wiring: the wasm function payload
  * lives in a target-side side-table (sym_to_func), not in obj sections,
  * and the alias's external linker name needs its own WasmExport entry. */
 void wasm_alias(CGTarget* tg, ObjSymId alias_sym, ObjSymId target_sym,
                 KitCgTypeId type) {
   WTarget* t = (WTarget*)tg;
   const ObjSym* tsym;
   const ObjSym* asym;
   (void)type;
   if (t->dead) return;
   /* Aliases are processed before any function body is emitted, so the module
    * may not exist yet; sym_to_wasm_func / wasm_add_export both need it. */
   ensure_module(t);
   tsym = obj_symbol_get(t->obj, target_sym);
   if (!tsym) wfail(t, "wasm: alias against unknown target symbol");
   if (tsym->kind == SK_FUNC) {
     /* Mirror sym_to_func so any later WIR_CALL against the alias resolves
      * to the target's wasm function index. */
     u32 idx = sym_to_wasm_func(t, target_sym, NULL);
     if (alias_sym >= t->sym_to_func_cap) {
       Heap* h = t->c->ctx->heap;
       u32 nc = t->sym_to_func_cap ? t->sym_to_func_cap : 16u;
       while (nc <= alias_sym) nc *= 2u;
       u32* p =
           (u32*)h->realloc(h, t->sym_to_func, sizeof(u32) * t->sym_to_func_cap,
                            sizeof(u32) * nc, _Alignof(u32));
       if (!p) wfail(t, "wasm: out of memory");
       for (u32 i = t->sym_to_func_cap; i < nc; ++i) p[i] = 0;
       t->sym_to_func = p;
       t->sym_to_func_cap = nc;
     }
     t->sym_to_func[alias_sym] = idx + 1u;
     /* Export under the alias's linker name when non-local. Mirrors the
      * export logic at the end of wasm_func_begin. */
     asym = obj_symbol_get(t->obj, alias_sym);
     if (asym && asym->bind != SB_LOCAL) {
       const char* name = pool_sym_cstr(t->c->global, asym->name, NULL);
       if (name && *name) {
         Heap* h = t->c->ctx->heap;
         size_t nlen = strlen(name);
         char* exp_name = (char*)h->alloc(h, nlen + 1u, 1);
         WasmExport* e;
         memcpy(exp_name, name, nlen + 1u);
         e = wasm_add_export(t->c, t->module);
         e->name = exp_name;
         e->kind = 0; /* function export */
         e->index = idx;
       }
     }
     return;
   }
   if (tsym->kind == SK_OBJ) {
     /* Data aliases: obj_symbol_define has already shared (section_id,
      * value), and apply_sym_fixups reads those directly. Nothing more
      * to do here — but diagnose if the target hasn't been defined yet
      * (it would produce a bogus address at finalize). */
     if (tsym->section_id == OBJ_SEC_NONE) {
       wfail(t, "wasm: data alias against undefined target symbol");
     }
     return;
   }
   wfail(t, "wasm target: alias of symbol kind %u not yet supported",
         (unsigned)tsym->kind);
 }
 
 /* Assign each SF_ALLOC (non-EXEC) ObjBuilder section a compact base in
  * linear memory. Walks sections in id order so the layout is deterministic.
  * Each base is aligned to the section's required alignment and lives in
  * t->section_base[sid]. Returns the next unused offset (end of data image). */
 static u32 assign_section_bases(WTarget* t) {
   Heap* h = t->c->ctx->heap;
   u32 nsec = obj_section_count(t->obj);
   if (nsec > t->section_base_cap) {
     u32 nc = t->section_base_cap ? t->section_base_cap : 4u;
     while (nc < nsec) nc *= 2u;
     void* p = h->realloc(h, t->section_base, sizeof(u32) * t->section_base_cap,
                          sizeof(u32) * nc, _Alignof(u32));
     if (!p) wfail(t, "wasm: out of memory");
     t->section_base = (u32*)p;
     for (u32 i = t->section_base_cap; i < nc; ++i)
       t->section_base[i] = 0xFFFFFFFFu;
     t->section_base_cap = nc;
   }
   u32 next = WASM_DATA_NULL_GUARD;
   for (ObjSecId sid = 0; sid < nsec; ++sid) t->section_base[sid] = 0xFFFFFFFFu;
   for (ObjSecId sid = 1; sid < nsec; ++sid) {
     const Section* s = obj_section_get(t->obj, sid);
     if (!s || s->removed || !(s->flags & SF_ALLOC) || s->flags & SF_EXEC)
       continue;
     u32 align = s->align ? s->align : 1u;
     if (align < 1u) align = 1u;
     next = align_to_u32(next, align);
     t->section_base[sid] = next;
     u32 sz = (s->kind == SEC_BSS || s->sem == SSEM_NOBITS)
                  ? s->bss_size
                  : (u32)s->bytes.total;
     if (sz > UINT32_MAX - next) wfail(t, "wasm: linear memory image too large");
     next += sz;
   }
   return next;
 }
 
 /* Patch a single i32/i64 value into the linear-memory image buffer at
  * `offset`. Wasm is little-endian. */
 static void mem_write_le(u8* buf, u32 offset, u64 value, u32 width) {
   for (u32 i = 0; i < width; ++i) buf[offset + i] = (u8)(value >> (i * 8u));
 }
 
 /* Allocate aligned BSS-style space in linear memory for every SK_COMMON
  * symbol the ObjBuilder knows about. Called after assign_section_bases so
  * common storage sits past the last SF_ALLOC section. Records the assigned
  * base in t->common_base[id]; returns the next free cursor. */
 static u32 assign_common_bases(WTarget* t, u32 next) {
   Heap* h = t->c->ctx->heap;
   ObjSymIter* it = obj_symiter_new(t->obj);
   ObjSymEntry e;
   while (obj_symiter_next(it, &e)) {
     const ObjSym* os = e.sym;
     if (!os || os->removed) continue;
     if (os->kind != SK_COMMON) continue;
     u32 align = os->common_align ? (u32)os->common_align : 1u;
     if (align < 1u) align = 1u;
     if (e.id >= t->common_base_cap) {
       u32 nc = t->common_base_cap ? t->common_base_cap : 8u;
       while (nc <= e.id) nc *= 2u;
       void* p = h->realloc(h, t->common_base, sizeof(u32) * t->common_base_cap,
                            sizeof(u32) * nc, _Alignof(u32));
       if (!p) wfail(t, "wasm: out of memory");
       t->common_base = (u32*)p;
       for (u32 i = t->common_base_cap; i < nc; ++i)
         t->common_base[i] = 0xFFFFFFFFu;
       t->common_base_cap = nc;
     }
     next = align_to_u32(next, align);
     t->common_base[e.id] = next;
     u32 sz = (u32)os->size;
     if (sz > UINT32_MAX - next)
       wfail(t, "wasm: linear memory image too large (common symbols)");
     next += sz;
   }
   obj_symiter_free(it);
   return next;
 }
 
 /* Resolve `sym + addend` to a linear-memory address. Handles both
  * section-defined symbols (via t->section_base[sym->section_id]) and
  * common symbols (via t->common_base[sym]). Returns 0 and sets *ok=0 if
  * the symbol can't be resolved here; callers diagnose. */
 static u32 wasm_sym_linear_addr(WTarget* t, ObjSymId sym, i64 addend, int* ok) {
   const ObjSym* os = obj_symbol_get(t->obj, sym);
   *ok = 0;
   if (!os) return 0;
   if (os->kind == SK_COMMON) {
     if (sym >= t->common_base_cap || t->common_base[sym] == 0xFFFFFFFFu)
       return 0;
     *ok = 1;
     return t->common_base[sym] + (u32)addend;
   }
   if (os->section_id == OBJ_SEC_NONE) return 0;
   if (os->section_id >= t->section_base_cap ||
       t->section_base[os->section_id] == 0xFFFFFFFFu)
     return 0;
   *ok = 1;
   return t->section_base[os->section_id] + (u32)os->value + (u32)addend;
 }
 
 /* Apply each ObjBuilder relocation to the linear-memory image. Only
  * absolute (R_ABS32/R_ABS64) relocations are supported for now; PC-relative
  * and other kinds diagnose. */
 static void apply_data_relocs(WTarget* t, u8* mem) {
   u32 ntotal = obj_reloc_total(t->obj);
   for (u32 i = 0; i < ntotal; ++i) {
     const Reloc* r = obj_reloc_at(t->obj, i);
     if (!r || r->removed) continue;
     if (r->section_id == OBJ_SEC_NONE) continue;
     if (r->section_id >= t->section_base_cap ||
         t->section_base[r->section_id] == 0xFFFFFFFFu)
       continue;
     const Section* rs = obj_section_get(t->obj, r->section_id);
     if (!rs || rs->flags & SF_EXEC) continue;
     const ObjSym* tos = obj_symbol_get(t->obj, r->sym);
     if (!tos)
       wfail(t, "wasm: data relocation against unresolved symbol not supported");
     /* Function-symbol references in data sections (e.g. a static vtable
      * `static fn_t v = &foo;`) resolve to wasm function-table indices, not
      * linear-memory addresses. The funcref table is built before
      * apply_data_relocs runs, so the index is already known. */
     u32 width;
     u64 value;
     if (tos->kind == SK_FUNC) {
       if (r->kind != R_ABS32)
         wfail(t,
               "wasm: function-pointer data relocation kind %u not supported "
               "(only R_ABS32 on wasm32 target)",
               (unsigned)r->kind);
       if (r->addend != 0)
         wfail(t, "wasm: nonzero addend on function-pointer data relocation");
       u32 tbl_idx = func_table_index_for(t, r->sym);
       width = 4;
       value = (u64)tbl_idx;
       u32 dst_off = t->section_base[r->section_id] + r->offset;
       mem_write_le(mem, dst_off, value, width);
       continue;
     }
     if (tos->section_id == OBJ_SEC_NONE && tos->kind != SK_COMMON)
       wfail(t, "wasm: data relocation against unresolved symbol not supported");
     {
       int ok = 0;
       /* The addend is already added by `value = sym_addr + r->addend` below;
        * pass 0 here so we don't double-count. */
       u32 sym_addr = wasm_sym_linear_addr(t, r->sym, 0, &ok);
       if (!ok)
         wfail(t,
               "wasm: data relocation target symbol has no linear-memory "
               "address");
       switch (r->kind) {
         case R_ABS32:
           width = 4;
           value = (u64)(u32)((i64)sym_addr + r->addend);
           break;
         case R_ABS64:
           wfail(t,
                 "wasm: R_ABS64 data relocation not supported on wasm32 target");
         default:
           wfail(t, "wasm: unsupported data relocation kind %u",
                 (unsigned)r->kind);
       }
     }
     u32 dst_off = t->section_base[r->section_id] + r->offset;
     mem_write_le(mem, dst_off, value, width);
   }
 }
 
 /* Walk the deferred WSymFixup queue and patch the placeholder i32.const
  * imm in each WasmFunc.insns[] with the resolved absolute address. */
 static void apply_sym_fixups(WTarget* t) {
   for (u32 i = 0; i < t->sym_fixups_count; ++i) {
     WSymFixup fx = t->sym_fixups[i];
     int ok = 0;
     u32 addr = wasm_sym_linear_addr(t, fx.sym, fx.addend, &ok);
     if (!ok) wfail(t, "wasm: deferred symbol fixup against unresolved symbol");
     WasmFunc* f = &t->module->funcs[fx.wasm_func_idx];
     if (fx.insn_idx >= f->ninsns)
       wfail(t, "wasm: deferred symbol fixup insn_idx out of range");
     f->insns[fx.insn_idx].imm = (i64)addr;
   }
 }
 
 static void wasm_materialize_data(WTarget* t) {
   if (!t->has_memory) {
     /* No linear memory was needed by any function body or addr_of, so
      * symbol fixups should be empty by construction. */
     return;
   }
   u32 image_end = assign_section_bases(t);
   image_end = assign_common_bases(t, image_end);
   u32 stack_size = t->has_stack_pointer ? t->stack_size : 0u;
   u32 image = image_end ? align_to_u32(image_end, 16u) : WASM_DATA_NULL_GUARD;
   if (image > UINT32_MAX - stack_size)
     wfail(t, "wasm: linear memory image too large");
   /* Build a single active data segment covering 0..image. Passive segments
    * + memory.init would be needed for multi-TU linking; single-TU output
    * stays with the simpler shape. */
   WasmDataSegment* seg = NULL;
   if (image) {
     seg = wasm_add_data(t->c, t->module);
     seg->mode = WASM_SEG_ACTIVE;
     seg->memidx = 0;
     seg->offset = 0;
     wasm_data_set_bytes(t->c, t->module, seg, NULL, (u64)image);
   }
   u32 nsec = obj_section_count(t->obj);
   for (ObjSecId sid = 1; sid < nsec; ++sid) {
     const Section* s = obj_section_get(t->obj, sid);
     if (!s || s->removed || !(s->flags & SF_ALLOC) || s->flags & SF_EXEC)
       continue;
     if (s->kind == SEC_BSS || s->sem == SSEM_NOBITS || !s->bytes.total)
       continue;
     buf_flatten(&s->bytes, seg->bytes + t->section_base[sid]);
   }
   if (seg) apply_data_relocs(t, seg->bytes);
   apply_sym_fixups(t);
   t->data_end = image;
   u32 stack_top = (u32)align_to_u32(image + stack_size, 16u);
   t->module->memories[0].min_pages = (stack_top + 65535u) / 65536u;
   /* Shared memory requires has_max and max >= min. ensure_shared_memory set a
    * provisional wasm32-ceiling cap (65536 pages = 4 GiB); now that the final
    * layout is known, tighten max down to min so the module declares a snug,
    * fixed shared memory. The backend never emits memory.grow, so the memory is
    * non-growable regardless, and a 4 GiB declared max would otherwise force an
    * embedder (e.g. `kit run`) to reserve the full ceiling up front. */
   if (t->module->memories[0].shared) {
     t->module->memories[0].has_max = 1;
     t->module->memories[0].max_pages = t->module->memories[0].min_pages;
   }
   if (t->has_stack_pointer && t->stack_pointer_global < t->module->nglobals) {
     t->module->globals[t->stack_pointer_global].init.imm = stack_top;
   }
 }
 
 /* Static-data initializers (e.g. `static fn_t v[] = {&foo, &bar};`) go
  * through ObjBuilder relocations rather than wasm_addr_of, so they never
  * touch queue_func_table_fixup. Scan the reloc table once before building
  * the funcref table so every function whose address is referenced from data
  * also gets a table slot. apply_data_relocs then patches the linear-memory
  * image with the assigned index. */
 static void wasm_collect_func_data_refs(WTarget* t) {
   u32 ntotal = obj_reloc_total(t->obj);
   for (u32 i = 0; i < ntotal; ++i) {
     const Reloc* r = obj_reloc_at(t->obj, i);
     const ObjSym* tos;
     if (!r || r->removed) continue;
     if (r->section_id == OBJ_SEC_NONE) continue;
     {
       const Section* rs = obj_section_get(t->obj, r->section_id);
       if (!rs || rs->flags & SF_EXEC) continue; /* code-section relocs */
     }
     tos = obj_symbol_get(t->obj, r->sym);
     if (!tos || tos->kind != SK_FUNC) continue;
     (void)func_table_index_for(t, r->sym);
     (void)sym_to_wasm_func(t, r->sym, NULL);
   }
 }
 
 /* Build the single funcref table and its active element segment, then patch
  * every queued WFuncTableFixup's placeholder `i32.const 0` with the assigned
  * table index. Slot 0 stays reserved (call_indirect through index 0 traps on
  * the type check), so the first recorded function lands at index 1. Each
  * WasmElemSegment caps its funcs array at 64 entries; we chunk across
  * multiple segments when the address-taken set is larger. */
 static void wasm_materialize_functable(WTarget* t) {
   wasm_collect_func_data_refs(t);
   if (!t->has_func_table || t->func_table_count == 0) return;
   ensure_module(t);
   /* Table: non-growable, sized to hold the reserved null slot plus every
    * assigned entry. */
   WasmTable* tbl = wasm_add_table(t->c, t->module);
   tbl->elem_type = WASM_VAL_FUNCREF;
   tbl->min = 1u + t->func_table_count;
   tbl->max = tbl->min;
   tbl->has_max = 1;
   /* Active element segment populates table 0 starting at offset 1 (slot 0
    * stays null). Element segments are now heap-grown — no chunking needed. */
   {
     WasmElemSegment* seg = wasm_add_elem(t->c, t->module);
     seg->mode = WASM_SEG_ACTIVE;
     seg->elem_type = WASM_VAL_FUNCREF;
     seg->tableidx = 0;
     seg->offset = 1;
     for (u32 i = 0; i < t->func_table_count; ++i) {
       ObjSymId sym = t->func_table[i];
       wasm_elem_push_func(t->c, t->module, seg, sym_to_wasm_func(t, sym, NULL));
     }
   }
   /* Patch placeholders. */
   for (u32 i = 0; i < t->func_table_fixups_count; ++i) {
     WFuncTableFixup fx = t->func_table_fixups[i];
     u32 tbl_idx = func_table_index_for(t, fx.sym);
     WasmFunc* f = &t->module->funcs[fx.wasm_func_idx];
     if (fx.insn_idx >= f->ninsns)
       wfail(t, "wasm: function-pointer fixup insn_idx out of range");
     f->insns[fx.insn_idx].imm = (i64)tbl_idx;
   }
 }
 
 /* Wasm requires every import to occupy a lower function index than any
  * defined function. The backend, however, allocates a WasmFunc for any
  * direct-call target in walk order — so a defined function may end up at a
  * lower array index than an import created later by promote_import_func.
  * Reorder m->funcs so all imports precede all definitions, then walk every
  * function-index reference in the module and apply the old->new mapping. */
 static void wasm_reorder_funcs_imports_first(WTarget* t) {
   WasmModule* m = t->module;
   if (!m || m->nfuncs == 0) return;
   Heap* h = m->heap;
   u32 n = m->nfuncs;
   /* Quick check: bail out if imports are already before all definitions. */
   int seen_def = 0;
   int needs_reorder = 0;
   for (u32 i = 0; i < n; ++i) {
     if (m->funcs[i].is_import) {
       if (seen_def) {
         needs_reorder = 1;
         break;
       }
     } else {
       seen_def = 1;
     }
   }
   if (!needs_reorder) return;
   u32* old_to_new = (u32*)h->alloc(h, sizeof(u32) * n, _Alignof(u32));
   WasmFunc* new_funcs =
       (WasmFunc*)h->alloc(h, sizeof(WasmFunc) * n, _Alignof(WasmFunc));
   if (!old_to_new || !new_funcs) wfail(t, "wasm: out of memory");
   u32 w_idx = 0;
   for (u32 i = 0; i < n; ++i) {
     if (m->funcs[i].is_import) {
       new_funcs[w_idx] = m->funcs[i];
       old_to_new[i] = w_idx++;
     }
   }
   for (u32 i = 0; i < n; ++i) {
     if (!m->funcs[i].is_import) {
       new_funcs[w_idx] = m->funcs[i];
       old_to_new[i] = w_idx++;
     }
   }
   /* Swap arrays. Old buffer is freed via the module's heap-tracked
    * realloc bookkeeping when wasm_module_free runs; we just overwrite the
    * pointer + length here. */
   h->free(h, m->funcs, sizeof(WasmFunc) * m->cap_funcs);
   m->funcs = new_funcs;
   m->cap_funcs = n;
   /* Remap every funcidx-bearing slot in the module. */
   for (u32 fi = 0; fi < n; ++fi) {
     WasmFunc* f = &m->funcs[fi];
     for (u32 j = 0; j < f->ninsns; ++j) {
       WasmInsn* in = &f->insns[j];
       if (in->kind == WASM_INSN_CALL || in->kind == WASM_INSN_RETURN_CALL ||
           in->kind == WASM_INSN_REF_FUNC) {
         u32 old = (u32)in->imm;
         if (old < n) in->imm = (int64_t)old_to_new[old];
       }
     }
   }
   for (u32 i = 0; i < m->nexports; ++i) {
     if (m->exports[i].kind == 0u && m->exports[i].index < n)
       m->exports[i].index = old_to_new[m->exports[i].index];
   }
   for (u32 i = 0; i < m->nelems; ++i) {
     WasmElemSegment* seg = &m->elems[i];
     for (u32 j = 0; j < seg->nfuncs; ++j) {
       if (seg->funcs[j] < n) seg->funcs[j] = old_to_new[seg->funcs[j]];
     }
   }
   if (m->has_start && m->start_func < n)
     m->start_func = old_to_new[m->start_func];
   /* Update the backend's sym_to_func reverse map so any post-finalize lookups
    * (e.g. data-reloc fixups) resolve to the new indices. The map stores
    * idx+1 so 0 = "unassigned"; preserve that convention. */
   for (ObjSymId sym = 0; sym < t->sym_to_func_cap; ++sym) {
     if (t->sym_to_func[sym]) {
       u32 old = t->sym_to_func[sym] - 1u;
       if (old < n) t->sym_to_func[sym] = old_to_new[old] + 1u;
     }
   }
   h->free(h, old_to_new, sizeof(u32) * n);
 }
 
 /* Export the module's linear memory under the conventional name "memory" so
  * standard runtimes (browser/wasmtime/wasmer/Node) can find it. Only emits
  * when the module has at least one defined (non-import) memory and no
  * memory export already exists. */
 static void wasm_export_memory(WTarget* t) {
   WasmModule* m = t->module;
   if (!m) return;
   ensure_linear_memory(t);
   m = t->module;
   /* Find the first defined (non-import) memory. */
   u32 mem_idx = 0;
   int found = 0;
   for (u32 i = 0; i < m->nmemories; ++i) {
     if (!m->memories[i].is_import) {
       mem_idx = i;
       found = 1;
       break;
     }
   }
   if (!found) return;
   /* Skip if the user already added a memory export (e.g. via the WAT path
    * or future explicit-export hook). */
   for (u32 i = 0; i < m->nexports; ++i) {
     if (m->exports[i].kind == 2u && m->exports[i].index == mem_idx) return;
   }
   Heap* h = t->c->ctx->heap;
   WasmExport* e = wasm_add_export(t->c, m);
   static const char kName[] = "memory";
   char* dup = (char*)h->alloc(h, sizeof(kName), 1);
   if (!dup) wfail(t, "wasm: out of memory");
   memcpy(dup, kName, sizeof(kName));
   e->name = dup;
   e->kind = 2u; /* memory export */
   e->index = mem_idx;
 }
 
 /* Diagnose any WasmFunc that has neither a body nor import status — that's a
  * declaration with no definition, e.g. a function-pointer reference to an
  * extern whose call site never appeared (so we never saw an ABI to synthesize
  * an import signature from). Emitting such a function would produce a
  * malformed module. Diagnose by sym name so users can fix the source. */
 static void wasm_diagnose_unresolved_funcs(WTarget* t) {
   if (!t->module) return;
   for (ObjSymId sym = 1; sym < t->sym_to_func_cap; ++sym) {
     if (!t->sym_to_func[sym]) continue;
     u32 idx = t->sym_to_func[sym] - 1u;
     if (idx >= t->module->nfuncs) continue;
     WasmFunc* f = &t->module->funcs[idx];
     if (f->is_import) continue;
     if (f->ninsns != 0) continue;
     const ObjSym* os = obj_symbol_get(t->obj, sym);
     if (!os || os->section_id != OBJ_SEC_NONE) continue;
     const char* name = pool_sym_cstr(t->c->global, os->name, NULL);
     wfail(t,
           "wasm: undefined function '%s' has its address taken but no direct "
           "call was seen — cannot synthesize import signature; add a direct "
           "call or annotate the declaration",
           name ? name : "(anonymous)");
   }
 }
 
 void wasm_finalize(CGTarget* tg) {
   WTarget* t = (WTarget*)tg;
   wasm_materialize_functable(t);
   wasm_materialize_data(t);
   if (t->module) {
     wasm_diagnose_unresolved_funcs(t);
     wasm_export_memory(t);
     wasm_reorder_funcs_imports_first(t);
   }
   /* WasmModule remains attached to ObjBuilder via OBJ_EXT_WASM; emit_wasm
    * flushes it. */
 }
 
 static void wasm_module_freefn(Compiler* c, void* p) {
   (void)c;
   WasmModule* m = (WasmModule*)p;
   Heap* h = m->heap;
   wasm_module_free(m);
   h->free(h, m, sizeof *m);
 }
 
 WTarget* wasm_emit_target_new(Compiler* c, ObjBuilder* o, MCEmitter* mc) {
   Heap* h;
   WTarget* t;
   if (!c) return NULL;
   h = (Heap*)c->ctx->heap;
   t = (WTarget*)h->alloc(h, sizeof *t, _Alignof(WTarget));
   if (!t) return NULL;
   memset(t, 0, sizeof *t);
   t->base.c = c;
   t->base.obj = o;
   t->c = c;
   t->obj = o;
   (void)mc;
   return t;
 }
 
 void wasm_destroy(CGTarget* tg) {
   WTarget* t = (WTarget*)tg;
   Heap* h = t->c->ctx->heap;
   if (t->reg_to_local) h->free(h, t->reg_to_local, sizeof(u32) * t->reg_cap);
   if (t->reg_type) h->free(h, t->reg_type, sizeof(KitCgTypeId) * t->reg_cap);
   if (t->reg_cls) h->free(h, t->reg_cls, t->reg_cap);
   if (t->wir) h->free(h, t->wir, sizeof(WIR) * t->wir_cap);
   if (t->labels) h->free(h, t->labels, sizeof(WLabel) * t->labels_cap);
   if (t->slots) h->free(h, t->slots, sizeof(WSlot) * t->slots_cap);
   if (t->param_local_idx)
     h->free(h, t->param_local_idx, sizeof(u32) * t->param_local_idx_cap);
   if (t->byval_copies)
     h->free(h, t->byval_copies, sizeof(WByvalCopy) * t->byval_copies_cap);
   if (t->sym_to_func)
     h->free(h, t->sym_to_func, sizeof(u32) * t->sym_to_func_cap);
   if (t->funcs) h->free(h, t->funcs, sizeof(WFunc) * t->funcs_cap);
   if (t->section_base)
     h->free(h, t->section_base, sizeof(u32) * t->section_base_cap);
   if (t->common_base)
     h->free(h, t->common_base, sizeof(u32) * t->common_base_cap);
   if (t->sym_fixups)
     h->free(h, t->sym_fixups, sizeof(WSymFixup) * t->sym_fixups_cap);
   if (t->func_table)
     h->free(h, t->func_table, sizeof(ObjSymId) * t->func_table_cap);
   if (t->func_table_fixups)
     h->free(h, t->func_table_fixups,
             sizeof(WFuncTableFixup) * t->func_table_fixups_cap);
   h->free(h, t, sizeof *t);
 }
 
 /* -----------------------------------------------------------------
  * Module bootstrap: attach a WasmModule to the ObjBuilder so emit_wasm
  * can find it. Lazily on first func_begin.
  * ----------------------------------------------------------------- */
 
 static struct WasmModule* ensure_module(WTarget* t) {
   if (t->module) return t->module;
   Heap* h = t->c->ctx->heap;
   WasmModule* m = (WasmModule*)h->alloc(h, sizeof *m, _Alignof(WasmModule));
   if (!m) wfail(t, "wasm: out of memory");
   wasm_module_init(m, h);
   /* kit-produced modules always declare bulk-memory support: WIR_COPY_BYTES
    * / WIR_SET_BYTES lower to memory.copy / memory.fill unconditionally, and
    * the sret-return path emits memory.copy too. */
   m->features |= WASM_FEATURE_BULK_MEMORY;
   t->module = m;
   obj_ext_set(t->obj, OBJ_EXT_WASM, m, wasm_module_freefn);
   return m;
 }