kit

cg.c (168827B)

---

 #include <kit/support/arena.h>
 
 #include "runtime_abi.h"
 #include "wasm/wasm.h"
 
 static KitCgTypeId wasm_cg_type(KitCompiler* c, KitCgBuiltinTypes b,
                                 WasmValType vt) {
   switch (vt) {
     case WASM_VAL_I32:
       return b.id[KIT_CG_BUILTIN_I32];
     case WASM_VAL_I64:
       return b.id[KIT_CG_BUILTIN_I64];
     case WASM_VAL_F32:
       return b.id[KIT_CG_BUILTIN_F32];
     case WASM_VAL_F64:
       return b.id[KIT_CG_BUILTIN_F64];
     case WASM_VAL_FUNCREF:
     case WASM_VAL_EXTERNREF:
       return kit_cg_type_ptr(c, b.id[KIT_CG_BUILTIN_VOID], 0);
   }
   wasm_error(c, wasm_loc(0, 0), "wasm: unsupported value type");
   return b.id[KIT_CG_BUILTIN_I32];
 }
 
 static KitCgMemAccess wasm_cg_mem(KitCompiler* c, KitCgBuiltinTypes b,
                                   WasmValType vt) {
   KitCgMemAccess mem;
   memset(&mem, 0, sizeof mem);
   mem.type = wasm_cg_type(c, b, vt);
   return mem;
 }
 
 static KitCgMemAccess wasm_cg_mem_type(KitCgTypeId ty) {
   KitCgMemAccess mem;
   memset(&mem, 0, sizeof mem);
   mem.type = ty;
   return mem;
 }
 
 static void wasm_cg_push_zero(KitCompiler* c, KitCg* cg, KitCgBuiltinTypes b,
                               WasmValType vt) {
   KitCgTypeId ty = wasm_cg_type(c, b, vt);
   if (wasm_is_ref_type(vt))
     kit_cg_push_null(cg, ty);
   else if (vt == WASM_VAL_F32 || vt == WASM_VAL_F64)
     kit_cg_push_float(cg, 0.0, ty);
   else
     kit_cg_push_int(cg, 0, ty);
 }
 
 static KitCgTypeId wasm_load_storage_type(KitCgBuiltinTypes b, uint8_t kind) {
   switch (kind) {
     case WASM_INSN_I32_LOAD8_S:
     case WASM_INSN_I32_LOAD8_U:
     case WASM_INSN_I64_LOAD8_S:
     case WASM_INSN_I64_LOAD8_U:
       return b.id[KIT_CG_BUILTIN_I8];
     case WASM_INSN_I32_LOAD16_S:
     case WASM_INSN_I32_LOAD16_U:
     case WASM_INSN_I64_LOAD16_S:
     case WASM_INSN_I64_LOAD16_U:
       return b.id[KIT_CG_BUILTIN_I16];
     case WASM_INSN_I64_LOAD32_S:
     case WASM_INSN_I64_LOAD32_U:
     case WASM_INSN_I32_LOAD:
       return b.id[KIT_CG_BUILTIN_I32];
     case WASM_INSN_F32_LOAD:
       return b.id[KIT_CG_BUILTIN_F32];
     case WASM_INSN_F64_LOAD:
       return b.id[KIT_CG_BUILTIN_F64];
     default:
       return b.id[KIT_CG_BUILTIN_I64];
   }
 }
 
 static KitCgTypeId wasm_store_storage_type(KitCgBuiltinTypes b, uint8_t kind) {
   switch (kind) {
     case WASM_INSN_I32_STORE8:
     case WASM_INSN_I64_STORE8:
       return b.id[KIT_CG_BUILTIN_I8];
     case WASM_INSN_I32_STORE16:
     case WASM_INSN_I64_STORE16:
       return b.id[KIT_CG_BUILTIN_I16];
     case WASM_INSN_I64_STORE32:
     case WASM_INSN_I32_STORE:
       return b.id[KIT_CG_BUILTIN_I32];
     case WASM_INSN_F32_STORE:
       return b.id[KIT_CG_BUILTIN_F32];
     case WASM_INSN_F64_STORE:
       return b.id[KIT_CG_BUILTIN_F64];
     default:
       return b.id[KIT_CG_BUILTIN_I64];
   }
 }
 
 typedef enum WasmTrapKind {
   WASM_TRAP_UNREACHABLE,
   WASM_TRAP_DIVISION,
   WASM_TRAP_INVALID_CONVERSION,
   WASM_TRAP_BOUNDS,
   WASM_TRAP_TABLE,
   WASM_TRAP_SIGNATURE,
   WASM_TRAP_COUNT,
 } WasmTrapKind;
 
 typedef struct WasmCgRuntime {
   KitCgTypeId i8_ptr_ty;
   KitCgTypeId void_ptr_ty;
   KitCgTypeId memory_ty;
   KitCgTypeId func_import_ty;
   KitCgTypeId global_import_ty;
   KitCgTypeId table_entry_ty;
   KitCgTypeId table_entry_ptr_ty;
   KitCgTypeId table_ty;
   KitCgTypeId instance_ty;
   KitCgTypeId instance_ptr_ty;
   uint64_t table_entry_size;
   /* Byte offsets within the instance struct for each top-level slot. */
   uint64_t* memory_offset;
   uint64_t* func_import_offset;
   uint64_t* func_ref_entry_offset;
   uint64_t* global_offset;
   uint64_t* table_offset;
   uint64_t* table_entries_offset;
   /* Byte offsets within their containing record. */
   uint64_t memory_data_offset;
   uint64_t memory_pages_offset;
   uint64_t memory_max_pages_offset;
   uint64_t memory_flags_offset;
   uint64_t func_import_fn_offset;
   uint64_t global_import_addr_offset;
   uint64_t table_entries_ptr_offset;
   uint64_t table_len_offset;
   uint64_t table_max_offset;
   uint64_t table_entry_fn_offset;
   uint64_t table_entry_typeidx_offset;
   /* memory/func/global/table-indexed instance-record field numbers. Sized to
    * the actual module counts (m->nmemories, m->nfuncs, m->nglobals,
    * m->ntables) and arena-allocated in wasm_cg_build_runtime so there's no
    * fixed cap on any of them. */
   uint32_t* memory_field;
   uint32_t memory_data_field;
   uint32_t memory_pages_field;
   uint32_t memory_max_pages_field;
   uint32_t memory_flags_field;
   uint32_t* func_import_field;
   uint32_t* func_ref_entry_field;
   uint32_t* global_field;
   uint32_t global_import_addr_field;
   uint32_t* table_field;
   uint32_t* table_entries_field;
   uint32_t table_entry_fn_field;
   uint32_t table_entry_typeidx_field;
   uint32_t table_entries_ptr_field;
   uint32_t table_len_field;
   uint32_t table_max_field;
   /* Bulk-memory passive segment storage. Each module data/elem segment with
    * mode == WASM_SEG_PASSIVE has a slot in the instance; active segments also
    * get a slot for uniform indexing (memory.init/table.init are validated to
    * traps on dropped/active segment use; the slot's len is left zero for
    * active segments so memory.init traps on non-zero copies). */
   KitCgTypeId passive_data_ty;
   KitCgTypeId passive_elem_ty;
   uint32_t* passive_data_field;
   uint32_t passive_data_base_field;
   uint32_t passive_data_len_field;
   uint32_t* passive_elem_field;
   uint32_t* passive_elem_storage_field;
   uint32_t passive_elem_entries_field;
   uint32_t passive_elem_length_field;
   KitCgTypeId trap_func_ty;
   KitCgSym trap_syms[WASM_TRAP_COUNT];
 } WasmCgRuntime;
 
 static const char* wasm_trap_name(WasmTrapKind kind) {
   switch (kind) {
     case WASM_TRAP_UNREACHABLE:
       return "__kit_wasm_trap_unreachable";
     case WASM_TRAP_DIVISION:
       return "__kit_wasm_trap_division";
     case WASM_TRAP_INVALID_CONVERSION:
       return "__kit_wasm_trap_invalid_conversion";
     case WASM_TRAP_BOUNDS:
       return "__kit_wasm_trap_bounds";
     case WASM_TRAP_TABLE:
       return "__kit_wasm_trap_table";
     case WASM_TRAP_SIGNATURE:
       return "__kit_wasm_trap_signature";
     default:
       return "__kit_wasm_trap";
   }
 }
 
 static void wasm_cg_emit_raw_trap(KitCg* cg) {
   kit_cg_intrinsic(cg, KIT_CG_INTRIN_TRAP, 0, KIT_CG_TYPE_NONE);
   kit_cg_unreachable(cg);
 }
 
 static void wasm_cg_trap(KitCg* cg, const WasmCgRuntime* rt,
                          WasmTrapKind kind) {
   if (rt && kind < WASM_TRAP_COUNT && rt->trap_syms[kind])
     kit_cg_call_symbol(cg, rt->trap_syms[kind], 0,
                        (KitCgCallAttrs){.flags = KIT_CG_CALL_COLD});
   else
     kit_cg_intrinsic(cg, KIT_CG_INTRIN_TRAP, 0, KIT_CG_TYPE_NONE);
   kit_cg_unreachable(cg);
 }
 
 static void wasm_cg_trap_unreachable(KitCg* cg, const WasmCgRuntime* rt) {
   wasm_cg_trap(cg, rt, WASM_TRAP_UNREACHABLE);
 }
 static void wasm_cg_trap_division(KitCg* cg, const WasmCgRuntime* rt) {
   wasm_cg_trap(cg, rt, WASM_TRAP_DIVISION);
 }
 static void wasm_cg_trap_invalid_conversion(KitCg* cg,
                                             const WasmCgRuntime* rt) {
   wasm_cg_trap(cg, rt, WASM_TRAP_INVALID_CONVERSION);
 }
 static void wasm_cg_trap_bounds(KitCg* cg, const WasmCgRuntime* rt) {
   wasm_cg_trap(cg, rt, WASM_TRAP_BOUNDS);
 }
 static void wasm_cg_trap_table(KitCg* cg, const WasmCgRuntime* rt) {
   wasm_cg_trap(cg, rt, WASM_TRAP_TABLE);
 }
 static void wasm_cg_trap_signature(KitCg* cg, const WasmCgRuntime* rt) {
   wasm_cg_trap(cg, rt, WASM_TRAP_SIGNATURE);
 }
 
 static void wasm_indexed_name(char* name, size_t cap, const char* prefix,
                               uint32_t index) {
   size_t pos = 0;
   uint32_t n = index, div = 1000000000u;
   while (*prefix && pos + 1u < cap) name[pos++] = *prefix++;
   while (div > 1u && n / div == 0) div /= 10u;
   while (div && pos + 1u < cap) {
     name[pos++] = (char)('0' + (n / div) % 10u);
     div /= 10u;
   }
   name[pos] = '\0';
 }
 
 static uint64_t wasm_cg_field_offset(KitCompiler* c, KitCgTypeId ty,
                                      uint32_t index) {
   uint64_t off = 0;
   KitStatus st = kit_cg_type_record_field(c, ty, index, NULL, &off);
   if (st != KIT_OK)
     wasm_error(c, wasm_loc(0, 0), "wasm: failed to query field offset");
   return off;
 }
 
 static uint32_t wasm_cg_checked_add_u32(KitCompiler* c, uint32_t a, uint32_t b,
                                         KitSrcLoc loc) {
   if (UINT32_MAX - a < b)
     wasm_error(c, loc, "wasm: module layout is too large");
   return a + b;
 }
 
 static void wasm_cg_build_runtime(KitCompiler* c, KitCgBuiltinTypes b,
                                   const WasmModule* m, WasmCgRuntime* rt,
                                   KitArena* arena) {
   KitCgField memory_fields[4];
   KitCgField func_import_fields[1];
   KitCgField global_import_fields[1];
   KitCgField table_entry_fields[2];
   KitCgField table_fields[3];
   KitCgField passive_data_fields[2];
   KitCgField passive_elem_fields[2];
   /* Total instance fields = nmemories + (#import funcs) + nfuncs (one
    * func_ref entry per func) + nglobals + 2*ntables + ndata + nelems.
    * Allocate the upper bound from the arena so nfields can grow with the
    * module. */
   uint32_t instance_cap = 0;
   KitCgField* instance_fields =
       instance_cap ? kit_arena_zarray(arena, KitCgField, instance_cap) : NULL;
   uint32_t nfields = 0;
   uint32_t* memory_field_idx = NULL;
   uint32_t* func_import_field_idx = NULL;
   uint32_t* func_ref_entry_field_idx = NULL;
   uint32_t* global_field_idx = NULL;
   uint32_t* table_field_idx = NULL;
   uint32_t* table_entries_field_idx = NULL;
   memset(rt, 0, sizeof *rt);
 
   instance_cap =
       wasm_cg_checked_add_u32(c, instance_cap, m->nmemories, wasm_loc(0, 0));
   instance_cap =
       wasm_cg_checked_add_u32(c, instance_cap, m->nfuncs, wasm_loc(0, 0));
   instance_cap =
       wasm_cg_checked_add_u32(c, instance_cap, m->nfuncs, wasm_loc(0, 0));
   instance_cap =
       wasm_cg_checked_add_u32(c, instance_cap, m->nglobals, wasm_loc(0, 0));
   if (m->ntables > UINT32_MAX / 2u)
     wasm_error(c, wasm_loc(0, 0), "wasm: module layout is too large");
   instance_cap =
       wasm_cg_checked_add_u32(c, instance_cap, 2u * m->ntables, wasm_loc(0, 0));
   instance_cap =
       wasm_cg_checked_add_u32(c, instance_cap, m->ndata, wasm_loc(0, 0));
   if (m->nelems > UINT32_MAX / 2u)
     wasm_error(c, wasm_loc(0, 0), "wasm: module layout is too large");
   instance_cap =
       wasm_cg_checked_add_u32(c, instance_cap, 2u * m->nelems, wasm_loc(0, 0));
   instance_fields =
       instance_cap ? kit_arena_zarray(arena, KitCgField, instance_cap) : NULL;
 
   rt->memory_field =
       m->nmemories ? kit_arena_zarray(arena, uint32_t, m->nmemories) : NULL;
   rt->memory_offset =
       m->nmemories ? kit_arena_zarray(arena, uint64_t, m->nmemories) : NULL;
   rt->func_import_field =
       m->nfuncs ? kit_arena_zarray(arena, uint32_t, m->nfuncs) : NULL;
   rt->func_import_offset =
       m->nfuncs ? kit_arena_zarray(arena, uint64_t, m->nfuncs) : NULL;
   rt->func_ref_entry_field =
       m->nfuncs ? kit_arena_zarray(arena, uint32_t, m->nfuncs) : NULL;
   rt->func_ref_entry_offset =
       m->nfuncs ? kit_arena_zarray(arena, uint64_t, m->nfuncs) : NULL;
   rt->global_field =
       m->nglobals ? kit_arena_zarray(arena, uint32_t, m->nglobals) : NULL;
   rt->global_offset =
       m->nglobals ? kit_arena_zarray(arena, uint64_t, m->nglobals) : NULL;
   rt->table_field =
       m->ntables ? kit_arena_zarray(arena, uint32_t, m->ntables) : NULL;
   rt->table_offset =
       m->ntables ? kit_arena_zarray(arena, uint64_t, m->ntables) : NULL;
   rt->table_entries_field =
       m->ntables ? kit_arena_zarray(arena, uint32_t, m->ntables) : NULL;
   rt->table_entries_offset =
       m->ntables ? kit_arena_zarray(arena, uint64_t, m->ntables) : NULL;
   rt->passive_data_field =
       m->ndata ? kit_arena_zarray(arena, uint32_t, m->ndata) : NULL;
   rt->passive_elem_field =
       m->nelems ? kit_arena_zarray(arena, uint32_t, m->nelems) : NULL;
   rt->passive_elem_storage_field =
       m->nelems ? kit_arena_zarray(arena, uint32_t, m->nelems) : NULL;
   memory_field_idx =
       m->nmemories ? kit_arena_zarray(arena, uint32_t, m->nmemories) : NULL;
   func_import_field_idx =
       m->nfuncs ? kit_arena_zarray(arena, uint32_t, m->nfuncs) : NULL;
   func_ref_entry_field_idx =
       m->nfuncs ? kit_arena_zarray(arena, uint32_t, m->nfuncs) : NULL;
   global_field_idx =
       m->nglobals ? kit_arena_zarray(arena, uint32_t, m->nglobals) : NULL;
   table_field_idx =
       m->ntables ? kit_arena_zarray(arena, uint32_t, m->ntables) : NULL;
   table_entries_field_idx =
       m->ntables ? kit_arena_zarray(arena, uint32_t, m->ntables) : NULL;
   rt->i8_ptr_ty = kit_cg_type_ptr(c, b.id[KIT_CG_BUILTIN_I8], 0);
   rt->void_ptr_ty = rt->i8_ptr_ty;
   memset(memory_fields, 0, sizeof memory_fields);
   memory_fields[0].name = kit_sym_intern(c, KIT_SLICE_LIT("data"));
   memory_fields[0].type = rt->i8_ptr_ty;
   memory_fields[1].name = kit_sym_intern(c, KIT_SLICE_LIT("pages"));
   memory_fields[1].type = b.id[KIT_CG_BUILTIN_I64];
   memory_fields[2].name = kit_sym_intern(c, KIT_SLICE_LIT("max_pages"));
   memory_fields[2].type = b.id[KIT_CG_BUILTIN_I64];
   memory_fields[3].name = kit_sym_intern(c, KIT_SLICE_LIT("flags"));
   memory_fields[3].type = b.id[KIT_CG_BUILTIN_I32];
   rt->memory_ty = kit_cg_type_record(
       c, kit_sym_intern(c, KIT_SLICE_LIT("KitWasmMemory")), memory_fields, 4);
   rt->memory_data_offset = wasm_cg_field_offset(c, rt->memory_ty, 0);
   rt->memory_pages_offset = wasm_cg_field_offset(c, rt->memory_ty, 1);
   rt->memory_max_pages_offset = wasm_cg_field_offset(c, rt->memory_ty, 2);
   rt->memory_flags_offset = wasm_cg_field_offset(c, rt->memory_ty, 3);
   rt->memory_data_field = 0;
   rt->memory_pages_field = 1;
   rt->memory_max_pages_field = 2;
   rt->memory_flags_field = 3;
   memset(func_import_fields, 0, sizeof func_import_fields);
   func_import_fields[0].name = kit_sym_intern(c, KIT_SLICE_LIT("fn"));
   func_import_fields[0].type = rt->void_ptr_ty;
   rt->func_import_ty = kit_cg_type_record(
       c, kit_sym_intern(c, KIT_SLICE_LIT("KitWasmFuncImport")),
       func_import_fields, 1);
   rt->func_import_fn_offset = wasm_cg_field_offset(c, rt->func_import_ty, 0);
   memset(global_import_fields, 0, sizeof global_import_fields);
   global_import_fields[0].name = kit_sym_intern(c, KIT_SLICE_LIT("addr"));
   global_import_fields[0].type = rt->void_ptr_ty;
   rt->global_import_ty = kit_cg_type_record(
       c, kit_sym_intern(c, KIT_SLICE_LIT("KitWasmGlobalImport")),
       global_import_fields, 1);
   rt->global_import_addr_offset =
       wasm_cg_field_offset(c, rt->global_import_ty, 0);
   memset(table_entry_fields, 0, sizeof table_entry_fields);
   table_entry_fields[0].name = kit_sym_intern(c, KIT_SLICE_LIT("fn"));
   table_entry_fields[0].type = rt->void_ptr_ty;
   table_entry_fields[1].name = kit_sym_intern(c, KIT_SLICE_LIT("typeidx"));
   table_entry_fields[1].type = b.id[KIT_CG_BUILTIN_I32];
   rt->table_entry_ty = kit_cg_type_record(
       c, kit_sym_intern(c, KIT_SLICE_LIT("KitWasmTableEntry")),
       table_entry_fields, 2);
   rt->table_entry_ptr_ty = kit_cg_type_ptr(c, rt->table_entry_ty, 0);
   rt->table_entry_fn_offset = wasm_cg_field_offset(c, rt->table_entry_ty, 0);
   rt->table_entry_typeidx_offset =
       wasm_cg_field_offset(c, rt->table_entry_ty, 1);
   rt->table_entry_size = kit_cg_type_size(c, rt->table_entry_ty);
   rt->table_entry_fn_field = 0;
   rt->table_entry_typeidx_field = 1;
   memset(table_fields, 0, sizeof table_fields);
   table_fields[0].name = kit_sym_intern(c, KIT_SLICE_LIT("entries"));
   table_fields[0].type = rt->table_entry_ptr_ty;
   table_fields[1].name = kit_sym_intern(c, KIT_SLICE_LIT("len"));
   table_fields[1].type = b.id[KIT_CG_BUILTIN_I32];
   table_fields[2].name = kit_sym_intern(c, KIT_SLICE_LIT("max"));
   table_fields[2].type = b.id[KIT_CG_BUILTIN_I32];
   rt->table_ty = kit_cg_type_record(
       c, kit_sym_intern(c, KIT_SLICE_LIT("KitWasmTable")), table_fields, 3);
   rt->table_entries_ptr_offset = wasm_cg_field_offset(c, rt->table_ty, 0);
   rt->table_len_offset = wasm_cg_field_offset(c, rt->table_ty, 1);
   rt->table_max_offset = wasm_cg_field_offset(c, rt->table_ty, 2);
   rt->table_entries_ptr_field = 0;
   rt->table_len_field = 1;
   rt->table_max_field = 2;
   memset(passive_data_fields, 0, sizeof passive_data_fields);
   passive_data_fields[0].name = kit_sym_intern(c, KIT_SLICE_LIT("base"));
   passive_data_fields[0].type = rt->i8_ptr_ty;
   passive_data_fields[1].name = kit_sym_intern(c, KIT_SLICE_LIT("len"));
   passive_data_fields[1].type = b.id[KIT_CG_BUILTIN_I64];
   rt->passive_data_ty = kit_cg_type_record(
       c, kit_sym_intern(c, KIT_SLICE_LIT("KitWasmPassiveDataSegment")),
       passive_data_fields, 2);
   rt->passive_data_base_field = 0;
   rt->passive_data_len_field = 1;
   memset(passive_elem_fields, 0, sizeof passive_elem_fields);
   passive_elem_fields[0].name = kit_sym_intern(c, KIT_SLICE_LIT("entries"));
   passive_elem_fields[0].type = rt->table_entry_ptr_ty;
   passive_elem_fields[1].name = kit_sym_intern(c, KIT_SLICE_LIT("length"));
   passive_elem_fields[1].type = b.id[KIT_CG_BUILTIN_I32];
   rt->passive_elem_ty = kit_cg_type_record(
       c, kit_sym_intern(c, KIT_SLICE_LIT("KitWasmPassiveElemSegment")),
       passive_elem_fields, 2);
   rt->passive_elem_entries_field = 0;
   rt->passive_elem_length_field = 1;
   for (uint32_t i = 0; i < m->nmemories; ++i) {
     char name[40];
     if (i == 0)
       memcpy(name, "memory", sizeof "memory");
     else
       wasm_indexed_name(name, sizeof name, "memory_", i);
     rt->memory_field[i] = nfields;
     memory_field_idx[i] = nfields;
     instance_fields[nfields].name = kit_sym_intern(c, kit_slice_cstr(name));
     instance_fields[nfields].type = rt->memory_ty;
     nfields++;
   }
   for (uint32_t i = 0; i < m->nfuncs; ++i) {
     char name[40];
     if (!m->funcs[i].is_import) continue;
     wasm_indexed_name(name, sizeof name, "import_func_", i);
     rt->func_import_field[i] = nfields;
     func_import_field_idx[i] = nfields;
     instance_fields[nfields].name = kit_sym_intern(c, kit_slice_cstr(name));
     instance_fields[nfields].type = rt->func_import_ty;
     nfields++;
   }
   for (uint32_t i = 0; i < m->nfuncs; ++i) {
     char name[40];
     wasm_indexed_name(name, sizeof name, "func_ref_", i);
     rt->func_ref_entry_field[i] = nfields;
     func_ref_entry_field_idx[i] = nfields;
     instance_fields[nfields].name = kit_sym_intern(c, kit_slice_cstr(name));
     instance_fields[nfields].type = rt->table_entry_ty;
     nfields++;
   }
   for (uint32_t i = 0; i < m->nglobals; ++i) {
     char name[32];
     wasm_indexed_name(name, sizeof name,
                       m->globals[i].is_import ? "import_global_" : "global_",
                       i);
     rt->global_field[i] = nfields;
     global_field_idx[i] = nfields;
     instance_fields[nfields].name = kit_sym_intern(c, kit_slice_cstr(name));
     instance_fields[nfields].type =
         m->globals[i].is_import ? rt->global_import_ty
                                 : wasm_cg_type(c, b, m->globals[i].type);
     nfields++;
   }
   for (uint32_t i = 0; i < m->ntables; ++i) {
     char name[40];
     uint32_t max = m->tables[i].has_max ? m->tables[i].max : m->tables[i].min;
     wasm_indexed_name(name, sizeof name, "table_", i);
     rt->table_field[i] = nfields;
     table_field_idx[i] = nfields;
     instance_fields[nfields].name = kit_sym_intern(c, kit_slice_cstr(name));
     instance_fields[nfields].type = rt->table_ty;
     nfields++;
     wasm_indexed_name(name, sizeof name, "table_entries_", i);
     rt->table_entries_field[i] = nfields;
     table_entries_field_idx[i] = nfields;
     instance_fields[nfields].name = kit_sym_intern(c, kit_slice_cstr(name));
     instance_fields[nfields].type =
         kit_cg_type_array(c, rt->table_entry_ty, max ? max : 1u);
     nfields++;
   }
   for (uint32_t i = 0; i < m->ndata; ++i) {
     char name[40];
     wasm_indexed_name(name, sizeof name, "passive_data_", i);
     rt->passive_data_field[i] = nfields;
     instance_fields[nfields].name = kit_sym_intern(c, kit_slice_cstr(name));
     instance_fields[nfields].type = rt->passive_data_ty;
     nfields++;
   }
   for (uint32_t i = 0; i < m->nelems; ++i) {
     char name[40];
     uint32_t nfuncs = m->elems[i].nfuncs;
     wasm_indexed_name(name, sizeof name, "passive_elem_", i);
     rt->passive_elem_field[i] = nfields;
     instance_fields[nfields].name = kit_sym_intern(c, kit_slice_cstr(name));
     instance_fields[nfields].type = rt->passive_elem_ty;
     nfields++;
     wasm_indexed_name(name, sizeof name, "passive_elem_storage_", i);
     rt->passive_elem_storage_field[i] = nfields;
     instance_fields[nfields].name = kit_sym_intern(c, kit_slice_cstr(name));
     instance_fields[nfields].type =
         kit_cg_type_array(c, rt->table_entry_ty, nfuncs ? nfuncs : 1u);
     nfields++;
   }
   rt->instance_ty =
       kit_cg_type_record(c, kit_sym_intern(c, KIT_SLICE_LIT("KitWasmInstance")),
                          instance_fields, nfields);
   rt->instance_ptr_ty = kit_cg_type_ptr(c, rt->instance_ty, 0);
   for (uint32_t i = 0; i < m->nmemories; ++i)
     rt->memory_offset[i] =
         wasm_cg_field_offset(c, rt->instance_ty, memory_field_idx[i]);
   for (uint32_t i = 0; i < m->nfuncs; ++i) {
     if (m->funcs[i].is_import)
       rt->func_import_offset[i] =
           wasm_cg_field_offset(c, rt->instance_ty, func_import_field_idx[i]);
     rt->func_ref_entry_offset[i] =
         wasm_cg_field_offset(c, rt->instance_ty, func_ref_entry_field_idx[i]);
   }
   for (uint32_t i = 0; i < m->nglobals; ++i)
     rt->global_offset[i] =
         wasm_cg_field_offset(c, rt->instance_ty, global_field_idx[i]);
   for (uint32_t i = 0; i < m->ntables; ++i) {
     rt->table_offset[i] =
         wasm_cg_field_offset(c, rt->instance_ty, table_field_idx[i]);
     rt->table_entries_offset[i] =
         wasm_cg_field_offset(c, rt->instance_ty, table_entries_field_idx[i]);
   }
 }
 
 /* Push the instance pointer rvalue (loaded from the instance parameter slot).
  * Stack: [] -> [void*]. Callers fold a struct offset into the memop's EA. */
 static void wasm_cg_push_instance_ptr(KitCg* cg, const WasmCgRuntime* rt,
                                       KitCgLocal instance_local) {
   kit_cg_push_local(cg, instance_local);
   kit_cg_load(cg, wasm_cg_mem_type(rt->instance_ptr_ty));
 }
 
 /* Push the instance as a record PLACE (the pointee of the instance pointer).
  * Stack: [] -> [place]. Callers project struct fields off it with field/elem.
  */
 static void wasm_cg_push_instance_lvalue(KitCg* cg, const WasmCgRuntime* rt,
                                          KitCgLocal instance_local) {
   wasm_cg_push_instance_ptr(cg, rt, instance_local);
   kit_cg_deref(cg, 0);
 }
 
 /* Add a constant byte offset to the pointer rvalue on TOS, retyping to
  * `result_ptr_ty`. No-op when offset == 0 (and the type is not retyped). */
 static void wasm_cg_ptr_add_offset(KitCg* cg, KitCgBuiltinTypes b,
                                    uint64_t offset, KitCgTypeId result_ptr_ty) {
   KitCgTypeId i64_ty = b.id[KIT_CG_BUILTIN_I64];
   if (offset == 0) {
     kit_cg_bitcast(cg, result_ptr_ty);
     return;
   }
   kit_cg_ptr_to_int(cg, i64_ty);
   kit_cg_push_int(cg, offset, i64_ty);
   kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
   kit_cg_int_to_ptr(cg, result_ptr_ty);
 }
 
 /* Load the i8* data pointer from instance->memories[memidx].data. */
 static void wasm_cg_push_memory_data_ptr(KitCg* cg, const WasmCgRuntime* rt,
                                          KitCgLocal instance_local,
                                          uint32_t memidx) {
   wasm_cg_push_instance_ptr(cg, rt, instance_local);
   kit_cg_deref(cg,
                (int64_t)(rt->memory_offset[memidx] + rt->memory_data_offset));
   kit_cg_load(cg, wasm_cg_mem_type(rt->i8_ptr_ty));
 }
 
 static void wasm_cg_push_memory_pages_lvalue(KitCg* cg, const WasmCgRuntime* rt,
                                              KitCgLocal instance_local,
                                              uint32_t memidx) {
   wasm_cg_push_instance_lvalue(cg, rt, instance_local);
   kit_cg_field(cg, rt->memory_field[memidx]);
   kit_cg_field(cg, rt->memory_pages_field);
 }
 
 /* Load instance->import_funcs[func_index].fn as a void*. */
 static void wasm_cg_push_import_func_ptr(KitCg* cg, const WasmCgRuntime* rt,
                                          KitCgLocal instance_local,
                                          uint32_t func_index) {
   wasm_cg_push_instance_ptr(cg, rt, instance_local);
   kit_cg_deref(cg, (int64_t)(rt->func_import_offset[func_index] +
                              rt->func_import_fn_offset));
   kit_cg_load(cg, wasm_cg_mem_type(rt->void_ptr_ty));
 }
 
 /* Push a pointer rvalue to instance->globals[global_index]'s value cell,
  * dereferencing the import indirection if needed. Result type is T* where T is
  * the global's value type. */
 static void wasm_cg_push_global_value_ptr(
     KitCompiler* c, KitCg* cg, KitCgBuiltinTypes b, const WasmCgRuntime* rt,
     KitCgLocal instance_local, const WasmModule* m, uint32_t global_index) {
   KitCgTypeId ptr_ty =
       kit_cg_type_ptr(c, wasm_cg_type(c, b, m->globals[global_index].type), 0);
   if (m->globals[global_index].is_import) {
     wasm_cg_push_instance_ptr(cg, rt, instance_local);
     kit_cg_deref(cg, (int64_t)(rt->global_offset[global_index] +
                                rt->global_import_addr_offset));
     kit_cg_load(cg, wasm_cg_mem_type(rt->void_ptr_ty));
     kit_cg_dup(cg);
     kit_cg_push_null(cg, rt->void_ptr_ty);
     kit_cg_int_cmp(cg, KIT_CG_INT_NE);
     {
       KitCgLabel ok = kit_cg_label_new(cg);
       kit_cg_branch_true(cg, ok);
       wasm_cg_trap_table(cg, rt);
       kit_cg_label_place(cg, ok);
     }
     kit_cg_bitcast(cg, ptr_ty);
   } else {
     wasm_cg_push_instance_ptr(cg, rt, instance_local);
     wasm_cg_ptr_add_offset(cg, b, rt->global_offset[global_index], ptr_ty);
   }
 }
 
 static void wasm_cg_push_table_lvalue(KitCg* cg, const WasmCgRuntime* rt,
                                       KitCgLocal instance_local,
                                       uint32_t table_index) {
   wasm_cg_push_instance_lvalue(cg, rt, instance_local);
   kit_cg_field(cg, rt->table_field[table_index]);
 }
 
 static void wasm_cg_push_table_entry_lvalue(KitCg* cg, const WasmCgRuntime* rt,
                                             KitCgLocal instance_local,
                                             uint32_t table_index,
                                             KitCgLocal index_local,
                                             KitCgMemAccess index_mem) {
   wasm_cg_push_table_lvalue(cg, rt, instance_local, table_index);
   kit_cg_field(cg, rt->table_entries_ptr_field);
   kit_cg_load(cg, wasm_cg_mem_type(rt->table_entry_ptr_ty));
   kit_cg_push_local(cg, index_local);
   kit_cg_load(cg, index_mem);
   kit_cg_elem(cg, 0);
 }
 
 /* Push a pointer rvalue to instance->tables[table_index].entries[index_local].
  * The index is loaded from a temp local supplied by the caller (mirrors the
  * previous helper's signature). */
 static void wasm_cg_push_table_entry_ptr(KitCg* cg, KitCgBuiltinTypes b,
                                          const WasmCgRuntime* rt,
                                          KitCgLocal instance_local,
                                          uint32_t table_index,
                                          KitCgLocal index_local,
                                          KitCgMemAccess index_mem) {
   KitCgTypeId i64_ty = b.id[KIT_CG_BUILTIN_I64];
   /* Load entries pointer from instance->tables[i].entries. */
   wasm_cg_push_instance_ptr(cg, rt, instance_local);
   kit_cg_deref(cg, (int64_t)(rt->table_offset[table_index] +
                              rt->table_entries_ptr_offset));
   kit_cg_load(cg, wasm_cg_mem_type(rt->table_entry_ptr_ty));
   /* Compute entries + index * sizeof(entry) into a pointer rvalue. */
   kit_cg_ptr_to_int(cg, i64_ty);
   kit_cg_push_local(cg, index_local);
   kit_cg_load(cg, index_mem);
   kit_cg_zext(cg, i64_ty);
   kit_cg_push_int(cg, rt->table_entry_size, i64_ty);
   kit_cg_int_binop(cg, KIT_CG_INT_MUL, 0);
   kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
   kit_cg_int_to_ptr(cg, rt->table_entry_ptr_ty);
 }
 
 static void wasm_cg_push_passive_data_lvalue(KitCg* cg, const WasmCgRuntime* rt,
                                              KitCgLocal instance_local,
                                              uint32_t dataidx) {
   wasm_cg_push_instance_lvalue(cg, rt, instance_local);
   kit_cg_field(cg, rt->passive_data_field[dataidx]);
 }
 
 static void wasm_cg_push_passive_elem_lvalue(KitCg* cg, const WasmCgRuntime* rt,
                                              KitCgLocal instance_local,
                                              uint32_t elemidx) {
   wasm_cg_push_instance_lvalue(cg, rt, instance_local);
   kit_cg_field(cg, rt->passive_elem_field[elemidx]);
 }
 
 static void wasm_cg_push_passive_elem_storage_array_lvalue(
     KitCg* cg, const WasmCgRuntime* rt, KitCgLocal instance_local,
     uint32_t elemidx) {
   wasm_cg_push_instance_lvalue(cg, rt, instance_local);
   kit_cg_field(cg, rt->passive_elem_storage_field[elemidx]);
 }
 
 static void wasm_cg_memory_check(KitCompiler* c, KitCg* cg, KitCgBuiltinTypes b,
                                  const WasmModule* m, const WasmCgRuntime* rt,
                                  KitCgLocal instance_local,
                                  const WasmInsn* in) {
   uint32_t width = wasm_mem_width(in->kind);
   uint64_t end;
   KitCgLabel ok = kit_cg_label_new(cg);
   uint64_t max_pages = m->memories[in->memidx].has_max
                            ? m->memories[in->memidx].max_pages
                            : m->memories[in->memidx].min_pages;
   if (in->offset64 > UINT64_MAX - (uint64_t)width) {
     wasm_cg_trap_bounds(cg, rt);
     return;
   }
   end = in->offset64 + (uint64_t)width;
   if (max_pages > UINT64_MAX / 65536u || end > max_pages * 65536u) {
     wasm_cg_trap_bounds(cg, rt);
     return;
   }
   (void)c;
   kit_cg_dup(cg);
   wasm_cg_push_instance_ptr(cg, rt, instance_local);
   kit_cg_deref(
       cg, (int64_t)(rt->memory_offset[in->memidx] + rt->memory_pages_offset));
   kit_cg_load(cg, wasm_cg_mem(c, b, WASM_VAL_I64));
   kit_cg_push_int(cg, 65536u, b.id[KIT_CG_BUILTIN_I64]);
   kit_cg_int_binop(cg, KIT_CG_INT_MUL, 0);
   kit_cg_push_int(cg, end, b.id[KIT_CG_BUILTIN_I64]);
   kit_cg_int_binop(cg, KIT_CG_INT_SUB, 0);
   if (!m->memories[in->memidx].is64) {
     kit_cg_swap(cg);
     kit_cg_zext(cg, b.id[KIT_CG_BUILTIN_I64]);
     kit_cg_swap(cg);
   }
   kit_cg_int_cmp(cg, KIT_CG_INT_LE_U);
   kit_cg_branch_true(cg, ok);
   wasm_cg_trap_bounds(cg, rt);
   kit_cg_label_place(cg, ok);
 }
 
 /* Compute the absolute address (data_ptr + addr_on_tos + offset) as a
  * pointer rvalue. Stack: [addr] -> [void*]. */
 static void wasm_cg_memory_addr_from_tos(KitCg* cg, KitCgBuiltinTypes b,
                                          const WasmCgRuntime* rt,
                                          const WasmModule* m,
                                          KitCgLocal instance_local,
                                          uint32_t memidx, uint64_t offset) {
   KitCgTypeId i64_ty = b.id[KIT_CG_BUILTIN_I64];
   /* TOS: [addr]. */
   if (!m->memories[memidx].is64) kit_cg_zext(cg, i64_ty);
   if (offset) {
     kit_cg_push_int(cg, offset, i64_ty);
     kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
   }
   /* TOS: [addr + offset]. */
   wasm_cg_push_memory_data_ptr(cg, rt, instance_local, memidx);
   kit_cg_ptr_to_int(cg, i64_ty);
   kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
   kit_cg_int_to_ptr(cg, rt->i8_ptr_ty);
 }
 
 static void wasm_cg_rotate(KitCompiler* c, KitCg* cg, KitCgBuiltinTypes b,
                            WasmValType vt, int right) {
   KitCgTypeId ty = wasm_cg_type(c, b, vt);
   KitCgMemAccess mem = wasm_cg_mem(c, b, vt);
   KitCgLocalAttrs attrs;
   KitCgLocal lhs, rhs;
   uint32_t mask = vt == WASM_VAL_I32 ? 31u : 63u;
   memset(&attrs, 0, sizeof attrs);
   attrs.flags = KIT_CG_LOCAL_COMPILER_TEMP;
   rhs = kit_cg_local(cg, ty, attrs);
   lhs = kit_cg_local(cg, ty, attrs);
   kit_cg_push_local(cg, rhs);
   kit_cg_swap(cg);
   kit_cg_store(cg, mem);
   kit_cg_push_local(cg, lhs);
   kit_cg_swap(cg);
   kit_cg_store(cg, mem);
 
   kit_cg_push_local(cg, lhs);
   kit_cg_load(cg, mem);
   kit_cg_push_local(cg, rhs);
   kit_cg_load(cg, mem);
   kit_cg_push_int(cg, mask, ty);
   kit_cg_int_binop(cg, KIT_CG_INT_AND, 0);
   kit_cg_int_binop(cg, right ? KIT_CG_INT_LSHR : KIT_CG_INT_SHL, 0);
 
   kit_cg_push_local(cg, lhs);
   kit_cg_load(cg, mem);
   kit_cg_push_int(cg, 0, ty);
   kit_cg_push_local(cg, rhs);
   kit_cg_load(cg, mem);
   kit_cg_int_binop(cg, KIT_CG_INT_SUB, 0);
   kit_cg_push_int(cg, mask, ty);
   kit_cg_int_binop(cg, KIT_CG_INT_AND, 0);
   kit_cg_int_binop(cg, right ? KIT_CG_INT_SHL : KIT_CG_INT_LSHR, 0);
   kit_cg_int_binop(cg, KIT_CG_INT_OR, 0);
 }
 
 static void wasm_cg_checked_divrem(KitCompiler* c, KitCg* cg,
                                    KitCgBuiltinTypes b, const WasmCgRuntime* rt,
                                    WasmValType vt, KitCgIntBinOp op) {
   KitCgTypeId ty = wasm_cg_type(c, b, vt);
   KitCgMemAccess mem = wasm_cg_mem(c, b, vt);
   KitCgLocalAttrs attrs;
   KitCgLocal lhs, rhs;
   KitCgLabel ok = kit_cg_label_new(cg);
   memset(&attrs, 0, sizeof attrs);
   attrs.flags = KIT_CG_LOCAL_COMPILER_TEMP;
   rhs = kit_cg_local(cg, ty, attrs);
   lhs = kit_cg_local(cg, ty, attrs);
   kit_cg_push_local(cg, rhs);
   kit_cg_swap(cg);
   kit_cg_store(cg, mem);
   kit_cg_push_local(cg, lhs);
   kit_cg_swap(cg);
   kit_cg_store(cg, mem);
   kit_cg_push_local(cg, rhs);
   kit_cg_load(cg, mem);
   kit_cg_push_int(cg, 0, ty);
   kit_cg_int_cmp(cg, KIT_CG_INT_NE);
   kit_cg_branch_true(cg, ok);
   wasm_cg_trap_division(cg, rt);
   kit_cg_label_place(cg, ok);
   if (op == KIT_CG_INT_SDIV) {
     KitCgLabel no_overflow = kit_cg_label_new(cg);
     uint64_t min_val = vt == WASM_VAL_I32 ? UINT64_C(0x80000000)
                                           : UINT64_C(0x8000000000000000);
     kit_cg_push_local(cg, lhs);
     kit_cg_load(cg, mem);
     kit_cg_push_int(cg, min_val, ty);
     kit_cg_int_cmp(cg, KIT_CG_INT_NE);
     kit_cg_branch_true(cg, no_overflow);
     kit_cg_push_local(cg, rhs);
     kit_cg_load(cg, mem);
     kit_cg_push_int(cg, UINT64_MAX, ty);
     kit_cg_int_cmp(cg, KIT_CG_INT_NE);
     kit_cg_branch_true(cg, no_overflow);
     wasm_cg_trap_division(cg, rt);
     kit_cg_label_place(cg, no_overflow);
   }
   kit_cg_push_local(cg, lhs);
   kit_cg_load(cg, mem);
   kit_cg_push_local(cg, rhs);
   kit_cg_load(cg, mem);
   kit_cg_int_binop(cg, op, 0);
 }
 
 static void wasm_cg_checked_trunc(KitCompiler* c, KitCg* cg,
                                   KitCgBuiltinTypes b, const WasmCgRuntime* rt,
                                   WasmValType src, WasmValType dst,
                                   int is_unsigned) {
   KitCgTypeId src_ty = wasm_cg_type(c, b, src);
   KitCgTypeId dst_ty = wasm_cg_type(c, b, dst);
   KitCgTypeId bit_ty =
       b.id[src == WASM_VAL_F32 ? KIT_CG_BUILTIN_I32 : KIT_CG_BUILTIN_I64];
   KitCgMemAccess src_mem = wasm_cg_mem(c, b, src);
   KitCgMemAccess bit_mem = wasm_cg_mem_type(bit_ty);
   KitCgLocalAttrs attrs;
   KitCgLocal value, bits, abs_bits;
   uint64_t abs_mask =
       src == WASM_VAL_F32 ? UINT64_C(0x7fffffff) : UINT64_C(0x7fffffffffffffff);
   uint64_t sign_mask =
       src == WASM_VAL_F32 ? UINT64_C(0x80000000) : UINT64_C(0x8000000000000000);
   uint64_t inf_bits =
       src == WASM_VAL_F32 ? UINT64_C(0x7f800000) : UINT64_C(0x7ff0000000000000);
   uint64_t limit_bits;
   memset(&attrs, 0, sizeof attrs);
   attrs.flags = KIT_CG_LOCAL_COMPILER_TEMP;
   value = kit_cg_local(cg, src_ty, attrs);
   bits = kit_cg_local(cg, bit_ty, attrs);
   abs_bits = kit_cg_local(cg, bit_ty, attrs);
   kit_cg_push_local(cg, value);
   kit_cg_swap(cg);
   kit_cg_store(cg, src_mem);
 
   kit_cg_push_local(cg, bits);
   kit_cg_push_local(cg, value);
   kit_cg_load(cg, src_mem);
   kit_cg_bitcast(cg, bit_ty);
   kit_cg_store(cg, bit_mem);
 
   kit_cg_push_local(cg, abs_bits);
   kit_cg_push_local(cg, bits);
   kit_cg_load(cg, bit_mem);
   kit_cg_push_int(cg, abs_mask, bit_ty);
   kit_cg_int_binop(cg, KIT_CG_INT_AND, 0);
   kit_cg_store(cg, bit_mem);
 
   {
     KitCgLabel finite = kit_cg_label_new(cg);
     kit_cg_push_local(cg, abs_bits);
     kit_cg_load(cg, bit_mem);
     kit_cg_push_int(cg, inf_bits, bit_ty);
     kit_cg_int_cmp(cg, KIT_CG_INT_LE_U);
     kit_cg_branch_true(cg, finite);
     wasm_cg_trap_invalid_conversion(cg, rt);
     kit_cg_label_place(cg, finite);
   }
 
   if (src == WASM_VAL_F32) {
     if (dst == WASM_VAL_I32)
       limit_bits = is_unsigned ? UINT64_C(0x4f800000) : UINT64_C(0x4f000000);
     else
       limit_bits = is_unsigned ? UINT64_C(0x5f800000) : UINT64_C(0x5f000000);
   } else {
     if (dst == WASM_VAL_I32)
       limit_bits = is_unsigned ? UINT64_C(0x41f0000000000000)
                                : UINT64_C(0x41e0000000000000);
     else
       limit_bits = is_unsigned ? UINT64_C(0x43f0000000000000)
                                : UINT64_C(0x43e0000000000000);
   }
 
   if (is_unsigned) {
     KitCgLabel nonnegative = kit_cg_label_new(cg);
     kit_cg_push_local(cg, abs_bits);
     kit_cg_load(cg, bit_mem);
     kit_cg_push_int(cg, 0, bit_ty);
     kit_cg_int_cmp(cg, KIT_CG_INT_EQ);
     kit_cg_branch_true(cg, nonnegative);
     kit_cg_push_local(cg, bits);
     kit_cg_load(cg, bit_mem);
     kit_cg_push_int(cg, sign_mask, bit_ty);
     kit_cg_int_cmp(cg, KIT_CG_INT_LT_U);
     kit_cg_branch_true(cg, nonnegative);
     wasm_cg_trap_invalid_conversion(cg, rt);
     kit_cg_label_place(cg, nonnegative);
 
     {
       KitCgLabel in_range = kit_cg_label_new(cg);
       kit_cg_push_local(cg, abs_bits);
       kit_cg_load(cg, bit_mem);
       kit_cg_push_int(cg, limit_bits, bit_ty);
       kit_cg_int_cmp(cg, KIT_CG_INT_LT_U);
       kit_cg_branch_true(cg, in_range);
       wasm_cg_trap_invalid_conversion(cg, rt);
       kit_cg_label_place(cg, in_range);
     }
   } else {
     KitCgLabel negative = kit_cg_label_new(cg);
     KitCgLabel in_range = kit_cg_label_new(cg);
     kit_cg_push_local(cg, bits);
     kit_cg_load(cg, bit_mem);
     kit_cg_push_int(cg, sign_mask, bit_ty);
     kit_cg_int_cmp(cg, KIT_CG_INT_GE_U);
     kit_cg_branch_true(cg, negative);
     kit_cg_push_local(cg, abs_bits);
     kit_cg_load(cg, bit_mem);
     kit_cg_push_int(cg, limit_bits, bit_ty);
     kit_cg_int_cmp(cg, KIT_CG_INT_LT_U);
     kit_cg_branch_true(cg, in_range);
     wasm_cg_trap_invalid_conversion(cg, rt);
     kit_cg_label_place(cg, negative);
     kit_cg_push_local(cg, abs_bits);
     kit_cg_load(cg, bit_mem);
     kit_cg_push_int(cg, limit_bits, bit_ty);
     kit_cg_int_cmp(cg, KIT_CG_INT_LE_U);
     kit_cg_branch_true(cg, in_range);
     wasm_cg_trap_invalid_conversion(cg, rt);
     kit_cg_label_place(cg, in_range);
   }
 
   kit_cg_push_local(cg, value);
   kit_cg_load(cg, src_mem);
   if (is_unsigned)
     kit_cg_float_to_uint(cg, dst_ty, KIT_CG_ROUND_TOWARD_ZERO);
   else
     kit_cg_float_to_sint(cg, dst_ty, KIT_CG_ROUND_TOWARD_ZERO);
 }
 
 static int wasm_fp_binop(uint8_t kind, KitCgFpBinOp* out) {
   switch (kind) {
     case WASM_INSN_F32_ADD:
     case WASM_INSN_F64_ADD:
       *out = KIT_CG_FP_ADD;
       return 1;
     case WASM_INSN_F32_SUB:
     case WASM_INSN_F64_SUB:
       *out = KIT_CG_FP_SUB;
       return 1;
     case WASM_INSN_F32_MUL:
     case WASM_INSN_F64_MUL:
       *out = KIT_CG_FP_MUL;
       return 1;
     case WASM_INSN_F32_DIV:
     case WASM_INSN_F64_DIV:
       *out = KIT_CG_FP_DIV;
       return 1;
     default:
       return 0;
   }
 }
 
 static int wasm_fp_cmp_op(uint8_t kind, KitCgFpCmpOp* out) {
   switch (kind) {
     case WASM_INSN_F32_EQ:
     case WASM_INSN_F64_EQ:
       *out = KIT_CG_FP_OEQ;
       return 1;
     case WASM_INSN_F32_NE:
     case WASM_INSN_F64_NE:
       /* wasm f.ne is the UNORDERED not-equal: true when a != b OR either
        * operand is NaN. Map to UNE, not ONE (ordered, false on NaN). */
       *out = KIT_CG_FP_UNE;
       return 1;
     case WASM_INSN_F32_LT:
     case WASM_INSN_F64_LT:
       *out = KIT_CG_FP_OLT;
       return 1;
     case WASM_INSN_F32_GT:
     case WASM_INSN_F64_GT:
       *out = KIT_CG_FP_OGT;
       return 1;
     case WASM_INSN_F32_LE:
     case WASM_INSN_F64_LE:
       *out = KIT_CG_FP_OLE;
       return 1;
     case WASM_INSN_F32_GE:
     case WASM_INSN_F64_GE:
       *out = KIT_CG_FP_OGE;
       return 1;
     default:
       return 0;
   }
 }
 
 static void wasm_cg_call_func(KitCompiler* c, KitCg* cg, KitCgBuiltinTypes b,
                               const WasmFunc* f, const WasmCgRuntime* rt,
                               KitCgSym sym, KitCgTypeId func_type,
                               KitCgLocal instance_local, uint32_t func_index,
                               int must_tail, KitSrcLoc loc, KitArena* arena) {
   KitCgLocalAttrs attrs;
   KitCgLocal* args =
       f->nparams ? kit_arena_array(arena, KitCgLocal, f->nparams) : NULL;
   KitCgLocal callee = KIT_CG_LOCAL_NONE;
   (void)loc;
   memset(&attrs, 0, sizeof attrs);
   attrs.flags = KIT_CG_LOCAL_COMPILER_TEMP;
   for (uint32_t p = 0; p < f->nparams; ++p) {
     uint32_t param = f->nparams - 1u - p;
     args[param] = kit_cg_local(cg, wasm_cg_type(c, b, f->params[param]), attrs);
     kit_cg_push_local(cg, args[param]);
     kit_cg_swap(cg);
     kit_cg_store(cg, wasm_cg_mem(c, b, f->params[param]));
   }
   if (f->is_import) {
     KitCgLabel ok = kit_cg_label_new(cg);
     callee = kit_cg_local(cg, rt->void_ptr_ty, attrs);
     kit_cg_push_local(cg, callee);
     wasm_cg_push_import_func_ptr(cg, rt, instance_local, func_index);
     kit_cg_store(cg, wasm_cg_mem_type(rt->void_ptr_ty));
     kit_cg_push_local(cg, callee);
     kit_cg_load(cg, wasm_cg_mem_type(rt->void_ptr_ty));
     kit_cg_push_null(cg, rt->void_ptr_ty);
     kit_cg_int_cmp(cg, KIT_CG_INT_NE);
     kit_cg_branch_true(cg, ok);
     wasm_cg_trap_table(cg, rt);
     kit_cg_label_place(cg, ok);
     kit_cg_push_local(cg, callee);
     kit_cg_load(cg, wasm_cg_mem_type(rt->void_ptr_ty));
     kit_cg_bitcast(cg, kit_cg_type_ptr(c, func_type, 0));
   }
   kit_cg_push_local(cg, instance_local);
   kit_cg_load(cg, wasm_cg_mem_type(rt->instance_ptr_ty));
   for (uint32_t p = 0; p < f->nparams; ++p) {
     kit_cg_push_local(cg, args[p]);
     kit_cg_load(cg, wasm_cg_mem(c, b, f->params[p]));
   }
   if (f->is_import)
     kit_cg_call(cg, f->nparams + 1u, func_type,
                 (KitCgCallAttrs){.tail = must_tail ? KIT_CG_TAIL_MUST
                                                    : KIT_CG_TAIL_DEFAULT});
   else
     kit_cg_call_symbol(
         cg, sym, f->nparams + 1u,
         (KitCgCallAttrs){.tail = must_tail ? KIT_CG_TAIL_MUST
                                            : KIT_CG_TAIL_DEFAULT});
   if (must_tail) {
     if (f->nresults) wasm_cg_push_zero(c, cg, b, f->results[0]);
     kit_cg_unreachable(cg);
   }
 }
 
 /* Intern a NUL-terminated string as readonly const data and return its
  * symbol. Each call mints a fresh symbol; the caller is responsible for
  * deduplication if that matters. */
 static KitCgSym wasm_cg_intern_cstr(KitCg* cg, KitCgBuiltinTypes b,
                                     const char* s) {
   size_t len = 0;
   while (s && s[len]) ++len;
   return kit_cg_const_data(cg, (const uint8_t*)(s ? s : ""), len + 1u, 1,
                            b.id[KIT_CG_BUILTIN_I8]);
 }
 
 /* Emit the host-import metadata symbols read by kit_wasm_bind_host_imports.
  * Layout matches runtime_abi.h. When the module has no imports, only the
  * count (=0) is emitted; descriptor/type arrays are omitted so the binder can
  * no-op early via kit_jit_lookup returning NULL. */
 static void wasm_cg_emit_host_import_metadata(KitCompiler* c, KitCg* cg,
                                               KitCgBuiltinTypes b,
                                               const WasmModule* m,
                                               const WasmCgRuntime* rt,
                                               KitArena* arena) {
   KitCgTypeId u32_ty = b.id[KIT_CG_BUILTIN_I32];
   KitCgTypeId u8_ty = b.id[KIT_CG_BUILTIN_I8];
   KitCgTypeId ptr_ty = rt->void_ptr_ty;
   uint32_t ptr_size = (uint32_t)kit_cg_type_size(c, ptr_ty);
   uint32_t ptr_align = ptr_size ? ptr_size : 8u;
   uint32_t nimports = 0;
   uint32_t nfunc_imports = 0;
   uint32_t nmemory_imports = 0;
   uint32_t ntable_imports = 0;
   uint32_t nglobal_imports = 0;
   uint32_t desc_size = 2u * ptr_size + 16u;
   uint32_t type_desc_size =
       2u * ptr_size + 8u + (ptr_align > 4u ? 2u * (ptr_align - 4u) : 0u);
   KitCgSym nimports_sym, imports_sym;
   KitCgDecl decl;
   KitCgDataDefAttrs data_attrs;
   uint32_t i;
 
   for (i = 0; i < m->nfuncs; ++i) {
     if (!m->funcs[i].is_import) continue;
     nimports++;
     nfunc_imports++;
   }
   for (i = 0; i < m->nmemories; ++i) {
     if (!m->memories[i].is_import) continue;
     nimports++;
     nmemory_imports++;
   }
   for (i = 0; i < m->ntables; ++i) {
     if (!m->tables[i].is_import) continue;
     nimports++;
     ntable_imports++;
   }
   for (i = 0; i < m->nglobals; ++i) {
     if (!m->globals[i].is_import) continue;
     nimports++;
     nglobal_imports++;
   }
 
   /* Always emit the count symbol so the binder can read it unconditionally.
    * If zero, the descriptor/type arrays are omitted. */
   memset(&decl, 0, sizeof decl);
   decl.kind = KIT_CG_DECL_OBJECT;
   decl.linkage_name = kit_cg_c_linkage_name(
       c, kit_sym_intern(c, KIT_SLICE_LIT("__kit_wasm_nimports")));
   decl.display_name = decl.linkage_name;
   decl.type = u32_ty;
   decl.sym.bind = KIT_SB_GLOBAL;
   decl.as.object.flags = KIT_CG_OBJ_READONLY;
   nimports_sym = kit_cg_decl(cg, decl);
   if (!nimports_sym)
     wasm_error(c, wasm_loc(0, 0),
                "wasm: failed to declare __kit_wasm_nimports");
 
   memset(&data_attrs, 0, sizeof data_attrs);
   data_attrs.flags = KIT_CG_DATADEF_READONLY;
   data_attrs.align = 4;
   kit_cg_data_begin(cg, nimports_sym, data_attrs);
   kit_cg_data_int(cg, (uint64_t)nimports, u32_ty);
   kit_cg_data_end(cg);
 
   if (!nimports) return;
 
   /* Walk imports once, build sym handles for module/field strings, capture
    * each import's kind-specific descriptor index and slot offset. Function
    * import signatures are deduplicated into __kit_wasm_types; the emitted
    * __kit_wasm_nfunc_import_types count lets the binder validate the compact
    * descriptor indexes before reading that table. */
   typedef struct WasmImportEmit {
     KitCgSym module_sym;
     KitCgSym field_sym;
     uint32_t kind;
     uint32_t desc_index;
     uint32_t slot_offset;
   } WasmImportEmit;
   WasmImportEmit* descs = kit_arena_zarray(arena, WasmImportEmit, nimports);
   KitWasmMemoryImportDesc* memory_descs =
       nmemory_imports
           ? kit_arena_zarray(arena, KitWasmMemoryImportDesc, nmemory_imports)
           : NULL;
   KitWasmTableImportDesc* table_descs =
       ntable_imports
           ? kit_arena_zarray(arena, KitWasmTableImportDesc, ntable_imports)
           : NULL;
   KitWasmGlobalImportDesc* global_descs =
       nglobal_imports
           ? kit_arena_zarray(arena, KitWasmGlobalImportDesc, nglobal_imports)
           : NULL;
   uint32_t* type_remap =
       nfunc_imports
           ? kit_arena_zarray(arena, uint32_t, m->ntypes ? m->ntypes : 1u)
           : NULL;
   uint32_t* local_to_module =
       nfunc_imports ? kit_arena_zarray(arena, uint32_t, nfunc_imports) : NULL;
   uint32_t ntypes = 0;
   uint32_t d = 0;
   uint32_t mem_d = 0;
   uint32_t table_d = 0;
   uint32_t global_d = 0;
   for (i = 0; type_remap && i < m->ntypes; ++i) type_remap[i] = UINT32_MAX;
   for (i = 0; i < m->nfuncs; ++i) {
     const WasmFunc* f = &m->funcs[i];
     uint64_t slot_off;
     uint32_t module_typeidx;
     if (!f->is_import) continue;
     module_typeidx = f->typeidx;
     if (module_typeidx >= m->ntypes) {
       wasm_error(c, f->loc, "wasm: import typeidx out of range");
       return;
     }
     if (type_remap[module_typeidx] == UINT32_MAX) {
       type_remap[module_typeidx] = ntypes;
       local_to_module[ntypes] = module_typeidx;
       ntypes++;
     }
     descs[d].module_sym =
         wasm_cg_intern_cstr(cg, b, f->import_module ? f->import_module : "");
     descs[d].field_sym =
         wasm_cg_intern_cstr(cg, b, f->import_name ? f->import_name : "");
     descs[d].kind = KIT_WASM_IMPORT_FUNC;
     descs[d].desc_index = type_remap[module_typeidx];
     slot_off = rt->func_import_offset[i];
     if (slot_off > UINT32_MAX) {
       wasm_error(c, f->loc, "wasm: import slot offset exceeds u32");
       return;
     }
     descs[d].slot_offset = (uint32_t)slot_off;
     d++;
   }
   for (i = 0; i < m->nmemories; ++i) {
     const WasmMemory* mem = &m->memories[i];
     uint64_t max_pages;
     uint32_t flags;
     if (!mem->is_import) continue;
     max_pages = mem->has_max ? mem->max_pages : mem->min_pages;
     flags = (mem->shared ? KIT_WASM_MEMORY_SHARED : 0u) |
             (mem->is64 ? KIT_WASM_MEMORY_64 : 0u);
     descs[d].module_sym = wasm_cg_intern_cstr(
         cg, b, mem->import_module ? mem->import_module : "");
     descs[d].field_sym =
         wasm_cg_intern_cstr(cg, b, mem->import_name ? mem->import_name : "");
     descs[d].kind = KIT_WASM_IMPORT_MEMORY;
     descs[d].desc_index = mem_d;
     if (rt->memory_offset[i] > UINT32_MAX) {
       wasm_error(c, wasm_loc(0, 0), "wasm: memory import slot exceeds u32");
       return;
     }
     descs[d].slot_offset = (uint32_t)rt->memory_offset[i];
     memory_descs[mem_d].min_pages = mem->min_pages;
     memory_descs[mem_d].max_pages = max_pages;
     memory_descs[mem_d].flags = flags;
     memory_descs[mem_d].has_max = mem->has_max ? 1u : 0u;
     mem_d++;
     d++;
   }
   for (i = 0; i < m->ntables; ++i) {
     const WasmTable* t = &m->tables[i];
     uint32_t max;
     if (!t->is_import) continue;
     max = t->has_max ? t->max : t->min;
     descs[d].module_sym =
         wasm_cg_intern_cstr(cg, b, t->import_module ? t->import_module : "");
     descs[d].field_sym =
         wasm_cg_intern_cstr(cg, b, t->import_name ? t->import_name : "");
     descs[d].kind = KIT_WASM_IMPORT_TABLE;
     descs[d].desc_index = table_d;
     if (rt->table_offset[i] > UINT32_MAX) {
       wasm_error(c, wasm_loc(0, 0), "wasm: table import slot exceeds u32");
       return;
     }
     descs[d].slot_offset = (uint32_t)rt->table_offset[i];
     table_descs[table_d].elem_type = (uint32_t)t->elem_type;
     table_descs[table_d].min = t->min;
     table_descs[table_d].max = max;
     table_descs[table_d].has_max = t->has_max ? 1u : 0u;
     table_d++;
     d++;
   }
   for (i = 0; i < m->nglobals; ++i) {
     const WasmGlobal* g = &m->globals[i];
     if (!g->is_import) continue;
     descs[d].module_sym =
         wasm_cg_intern_cstr(cg, b, g->import_module ? g->import_module : "");
     descs[d].field_sym =
         wasm_cg_intern_cstr(cg, b, g->import_name ? g->import_name : "");
     descs[d].kind = KIT_WASM_IMPORT_GLOBAL;
     descs[d].desc_index = global_d;
     if (rt->global_offset[i] > UINT32_MAX) {
       wasm_error(c, g->loc, "wasm: global import slot exceeds u32");
       return;
     }
     descs[d].slot_offset = (uint32_t)rt->global_offset[i];
     global_descs[global_d].type = (uint32_t)g->type;
     global_descs[global_d].mutable_ = g->mutable_ ? 1u : 0u;
     global_d++;
     d++;
   }
 
   /* __kit_wasm_types: array of KitWasmTypeDesc.
    *   { const u8* params; u32 nparams; const u8* results; u32 nresults; }
    * Each distinct imported function signature gets one entry. Param/result
    * bytes are emitted as interned const u8 arrays using raw WasmValType
    * bytes. */
   if (ntypes) {
     KitCgSym nfunc_import_types_sym;
     KitCgSym types_sym;
     KitCgSym* param_syms = kit_arena_zarray(arena, KitCgSym, ntypes);
     KitCgSym* result_syms = kit_arena_zarray(arena, KitCgSym, ntypes);
     KitCgTypeId types_array_ty =
         kit_cg_type_array(c, u8_ty, (uint64_t)type_desc_size * ntypes);
     for (uint32_t k = 0; k < ntypes; ++k) {
       const WasmFuncType* t = &m->types[local_to_module[k]];
       uint8_t* pbuf =
           t->nparams ? kit_arena_array(arena, uint8_t, t->nparams) : NULL;
       uint8_t* rbuf =
           t->nresults ? kit_arena_array(arena, uint8_t, t->nresults) : NULL;
       for (uint32_t p = 0; p < t->nparams; ++p) pbuf[p] = (uint8_t)t->params[p];
       for (uint32_t r = 0; r < t->nresults; ++r)
         rbuf[r] = (uint8_t)t->results[r];
       param_syms[k] = t->nparams
                           ? kit_cg_const_data(cg, pbuf, t->nparams, 1, u8_ty)
                           : (KitCgSym)0;
       result_syms[k] = t->nresults
                            ? kit_cg_const_data(cg, rbuf, t->nresults, 1, u8_ty)
                            : (KitCgSym)0;
     }
     memset(&decl, 0, sizeof decl);
     decl.kind = KIT_CG_DECL_OBJECT;
     decl.linkage_name = kit_cg_c_linkage_name(
         c, kit_sym_intern(c, KIT_SLICE_LIT("__kit_wasm_nfunc_import_types")));
     decl.display_name = decl.linkage_name;
     decl.type = u32_ty;
     decl.sym.bind = KIT_SB_GLOBAL;
     decl.as.object.flags = KIT_CG_OBJ_READONLY;
     nfunc_import_types_sym = kit_cg_decl(cg, decl);
     if (!nfunc_import_types_sym)
       wasm_error(c, wasm_loc(0, 0),
                  "wasm: failed to declare __kit_wasm_nfunc_import_types");
     memset(&data_attrs, 0, sizeof data_attrs);
     data_attrs.flags = KIT_CG_DATADEF_READONLY;
     data_attrs.align = 4;
     kit_cg_data_begin(cg, nfunc_import_types_sym, data_attrs);
     kit_cg_data_int(cg, (uint64_t)ntypes, u32_ty);
     kit_cg_data_end(cg);
 
     memset(&decl, 0, sizeof decl);
     decl.kind = KIT_CG_DECL_OBJECT;
     decl.linkage_name = kit_cg_c_linkage_name(
         c, kit_sym_intern(c, KIT_SLICE_LIT("__kit_wasm_types")));
     decl.display_name = decl.linkage_name;
     decl.type = types_array_ty;
     decl.sym.bind = KIT_SB_GLOBAL;
     decl.as.object.flags = KIT_CG_OBJ_READONLY;
     types_sym = kit_cg_decl(cg, decl);
     if (!types_sym)
       wasm_error(c, wasm_loc(0, 0), "wasm: failed to declare __kit_wasm_types");
     memset(&data_attrs, 0, sizeof data_attrs);
     data_attrs.flags = KIT_CG_DATADEF_READONLY;
     data_attrs.align = ptr_align;
     kit_cg_data_begin(cg, types_sym, data_attrs);
     for (uint32_t k = 0; k < ntypes; ++k) {
       const WasmFuncType* t = &m->types[local_to_module[k]];
       kit_cg_data_align(cg, ptr_align);
       if (param_syms[k])
         kit_cg_data_addr(cg, param_syms[k], 0, ptr_size, 0);
       else
         kit_cg_data_pad(cg, ptr_size, 0);
       kit_cg_data_int(cg, (uint64_t)t->nparams, u32_ty);
       /* Pad up to ptr alignment before next pointer field. */
       if (ptr_align > 4) kit_cg_data_align(cg, ptr_align);
       if (result_syms[k])
         kit_cg_data_addr(cg, result_syms[k], 0, ptr_size, 0);
       else
         kit_cg_data_pad(cg, ptr_size, 0);
       kit_cg_data_int(cg, (uint64_t)t->nresults, u32_ty);
       if (ptr_align > 4) kit_cg_data_align(cg, ptr_align);
     }
     kit_cg_data_end(cg);
   }
 
   if (nmemory_imports) {
     KitCgSym memory_types_sym;
     KitCgTypeId memory_array_ty = kit_cg_type_array(
         c, u8_ty, (uint64_t)sizeof(KitWasmMemoryImportDesc) * nmemory_imports);
     memset(&decl, 0, sizeof decl);
     decl.kind = KIT_CG_DECL_OBJECT;
     decl.linkage_name = kit_cg_c_linkage_name(
         c, kit_sym_intern(c, KIT_SLICE_LIT("__kit_wasm_memory_import_types")));
     decl.display_name = decl.linkage_name;
     decl.type = memory_array_ty;
     decl.sym.bind = KIT_SB_GLOBAL;
     decl.as.object.flags = KIT_CG_OBJ_READONLY;
     memory_types_sym = kit_cg_decl(cg, decl);
     if (!memory_types_sym)
       wasm_error(c, wasm_loc(0, 0),
                  "wasm: failed to declare __kit_wasm_memory_import_types");
     memset(&data_attrs, 0, sizeof data_attrs);
     data_attrs.flags = KIT_CG_DATADEF_READONLY;
     data_attrs.align = 8;
     kit_cg_data_begin(cg, memory_types_sym, data_attrs);
     for (uint32_t k = 0; k < nmemory_imports; ++k) {
       kit_cg_data_align(cg, 8);
       kit_cg_data_int(cg, memory_descs[k].min_pages, b.id[KIT_CG_BUILTIN_I64]);
       kit_cg_data_int(cg, memory_descs[k].max_pages, b.id[KIT_CG_BUILTIN_I64]);
       kit_cg_data_int(cg, memory_descs[k].flags, u32_ty);
       kit_cg_data_int(cg, memory_descs[k].has_max, u32_ty);
     }
     kit_cg_data_end(cg);
   }
 
   if (ntable_imports) {
     KitCgSym table_types_sym;
     KitCgTypeId table_array_ty = kit_cg_type_array(
         c, u8_ty, (uint64_t)sizeof(KitWasmTableImportDesc) * ntable_imports);
     memset(&decl, 0, sizeof decl);
     decl.kind = KIT_CG_DECL_OBJECT;
     decl.linkage_name = kit_cg_c_linkage_name(
         c, kit_sym_intern(c, KIT_SLICE_LIT("__kit_wasm_table_import_types")));
     decl.display_name = decl.linkage_name;
     decl.type = table_array_ty;
     decl.sym.bind = KIT_SB_GLOBAL;
     decl.as.object.flags = KIT_CG_OBJ_READONLY;
     table_types_sym = kit_cg_decl(cg, decl);
     if (!table_types_sym)
       wasm_error(c, wasm_loc(0, 0),
                  "wasm: failed to declare __kit_wasm_table_import_types");
     memset(&data_attrs, 0, sizeof data_attrs);
     data_attrs.flags = KIT_CG_DATADEF_READONLY;
     data_attrs.align = 4;
     kit_cg_data_begin(cg, table_types_sym, data_attrs);
     for (uint32_t k = 0; k < ntable_imports; ++k) {
       kit_cg_data_int(cg, table_descs[k].elem_type, u32_ty);
       kit_cg_data_int(cg, table_descs[k].min, u32_ty);
       kit_cg_data_int(cg, table_descs[k].max, u32_ty);
       kit_cg_data_int(cg, table_descs[k].has_max, u32_ty);
     }
     kit_cg_data_end(cg);
   }
 
   if (nglobal_imports) {
     KitCgSym global_types_sym;
     KitCgTypeId global_array_ty = kit_cg_type_array(
         c, u8_ty, (uint64_t)sizeof(KitWasmGlobalImportDesc) * nglobal_imports);
     memset(&decl, 0, sizeof decl);
     decl.kind = KIT_CG_DECL_OBJECT;
     decl.linkage_name = kit_cg_c_linkage_name(
         c, kit_sym_intern(c, KIT_SLICE_LIT("__kit_wasm_global_import_types")));
     decl.display_name = decl.linkage_name;
     decl.type = global_array_ty;
     decl.sym.bind = KIT_SB_GLOBAL;
     decl.as.object.flags = KIT_CG_OBJ_READONLY;
     global_types_sym = kit_cg_decl(cg, decl);
     if (!global_types_sym)
       wasm_error(c, wasm_loc(0, 0),
                  "wasm: failed to declare __kit_wasm_global_import_types");
     memset(&data_attrs, 0, sizeof data_attrs);
     data_attrs.flags = KIT_CG_DATADEF_READONLY;
     data_attrs.align = 4;
     kit_cg_data_begin(cg, global_types_sym, data_attrs);
     for (uint32_t k = 0; k < nglobal_imports; ++k) {
       kit_cg_data_int(cg, global_descs[k].type, u32_ty);
       kit_cg_data_int(cg, global_descs[k].mutable_, u32_ty);
     }
     kit_cg_data_end(cg);
   }
 
   /* __kit_wasm_imports: array of KitWasmImportDesc. */
   {
     KitCgTypeId imports_array_ty =
         kit_cg_type_array(c, u8_ty, (uint64_t)desc_size * nimports);
     memset(&decl, 0, sizeof decl);
     decl.kind = KIT_CG_DECL_OBJECT;
     decl.linkage_name = kit_cg_c_linkage_name(
         c, kit_sym_intern(c, KIT_SLICE_LIT("__kit_wasm_imports")));
     decl.display_name = decl.linkage_name;
     decl.type = imports_array_ty;
     decl.sym.bind = KIT_SB_GLOBAL;
     decl.as.object.flags = KIT_CG_OBJ_READONLY;
     imports_sym = kit_cg_decl(cg, decl);
     if (!imports_sym)
       wasm_error(c, wasm_loc(0, 0),
                  "wasm: failed to declare __kit_wasm_imports");
     memset(&data_attrs, 0, sizeof data_attrs);
     data_attrs.flags = KIT_CG_DATADEF_READONLY;
     data_attrs.align = ptr_align;
     kit_cg_data_begin(cg, imports_sym, data_attrs);
     for (uint32_t k = 0; k < nimports; ++k) {
       kit_cg_data_align(cg, ptr_align);
       kit_cg_data_addr(cg, descs[k].module_sym, 0, ptr_size, 0);
       kit_cg_data_addr(cg, descs[k].field_sym, 0, ptr_size, 0);
       kit_cg_data_int(cg, (uint64_t)descs[k].kind, u32_ty);
       kit_cg_data_int(cg, (uint64_t)descs[k].desc_index, u32_ty);
       kit_cg_data_int(cg, (uint64_t)descs[k].slot_offset, u32_ty);
       kit_cg_data_int(cg, 0, u32_ty);
       if (ptr_align > 4) kit_cg_data_align(cg, ptr_align);
     }
     kit_cg_data_end(cg);
   }
 }
 
 /* Emit the runtime layout symbols read by kit_wasm_get_runtime_layout:
  *
  *   __kit_wasm_instance_size    uint64
  *   __kit_wasm_nmemories        uint32
  *   __kit_wasm_memory_layouts   KitWasmMemoryLayout[]
  *
  * The memory-layout array intentionally contains only fixed-width integers, so
  * the host reader can consume it without target pointer-size ambiguity. */
 static void wasm_cg_emit_runtime_layout_metadata(KitCompiler* c, KitCg* cg,
                                                  KitCgBuiltinTypes b,
                                                  const WasmModule* m,
                                                  const WasmCgRuntime* rt) {
   KitCgTypeId u8_ty = b.id[KIT_CG_BUILTIN_I8];
   KitCgTypeId u32_ty = b.id[KIT_CG_BUILTIN_I32];
   KitCgTypeId u64_ty = b.id[KIT_CG_BUILTIN_I64];
   KitCgDecl decl;
   KitCgDataDefAttrs data_attrs;
   KitCgSym instance_size_sym;
   KitCgSym nmemories_sym;
   uint64_t instance_size = kit_cg_type_size(c, rt->instance_ty);
 
   memset(&decl, 0, sizeof decl);
   decl.kind = KIT_CG_DECL_OBJECT;
   decl.linkage_name = kit_cg_c_linkage_name(
       c, kit_sym_intern(c, KIT_SLICE_LIT("__kit_wasm_instance_size")));
   decl.display_name = decl.linkage_name;
   decl.type = u64_ty;
   decl.sym.bind = KIT_SB_GLOBAL;
   decl.as.object.flags = KIT_CG_OBJ_READONLY;
   instance_size_sym = kit_cg_decl(cg, decl);
   if (!instance_size_sym)
     wasm_error(c, wasm_loc(0, 0),
                "wasm: failed to declare __kit_wasm_instance_size");
   memset(&data_attrs, 0, sizeof data_attrs);
   data_attrs.flags = KIT_CG_DATADEF_READONLY;
   data_attrs.align = 8;
   kit_cg_data_begin(cg, instance_size_sym, data_attrs);
   kit_cg_data_int(cg, instance_size, u64_ty);
   kit_cg_data_end(cg);
 
   memset(&decl, 0, sizeof decl);
   decl.kind = KIT_CG_DECL_OBJECT;
   decl.linkage_name = kit_cg_c_linkage_name(
       c, kit_sym_intern(c, KIT_SLICE_LIT("__kit_wasm_nmemories")));
   decl.display_name = decl.linkage_name;
   decl.type = u32_ty;
   decl.sym.bind = KIT_SB_GLOBAL;
   decl.as.object.flags = KIT_CG_OBJ_READONLY;
   nmemories_sym = kit_cg_decl(cg, decl);
   if (!nmemories_sym)
     wasm_error(c, wasm_loc(0, 0),
                "wasm: failed to declare __kit_wasm_nmemories");
   memset(&data_attrs, 0, sizeof data_attrs);
   data_attrs.flags = KIT_CG_DATADEF_READONLY;
   data_attrs.align = 4;
   kit_cg_data_begin(cg, nmemories_sym, data_attrs);
   kit_cg_data_int(cg, (uint64_t)m->nmemories, u32_ty);
   kit_cg_data_end(cg);
 
   if (m->nmemories) {
     const uint64_t layout_size = sizeof(KitWasmMemoryLayout);
     KitCgTypeId layouts_array_ty =
         kit_cg_type_array(c, u8_ty, layout_size * m->nmemories);
     KitCgSym layouts_sym;
     memset(&decl, 0, sizeof decl);
     decl.kind = KIT_CG_DECL_OBJECT;
     decl.linkage_name = kit_cg_c_linkage_name(
         c, kit_sym_intern(c, KIT_SLICE_LIT("__kit_wasm_memory_layouts")));
     decl.display_name = decl.linkage_name;
     decl.type = layouts_array_ty;
     decl.sym.bind = KIT_SB_GLOBAL;
     decl.as.object.flags = KIT_CG_OBJ_READONLY;
     layouts_sym = kit_cg_decl(cg, decl);
     if (!layouts_sym)
       wasm_error(c, wasm_loc(0, 0),
                  "wasm: failed to declare __kit_wasm_memory_layouts");
     memset(&data_attrs, 0, sizeof data_attrs);
     data_attrs.flags = KIT_CG_DATADEF_READONLY;
     data_attrs.align = 8;
     kit_cg_data_begin(cg, layouts_sym, data_attrs);
     for (uint32_t i = 0; i < m->nmemories; ++i) {
       const WasmMemory* mem = &m->memories[i];
       uint64_t max_pages = mem->has_max ? mem->max_pages : mem->min_pages;
       uint32_t flags = (mem->shared ? KIT_WASM_MEMORY_SHARED : 0u) |
                        (mem->is64 ? KIT_WASM_MEMORY_64 : 0u);
       kit_cg_data_align(cg, 8);
       kit_cg_data_int(cg, rt->memory_offset[i], u64_ty);
       kit_cg_data_int(cg, mem->min_pages, u64_ty);
       kit_cg_data_int(cg, max_pages, u64_ty);
       kit_cg_data_int(cg, flags, u32_ty);
       kit_cg_data_int(cg, 0, u32_ty);
     }
     kit_cg_data_end(cg);
   }
 }
 
 /* Bounds-check that addr + n <= size. addr/n/size are all treated as i64
  * (caller zero-extends i32 inputs first). Traps on overflow or out-of-range
  * via the rt bounds trap; falls through on success. */
 static void wasm_cg_bulk_bounds_check(KitCg* cg, KitCgBuiltinTypes b,
                                       const WasmCgRuntime* rt,
                                       KitCgLocal addr_local, KitCgLocal n_local,
                                       KitCgLocal size_local) {
   KitCgMemAccess i64_mem = wasm_cg_mem_type(b.id[KIT_CG_BUILTIN_I64]);
   KitCgLabel ok = kit_cg_label_new(cg);
   /* size - addr >= n  (unsigned), equivalently addr + n <= size with no
    * overflow. First check size >= addr. */
   KitCgLabel addr_ok = kit_cg_label_new(cg);
   kit_cg_push_local(cg, size_local);
   kit_cg_load(cg, i64_mem);
   kit_cg_push_local(cg, addr_local);
   kit_cg_load(cg, i64_mem);
   kit_cg_int_cmp(cg, KIT_CG_INT_GE_U);
   kit_cg_branch_true(cg, addr_ok);
   wasm_cg_trap_bounds(cg, rt);
   kit_cg_label_place(cg, addr_ok);
   kit_cg_push_local(cg, size_local);
   kit_cg_load(cg, i64_mem);
   kit_cg_push_local(cg, addr_local);
   kit_cg_load(cg, i64_mem);
   kit_cg_int_binop(cg, KIT_CG_INT_SUB, 0);
   kit_cg_push_local(cg, n_local);
   kit_cg_load(cg, i64_mem);
   kit_cg_int_cmp(cg, KIT_CG_INT_GE_U);
   kit_cg_branch_true(cg, ok);
   wasm_cg_trap_bounds(cg, rt);
   kit_cg_label_place(cg, ok);
 }
 
 /* Emit a byte-wise copy loop: copies n_local bytes from src_base+src_addr
  * to dst_base+dst_addr. Handles overlap by choosing direction based on
  * pointer comparison. Locals are pre-stored i64 values; src_base and
  * dst_base are i8* values pre-stored in locals. */
 static void wasm_cg_emit_byte_copy_loop(
     KitCg* cg, KitCgBuiltinTypes b, const WasmCgRuntime* rt,
     KitCgLocal dst_base_local, KitCgLocal src_base_local,
     KitCgLocal dst_addr_local, KitCgLocal src_addr_local, KitCgLocal n_local) {
   KitCgMemAccess i64_mem = wasm_cg_mem_type(b.id[KIT_CG_BUILTIN_I64]);
   KitCgMemAccess i8_mem = wasm_cg_mem_type(b.id[KIT_CG_BUILTIN_I8]);
   KitCgMemAccess ptr_mem = wasm_cg_mem_type(rt->i8_ptr_ty);
   KitCgLocalAttrs attrs;
   KitCgLocal idx, dir;
   KitCgLabel done = kit_cg_label_new(cg);
   KitCgLabel loop_start = kit_cg_label_new(cg);
   KitCgLabel forward = kit_cg_label_new(cg);
   KitCgLabel after_dir = kit_cg_label_new(cg);
   memset(&attrs, 0, sizeof attrs);
   attrs.flags = KIT_CG_LOCAL_COMPILER_TEMP;
   idx = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
   dir = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I32], attrs);
   (void)rt;
   (void)ptr_mem;
   /* If n == 0, nothing to do. */
   kit_cg_push_local(cg, n_local);
   kit_cg_load(cg, i64_mem);
   kit_cg_push_int(cg, 0, b.id[KIT_CG_BUILTIN_I64]);
   kit_cg_int_cmp(cg, KIT_CG_INT_EQ);
   kit_cg_branch_true(cg, done);
 
   /* Choose direction: forward if dst_addr <= src_addr OR if base pointers
    * differ. Same-base + dst > src needs backward copy to handle overlap.
    * We approximate with: dir = (dst_base == src_base && dst_addr > src_addr)
    *                            ? backward : forward. */
   kit_cg_push_local(cg, dir);
   kit_cg_push_local(cg, dst_base_local);
   kit_cg_load(cg, ptr_mem);
   kit_cg_push_local(cg, src_base_local);
   kit_cg_load(cg, ptr_mem);
   kit_cg_int_cmp(cg, KIT_CG_INT_EQ);
   kit_cg_store(cg, wasm_cg_mem_type(b.id[KIT_CG_BUILTIN_I32]));
   kit_cg_push_local(cg, dir);
   kit_cg_load(cg, wasm_cg_mem_type(b.id[KIT_CG_BUILTIN_I32]));
   kit_cg_branch_false(cg, forward);
   kit_cg_push_local(cg, dst_addr_local);
   kit_cg_load(cg, i64_mem);
   kit_cg_push_local(cg, src_addr_local);
   kit_cg_load(cg, i64_mem);
   kit_cg_int_cmp(cg, KIT_CG_INT_GT_U);
   kit_cg_branch_false(cg, forward);
   /* Backward loop: idx = n - 1, while idx >= 0 then idx-- (stop at 0). */
   kit_cg_push_local(cg, idx);
   kit_cg_push_local(cg, n_local);
   kit_cg_load(cg, i64_mem);
   kit_cg_store(cg, i64_mem);
   {
     KitCgLabel back_loop = kit_cg_label_new(cg);
     kit_cg_label_place(cg, back_loop);
     kit_cg_push_local(cg, idx);
     kit_cg_load(cg, i64_mem);
     kit_cg_push_int(cg, 0, b.id[KIT_CG_BUILTIN_I64]);
     kit_cg_int_cmp(cg, KIT_CG_INT_EQ);
     kit_cg_branch_true(cg, done);
     /* idx-- */
     kit_cg_push_local(cg, idx);
     kit_cg_push_local(cg, idx);
     kit_cg_load(cg, i64_mem);
     kit_cg_push_int(cg, 1, b.id[KIT_CG_BUILTIN_I64]);
     kit_cg_int_binop(cg, KIT_CG_INT_SUB, 0);
     kit_cg_store(cg, i64_mem);
     /* dst_base[dst_addr + idx] = src_base[src_addr + idx] */
     kit_cg_push_local(cg, dst_base_local);
     kit_cg_load(cg, ptr_mem);
     kit_cg_push_local(cg, dst_addr_local);
     kit_cg_load(cg, i64_mem);
     kit_cg_push_local(cg, idx);
     kit_cg_load(cg, i64_mem);
     kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
     kit_cg_elem(cg, 0);
     kit_cg_push_local(cg, src_base_local);
     kit_cg_load(cg, ptr_mem);
     kit_cg_push_local(cg, src_addr_local);
     kit_cg_load(cg, i64_mem);
     kit_cg_push_local(cg, idx);
     kit_cg_load(cg, i64_mem);
     kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
     kit_cg_elem(cg, 0);
     kit_cg_load(cg, i8_mem);
     kit_cg_store(cg, i8_mem);
     kit_cg_jump(cg, back_loop);
   }
   kit_cg_label_place(cg, forward);
   /* Forward loop: idx = 0; while idx < n then dst[idx]=src[idx]; idx++. */
   kit_cg_push_local(cg, idx);
   kit_cg_push_int(cg, 0, b.id[KIT_CG_BUILTIN_I64]);
   kit_cg_store(cg, i64_mem);
   kit_cg_jump(cg, after_dir);
   kit_cg_label_place(cg, after_dir);
   kit_cg_label_place(cg, loop_start);
   kit_cg_push_local(cg, idx);
   kit_cg_load(cg, i64_mem);
   kit_cg_push_local(cg, n_local);
   kit_cg_load(cg, i64_mem);
   kit_cg_int_cmp(cg, KIT_CG_INT_GE_U);
   kit_cg_branch_true(cg, done);
   kit_cg_push_local(cg, dst_base_local);
   kit_cg_load(cg, ptr_mem);
   kit_cg_push_local(cg, dst_addr_local);
   kit_cg_load(cg, i64_mem);
   kit_cg_push_local(cg, idx);
   kit_cg_load(cg, i64_mem);
   kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
   kit_cg_elem(cg, 0);
   kit_cg_push_local(cg, src_base_local);
   kit_cg_load(cg, ptr_mem);
   kit_cg_push_local(cg, src_addr_local);
   kit_cg_load(cg, i64_mem);
   kit_cg_push_local(cg, idx);
   kit_cg_load(cg, i64_mem);
   kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
   kit_cg_elem(cg, 0);
   kit_cg_load(cg, i8_mem);
   kit_cg_store(cg, i8_mem);
   kit_cg_push_local(cg, idx);
   kit_cg_push_local(cg, idx);
   kit_cg_load(cg, i64_mem);
   kit_cg_push_int(cg, 1, b.id[KIT_CG_BUILTIN_I64]);
   kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
   kit_cg_store(cg, i64_mem);
   kit_cg_jump(cg, loop_start);
   kit_cg_label_place(cg, done);
 }
 
 /* Emit a byte-fill loop: writes (val & 0xff) into dst_base+dst_addr..+n. */
 static void wasm_cg_emit_byte_fill_loop(KitCg* cg, KitCgBuiltinTypes b,
                                         const WasmCgRuntime* rt,
                                         KitCgLocal dst_base_local,
                                         KitCgLocal dst_addr_local,
                                         KitCgLocal val_local,
                                         KitCgLocal n_local) {
   KitCgMemAccess i64_mem = wasm_cg_mem_type(b.id[KIT_CG_BUILTIN_I64]);
   KitCgMemAccess i32_mem = wasm_cg_mem_type(b.id[KIT_CG_BUILTIN_I32]);
   KitCgMemAccess i8_mem = wasm_cg_mem_type(b.id[KIT_CG_BUILTIN_I8]);
   KitCgMemAccess ptr_mem = wasm_cg_mem_type(rt->i8_ptr_ty);
   KitCgLocalAttrs attrs;
   KitCgLocal idx;
   KitCgLabel done = kit_cg_label_new(cg);
   KitCgLabel loop_start = kit_cg_label_new(cg);
   (void)rt;
   memset(&attrs, 0, sizeof attrs);
   attrs.flags = KIT_CG_LOCAL_COMPILER_TEMP;
   idx = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
   kit_cg_push_local(cg, idx);
   kit_cg_push_int(cg, 0, b.id[KIT_CG_BUILTIN_I64]);
   kit_cg_store(cg, i64_mem);
   kit_cg_label_place(cg, loop_start);
   kit_cg_push_local(cg, idx);
   kit_cg_load(cg, i64_mem);
   kit_cg_push_local(cg, n_local);
   kit_cg_load(cg, i64_mem);
   kit_cg_int_cmp(cg, KIT_CG_INT_GE_U);
   kit_cg_branch_true(cg, done);
   kit_cg_push_local(cg, dst_base_local);
   kit_cg_load(cg, ptr_mem);
   kit_cg_push_local(cg, dst_addr_local);
   kit_cg_load(cg, i64_mem);
   kit_cg_push_local(cg, idx);
   kit_cg_load(cg, i64_mem);
   kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
   kit_cg_elem(cg, 0);
   kit_cg_push_local(cg, val_local);
   kit_cg_load(cg, i32_mem);
   kit_cg_trunc(cg, b.id[KIT_CG_BUILTIN_I8]);
   kit_cg_store(cg, i8_mem);
   kit_cg_push_local(cg, idx);
   kit_cg_push_local(cg, idx);
   kit_cg_load(cg, i64_mem);
   kit_cg_push_int(cg, 1, b.id[KIT_CG_BUILTIN_I64]);
   kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
   kit_cg_store(cg, i64_mem);
   kit_cg_jump(cg, loop_start);
   kit_cg_label_place(cg, done);
 }
 
 /* Like wasm_cg_emit_byte_copy_loop but for table entries (struct-sized). */
 static void wasm_cg_emit_table_copy_loop(
     KitCompiler* c, KitCg* cg, KitCgBuiltinTypes b, const WasmCgRuntime* rt,
     KitCgLocal dst_base_local, KitCgLocal src_base_local,
     KitCgLocal dst_idx_local, KitCgLocal src_idx_local, KitCgLocal n_local) {
   KitCgMemAccess i64_mem = wasm_cg_mem_type(b.id[KIT_CG_BUILTIN_I64]);
   KitCgMemAccess ptr_mem = wasm_cg_mem_type(rt->table_entry_ptr_ty);
   KitCgMemAccess fn_mem = wasm_cg_mem_type(rt->void_ptr_ty);
   KitCgMemAccess i32_mem = wasm_cg_mem_type(b.id[KIT_CG_BUILTIN_I32]);
   KitCgLocalAttrs attrs;
   KitCgLocal idx, dir;
   KitCgLabel done = kit_cg_label_new(cg);
   KitCgLabel forward = kit_cg_label_new(cg);
   KitCgLabel loop_start = kit_cg_label_new(cg);
   (void)c;
   memset(&attrs, 0, sizeof attrs);
   attrs.flags = KIT_CG_LOCAL_COMPILER_TEMP;
   idx = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
   dir = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I32], attrs);
   kit_cg_push_local(cg, n_local);
   kit_cg_load(cg, i64_mem);
   kit_cg_push_int(cg, 0, b.id[KIT_CG_BUILTIN_I64]);
   kit_cg_int_cmp(cg, KIT_CG_INT_EQ);
   kit_cg_branch_true(cg, done);
   /* dir = (dst_base == src_base) ? (dst_idx > src_idx) : 0 */
   kit_cg_push_local(cg, dir);
   kit_cg_push_local(cg, dst_base_local);
   kit_cg_load(cg, ptr_mem);
   kit_cg_push_local(cg, src_base_local);
   kit_cg_load(cg, ptr_mem);
   kit_cg_int_cmp(cg, KIT_CG_INT_EQ);
   kit_cg_store(cg, i32_mem);
   kit_cg_push_local(cg, dir);
   kit_cg_load(cg, i32_mem);
   kit_cg_branch_false(cg, forward);
   kit_cg_push_local(cg, dst_idx_local);
   kit_cg_load(cg, i64_mem);
   kit_cg_push_local(cg, src_idx_local);
   kit_cg_load(cg, i64_mem);
   kit_cg_int_cmp(cg, KIT_CG_INT_GT_U);
   kit_cg_branch_false(cg, forward);
   {
     KitCgLabel back_loop = kit_cg_label_new(cg);
     kit_cg_push_local(cg, idx);
     kit_cg_push_local(cg, n_local);
     kit_cg_load(cg, i64_mem);
     kit_cg_store(cg, i64_mem);
     kit_cg_label_place(cg, back_loop);
     kit_cg_push_local(cg, idx);
     kit_cg_load(cg, i64_mem);
     kit_cg_push_int(cg, 0, b.id[KIT_CG_BUILTIN_I64]);
     kit_cg_int_cmp(cg, KIT_CG_INT_EQ);
     kit_cg_branch_true(cg, done);
     kit_cg_push_local(cg, idx);
     kit_cg_push_local(cg, idx);
     kit_cg_load(cg, i64_mem);
     kit_cg_push_int(cg, 1, b.id[KIT_CG_BUILTIN_I64]);
     kit_cg_int_binop(cg, KIT_CG_INT_SUB, 0);
     kit_cg_store(cg, i64_mem);
     /* copy fn and typeidx fields */
     kit_cg_push_local(cg, dst_base_local);
     kit_cg_load(cg, ptr_mem);
     kit_cg_push_local(cg, dst_idx_local);
     kit_cg_load(cg, i64_mem);
     kit_cg_push_local(cg, idx);
     kit_cg_load(cg, i64_mem);
     kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
     kit_cg_elem(cg, 0);
     kit_cg_field(cg, rt->table_entry_fn_field);
     kit_cg_push_local(cg, src_base_local);
     kit_cg_load(cg, ptr_mem);
     kit_cg_push_local(cg, src_idx_local);
     kit_cg_load(cg, i64_mem);
     kit_cg_push_local(cg, idx);
     kit_cg_load(cg, i64_mem);
     kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
     kit_cg_elem(cg, 0);
     kit_cg_field(cg, rt->table_entry_fn_field);
     kit_cg_load(cg, fn_mem);
     kit_cg_store(cg, fn_mem);
     kit_cg_push_local(cg, dst_base_local);
     kit_cg_load(cg, ptr_mem);
     kit_cg_push_local(cg, dst_idx_local);
     kit_cg_load(cg, i64_mem);
     kit_cg_push_local(cg, idx);
     kit_cg_load(cg, i64_mem);
     kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
     kit_cg_elem(cg, 0);
     kit_cg_field(cg, rt->table_entry_typeidx_field);
     kit_cg_push_local(cg, src_base_local);
     kit_cg_load(cg, ptr_mem);
     kit_cg_push_local(cg, src_idx_local);
     kit_cg_load(cg, i64_mem);
     kit_cg_push_local(cg, idx);
     kit_cg_load(cg, i64_mem);
     kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
     kit_cg_elem(cg, 0);
     kit_cg_field(cg, rt->table_entry_typeidx_field);
     kit_cg_load(cg, i32_mem);
     kit_cg_store(cg, i32_mem);
     kit_cg_jump(cg, back_loop);
   }
   kit_cg_label_place(cg, forward);
   kit_cg_push_local(cg, idx);
   kit_cg_push_int(cg, 0, b.id[KIT_CG_BUILTIN_I64]);
   kit_cg_store(cg, i64_mem);
   kit_cg_label_place(cg, loop_start);
   kit_cg_push_local(cg, idx);
   kit_cg_load(cg, i64_mem);
   kit_cg_push_local(cg, n_local);
   kit_cg_load(cg, i64_mem);
   kit_cg_int_cmp(cg, KIT_CG_INT_GE_U);
   kit_cg_branch_true(cg, done);
   kit_cg_push_local(cg, dst_base_local);
   kit_cg_load(cg, ptr_mem);
   kit_cg_push_local(cg, dst_idx_local);
   kit_cg_load(cg, i64_mem);
   kit_cg_push_local(cg, idx);
   kit_cg_load(cg, i64_mem);
   kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
   kit_cg_elem(cg, 0);
   kit_cg_field(cg, rt->table_entry_fn_field);
   kit_cg_push_local(cg, src_base_local);
   kit_cg_load(cg, ptr_mem);
   kit_cg_push_local(cg, src_idx_local);
   kit_cg_load(cg, i64_mem);
   kit_cg_push_local(cg, idx);
   kit_cg_load(cg, i64_mem);
   kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
   kit_cg_elem(cg, 0);
   kit_cg_field(cg, rt->table_entry_fn_field);
   kit_cg_load(cg, fn_mem);
   kit_cg_store(cg, fn_mem);
   kit_cg_push_local(cg, dst_base_local);
   kit_cg_load(cg, ptr_mem);
   kit_cg_push_local(cg, dst_idx_local);
   kit_cg_load(cg, i64_mem);
   kit_cg_push_local(cg, idx);
   kit_cg_load(cg, i64_mem);
   kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
   kit_cg_elem(cg, 0);
   kit_cg_field(cg, rt->table_entry_typeidx_field);
   kit_cg_push_local(cg, src_base_local);
   kit_cg_load(cg, ptr_mem);
   kit_cg_push_local(cg, src_idx_local);
   kit_cg_load(cg, i64_mem);
   kit_cg_push_local(cg, idx);
   kit_cg_load(cg, i64_mem);
   kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
   kit_cg_elem(cg, 0);
   kit_cg_field(cg, rt->table_entry_typeidx_field);
   kit_cg_load(cg, i32_mem);
   kit_cg_store(cg, i32_mem);
   kit_cg_push_local(cg, idx);
   kit_cg_push_local(cg, idx);
   kit_cg_load(cg, i64_mem);
   kit_cg_push_int(cg, 1, b.id[KIT_CG_BUILTIN_I64]);
   kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
   kit_cg_store(cg, i64_mem);
   kit_cg_jump(cg, loop_start);
   kit_cg_label_place(cg, done);
 }
 
 static void wasm_cg_cache_funcref_entry(
     KitCompiler* c, KitCg* cg, KitCgBuiltinTypes b, const WasmCgRuntime* rt,
     KitCgLocal ref_local, KitCgLocal fn_local, KitCgLocal typeidx_local,
     KitCgMemAccess ref_mem, KitCgMemAccess i32_mem) {
   KitCgLabel is_null = kit_cg_label_new(cg);
   KitCgLabel done = kit_cg_label_new(cg);
   kit_cg_push_local(cg, ref_local);
   kit_cg_load(cg, ref_mem);
   kit_cg_push_null(cg, rt->void_ptr_ty);
   kit_cg_int_cmp(cg, KIT_CG_INT_EQ);
   kit_cg_branch_true(cg, is_null);
 
   kit_cg_push_local(cg, fn_local);
   kit_cg_push_local(cg, ref_local);
   kit_cg_load(cg, ref_mem);
   kit_cg_bitcast(cg, kit_cg_type_ptr(c, rt->table_entry_ty, 0));
   kit_cg_deref(cg, 0);
   kit_cg_field(cg, rt->table_entry_fn_field);
   kit_cg_load(cg, ref_mem);
   kit_cg_store(cg, ref_mem);
 
   kit_cg_push_local(cg, typeidx_local);
   kit_cg_push_local(cg, ref_local);
   kit_cg_load(cg, ref_mem);
   kit_cg_bitcast(cg, kit_cg_type_ptr(c, rt->table_entry_ty, 0));
   kit_cg_deref(cg, 0);
   kit_cg_field(cg, rt->table_entry_typeidx_field);
   kit_cg_load(cg, i32_mem);
   kit_cg_store(cg, i32_mem);
   kit_cg_jump(cg, done);
 
   kit_cg_label_place(cg, is_null);
   kit_cg_push_local(cg, fn_local);
   kit_cg_push_null(cg, rt->void_ptr_ty);
   kit_cg_store(cg, ref_mem);
   kit_cg_push_local(cg, typeidx_local);
   kit_cg_push_int(cg, 0, b.id[KIT_CG_BUILTIN_I32]);
   kit_cg_store(cg, i32_mem);
   kit_cg_label_place(cg, done);
 }
 
 void wasm_emit_cg_into(KitCompiler* c, KitCg* cg, const WasmModule* m) {
   KitCgBuiltinTypes b = kit_cg_builtin_types(c);
   WasmCgRuntime rt;
   /* A KitArena owns transient frontend-side codegen state — sym tables, func
    * types, per-function local arrays, instance-record field tables, call
    * argument arrays. Lives for the duration of wasm_emit_cg_into; no fixed cap
    * on functions, params, locals, or instance fields. */
   KitArena* arena = NULL;
   KitCgSym init_sym = KIT_CG_SYM_NONE;
   uint32_t i, j;
   if (kit_arena_new(kit_compiler_context(c)->heap, 8192, &arena) != KIT_OK)
     wasm_error(c, wasm_loc(0, 0), "wasm: out of memory");
   if (!cg) wasm_error(c, wasm_loc(0, 0), "wasm: failed to initialize codegen");
   KitCgSym* syms =
       m->nfuncs ? kit_arena_zarray(arena, KitCgSym, m->nfuncs) : NULL;
   KitCgTypeId* func_types =
       m->nfuncs ? kit_arena_zarray(arena, KitCgTypeId, m->nfuncs) : NULL;
   wasm_cg_build_runtime(c, b, m, &rt, arena);
   {
     KitCgFuncSig sig;
     KitCgDecl decl;
     memset(&sig, 0, sizeof sig);
     sig.call_conv = KIT_CG_CC_TARGET_C;
     rt.trap_func_ty = kit_cg_type_func(c, sig);
     if (!rt.trap_func_ty)
       wasm_error(c, wasm_loc(0, 0), "wasm: failed to create trap type");
     for (uint32_t k = 0; k < WASM_TRAP_COUNT; ++k) {
       KitSym source_name = kit_sym_intern(c, kit_slice_cstr(wasm_trap_name(k)));
       memset(&decl, 0, sizeof decl);
       decl.kind = KIT_CG_DECL_FUNC;
       decl.linkage_name = kit_cg_c_linkage_name(c, source_name);
       decl.display_name = source_name;
       decl.type = rt.trap_func_ty;
       decl.sym.bind = KIT_SB_LOCAL;
       decl.as.func.flags = KIT_CG_FUNC_NORETURN | KIT_CG_FUNC_COLD;
       rt.trap_syms[k] = kit_cg_decl(cg, decl);
       if (!rt.trap_syms[k])
         wasm_error(c, wasm_loc(0, 0), "wasm: failed to declare trap helper");
     }
   }
   {
     KitCgDecl decl;
     KitCgFuncParam init_param;
     KitCgFuncSig sig;
     memset(&init_param, 0, sizeof init_param);
     init_param.type = rt.instance_ptr_ty;
     memset(&sig, 0, sizeof sig);
     sig.params = &init_param;
     sig.nparams = 1;
     sig.call_conv = KIT_CG_CC_TARGET_C;
     memset(&decl, 0, sizeof decl);
     decl.kind = KIT_CG_DECL_FUNC;
     decl.linkage_name = kit_cg_c_linkage_name(
         c, kit_sym_intern(c, KIT_SLICE_LIT("__kit_wasm_init")));
     decl.display_name = decl.linkage_name;
     decl.type = kit_cg_type_func(c, sig);
     decl.sym.bind = KIT_SB_GLOBAL;
     init_sym = kit_cg_decl(cg, decl);
   }
   for (i = 0; i < m->nfuncs; ++i) {
     const WasmFunc* f = &m->funcs[i];
     KitCgFuncParam* cg_params =
         kit_arena_zarray(arena, KitCgFuncParam, f->nparams + 1u);
     KitCgFuncSig sig;
     KitCgDecl decl;
     char local_name[40];
     KitSym source_name;
     kit_cg_set_loc(cg, f->loc);
     cg_params[0].type = rt.instance_ptr_ty;
     for (j = 0; j < f->nparams; ++j) {
       cg_params[j + 1u].type = wasm_cg_type(c, b, f->params[j]);
     }
     memset(&sig, 0, sizeof sig);
     if (f->nresults) sig.result.type = wasm_cg_type(c, b, f->results[0]);
     sig.params = cg_params;
     sig.nparams = f->nparams + 1u;
     sig.call_conv = KIT_CG_CC_TARGET_C;
     func_types[i] = kit_cg_type_func(c, sig);
     if (!func_types[i])
       wasm_error(c, f->loc, "wasm: failed to create function type");
     if (f->is_import) {
       syms[i] = KIT_CG_SYM_NONE;
       continue;
     }
     if (f->export_name) {
       source_name = kit_sym_intern(c, kit_slice_cstr(f->export_name));
     } else {
       size_t pos = 0;
       const char prefix[] = "__kit_wasm_func_";
       uint32_t n = i, div = 1000000000u;
       memcpy(local_name, prefix, sizeof(prefix) - 1u);
       pos = sizeof(prefix) - 1u;
       while (div > 1u && n / div == 0) div /= 10u;
       while (div) {
         local_name[pos++] = (char)('0' + (n / div) % 10u);
         div /= 10u;
       }
       local_name[pos] = '\0';
       source_name = kit_sym_intern(c, kit_slice_cstr(local_name));
     }
     memset(&decl, 0, sizeof decl);
     decl.kind = KIT_CG_DECL_FUNC;
     decl.linkage_name = kit_cg_c_linkage_name(c, source_name);
     decl.display_name = source_name;
     decl.type = func_types[i];
     decl.sym.bind = f->export_name ? KIT_SB_GLOBAL : KIT_SB_LOCAL;
     syms[i] = kit_cg_decl(cg, decl);
     if (!syms[i]) wasm_error(c, f->loc, "wasm: failed to declare function");
   }
   for (uint32_t k = 0; k < WASM_TRAP_COUNT; ++k) {
     kit_cg_func_begin(cg, rt.trap_syms[k]);
     wasm_cg_emit_raw_trap(cg);
     kit_cg_func_end(cg);
   }
   if (init_sym) {
     KitCgLocalAttrs attrs;
     KitCgLocal instance_local;
     memset(&attrs, 0, sizeof attrs);
     kit_cg_func_begin(cg, init_sym);
     instance_local = kit_cg_param(cg, 0, rt.instance_ptr_ty, attrs);
     for (i = 0; i < m->nmemories; ++i) {
       const WasmMemory* mem = &m->memories[i];
       uint64_t max_pages = mem->has_max ? mem->max_pages : mem->min_pages;
       uint32_t flags = (mem->shared ? 1u : 0u) | (mem->is64 ? 2u : 0u);
       wasm_cg_push_instance_ptr(cg, &rt, instance_local);
       kit_cg_deref(cg, (int64_t)(rt.memory_offset[i] + rt.memory_pages_offset));
       kit_cg_push_int(cg, mem->min_pages, b.id[KIT_CG_BUILTIN_I64]);
       kit_cg_store(cg, wasm_cg_mem(c, b, WASM_VAL_I64));
       wasm_cg_push_instance_ptr(cg, &rt, instance_local);
       kit_cg_deref(cg,
                    (int64_t)(rt.memory_offset[i] + rt.memory_max_pages_offset));
       kit_cg_push_int(cg, max_pages, b.id[KIT_CG_BUILTIN_I64]);
       kit_cg_store(cg, wasm_cg_mem(c, b, WASM_VAL_I64));
       wasm_cg_push_instance_ptr(cg, &rt, instance_local);
       kit_cg_deref(cg, (int64_t)(rt.memory_offset[i] + rt.memory_flags_offset));
       kit_cg_push_int(cg, flags, b.id[KIT_CG_BUILTIN_I32]);
       kit_cg_store(cg, wasm_cg_mem(c, b, WASM_VAL_I32));
       /* For each active data segment targeting this memory, memcpy its
        * bytes into the linear memory at the segment's offset. Passive
        * segments are not initialized here — they're consumed by an
        * explicit `memory.init` instruction at runtime. */
       for (uint32_t di = 0; di < m->ndata; ++di) {
         const WasmDataSegment* d = &m->data[di];
         if (d->mode != WASM_SEG_ACTIVE) continue;
         if (d->memidx != i) continue;
         if (!d->nbytes) continue;
         {
           KitCgSym data_sym = kit_cg_const_data(cg, d->bytes, d->nbytes, 16,
                                                 b.id[KIT_CG_BUILTIN_I8]);
           KitCgMemAccess byte_mem = wasm_cg_mem_type(b.id[KIT_CG_BUILTIN_I8]);
           wasm_cg_push_memory_data_ptr(cg, &rt, instance_local, i);
           if (d->offset) wasm_cg_ptr_add_offset(cg, b, d->offset, rt.i8_ptr_ty);
           kit_cg_push_symbol_addr(cg, data_sym, 0);
           kit_cg_memcpy(cg, d->nbytes, byte_mem, byte_mem);
         }
       }
     }
     /* Populate per-module passive data segment slots. Passive segments
      * intern their bytes as a const data symbol and store {base, len}
      * into the instance slot; memory.init/data.drop read/clear this slot.
      * Active segments leave their slot at zero-len (they're effectively
      * dropped after init), which makes memory.init on an active index
      * trap on any non-zero copy length. */
     for (uint32_t di = 0; di < m->ndata; ++di) {
       const WasmDataSegment* d = &m->data[di];
       wasm_cg_push_passive_data_lvalue(cg, &rt, instance_local, di);
       kit_cg_field(cg, rt.passive_data_base_field);
       if (d->mode == WASM_SEG_PASSIVE && d->nbytes) {
         KitCgSym data_sym = kit_cg_const_data(cg, d->bytes, d->nbytes, 16,
                                               b.id[KIT_CG_BUILTIN_I8]);
         kit_cg_push_symbol_addr(cg, data_sym, 0);
       } else {
         kit_cg_push_null(cg, rt.i8_ptr_ty);
       }
       kit_cg_store(cg, wasm_cg_mem_type(rt.i8_ptr_ty));
       wasm_cg_push_passive_data_lvalue(cg, &rt, instance_local, di);
       kit_cg_field(cg, rt.passive_data_len_field);
       kit_cg_push_int(cg, (d->mode == WASM_SEG_PASSIVE) ? d->nbytes : 0u,
                       b.id[KIT_CG_BUILTIN_I64]);
       kit_cg_store(cg, wasm_cg_mem(c, b, WASM_VAL_I64));
     }
     for (i = 0; i < m->nfuncs; ++i) {
       wasm_cg_push_instance_ptr(cg, &rt, instance_local);
       kit_cg_deref(cg, (int64_t)(rt.func_ref_entry_offset[i] +
                                  rt.table_entry_fn_offset));
       if (m->funcs[i].is_import) {
         wasm_cg_push_import_func_ptr(cg, &rt, instance_local, i);
       } else {
         kit_cg_push_symbol_addr(cg, syms[i], 0);
         kit_cg_bitcast(cg, rt.void_ptr_ty);
       }
       kit_cg_store(cg, wasm_cg_mem_type(rt.void_ptr_ty));
       wasm_cg_push_instance_ptr(cg, &rt, instance_local);
       kit_cg_deref(cg, (int64_t)(rt.func_ref_entry_offset[i] +
                                  rt.table_entry_typeidx_offset));
       kit_cg_push_int(cg, m->funcs[i].typeidx, b.id[KIT_CG_BUILTIN_I32]);
       kit_cg_store(cg, wasm_cg_mem(c, b, WASM_VAL_I32));
     }
     for (i = 0; i < m->ntables; ++i) {
       const WasmTable* t = &m->tables[i];
       uint32_t max = t->has_max ? t->max : t->min;
       /* tables[i].entries = &instance->table_entries_arr[i][0]. The address
        * of the entries array is instance + table_entries_offset[i]. */
       wasm_cg_push_instance_ptr(cg, &rt, instance_local);
       kit_cg_deref(cg,
                    (int64_t)(rt.table_offset[i] + rt.table_entries_ptr_offset));
       wasm_cg_push_instance_ptr(cg, &rt, instance_local);
       wasm_cg_ptr_add_offset(cg, b, rt.table_entries_offset[i],
                              rt.table_entry_ptr_ty);
       kit_cg_store(cg, wasm_cg_mem_type(rt.table_entry_ptr_ty));
       wasm_cg_push_instance_ptr(cg, &rt, instance_local);
       kit_cg_deref(cg, (int64_t)(rt.table_offset[i] + rt.table_len_offset));
       kit_cg_push_int(cg, t->min, b.id[KIT_CG_BUILTIN_I32]);
       kit_cg_store(cg, wasm_cg_mem(c, b, WASM_VAL_I32));
       wasm_cg_push_instance_ptr(cg, &rt, instance_local);
       kit_cg_deref(cg, (int64_t)(rt.table_offset[i] + rt.table_max_offset));
       kit_cg_push_int(cg, max, b.id[KIT_CG_BUILTIN_I32]);
       kit_cg_store(cg, wasm_cg_mem(c, b, WASM_VAL_I32));
     }
     for (i = 0; i < m->nelems; ++i) {
       const WasmElemSegment* seg = &m->elems[i];
       int is_passive = (seg->mode == WASM_SEG_PASSIVE);
       for (j = 0; j < seg->nfuncs; ++j) {
         uint32_t funcidx = seg->funcs[j];
         KitCgLocalAttrs tmp_attrs;
         KitCgLocal slot_local;
         memset(&tmp_attrs, 0, sizeof tmp_attrs);
         tmp_attrs.flags = KIT_CG_LOCAL_COMPILER_TEMP;
         slot_local = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I32], tmp_attrs);
         kit_cg_push_local(cg, slot_local);
         if (is_passive) {
           kit_cg_push_int(cg, j, b.id[KIT_CG_BUILTIN_I32]);
         } else {
           kit_cg_push_int(cg, (uint32_t)(seg->offset + j),
                           b.id[KIT_CG_BUILTIN_I32]);
         }
         kit_cg_store(cg, wasm_cg_mem(c, b, WASM_VAL_I32));
         /* Resolve the target table-entry lvalue: active segments write into
          * the module's table; passive segments write into the per-segment
          * inline storage array so the bytes survive instantiation for
          * later table.init. */
         if (is_passive) {
           wasm_cg_push_passive_elem_storage_array_lvalue(cg, &rt,
                                                          instance_local, i);
           kit_cg_addr(cg);
           kit_cg_bitcast(cg, rt.table_entry_ptr_ty);
           kit_cg_push_local(cg, slot_local);
           kit_cg_load(cg, wasm_cg_mem(c, b, WASM_VAL_I32));
           kit_cg_elem(cg, 0);
         } else {
           wasm_cg_push_table_entry_lvalue(cg, &rt, instance_local,
                                           seg->tableidx, slot_local,
                                           wasm_cg_mem(c, b, WASM_VAL_I32));
         }
         kit_cg_field(cg, rt.table_entry_fn_field);
         if (m->funcs[funcidx].is_import) {
           wasm_cg_push_import_func_ptr(cg, &rt, instance_local, funcidx);
         } else {
           kit_cg_push_symbol_addr(cg, syms[funcidx], 0);
           kit_cg_bitcast(cg, rt.void_ptr_ty);
         }
         kit_cg_store(cg, wasm_cg_mem_type(rt.void_ptr_ty));
         if (is_passive) {
           wasm_cg_push_passive_elem_storage_array_lvalue(cg, &rt,
                                                          instance_local, i);
           kit_cg_addr(cg);
           kit_cg_bitcast(cg, rt.table_entry_ptr_ty);
           kit_cg_push_local(cg, slot_local);
           kit_cg_load(cg, wasm_cg_mem(c, b, WASM_VAL_I32));
           kit_cg_elem(cg, 0);
         } else {
           wasm_cg_push_table_entry_lvalue(cg, &rt, instance_local,
                                           seg->tableidx, slot_local,
                                           wasm_cg_mem(c, b, WASM_VAL_I32));
         }
         kit_cg_field(cg, rt.table_entry_typeidx_field);
         kit_cg_push_int(cg, m->funcs[funcidx].typeidx,
                         b.id[KIT_CG_BUILTIN_I32]);
         kit_cg_store(cg, wasm_cg_mem(c, b, WASM_VAL_I32));
       }
       /* Initialize the passive_elem descriptor (entries pointer + length).
        * Active segments get a zero-length descriptor so table.init on an
        * active index traps on any non-zero copy. */
       wasm_cg_push_passive_elem_lvalue(cg, &rt, instance_local, i);
       kit_cg_field(cg, rt.passive_elem_entries_field);
       if (is_passive && seg->nfuncs) {
         wasm_cg_push_passive_elem_storage_array_lvalue(cg, &rt, instance_local,
                                                        i);
         kit_cg_addr(cg);
         kit_cg_bitcast(cg, rt.table_entry_ptr_ty);
         kit_cg_push_int(cg, 0, b.id[KIT_CG_BUILTIN_I32]);
         kit_cg_elem(cg, 0);
         kit_cg_addr(cg);
       } else {
         kit_cg_push_null(cg, rt.table_entry_ptr_ty);
       }
       kit_cg_store(cg, wasm_cg_mem_type(rt.table_entry_ptr_ty));
       wasm_cg_push_passive_elem_lvalue(cg, &rt, instance_local, i);
       kit_cg_field(cg, rt.passive_elem_length_field);
       kit_cg_push_int(cg, is_passive ? seg->nfuncs : 0u,
                       b.id[KIT_CG_BUILTIN_I32]);
       kit_cg_store(cg, wasm_cg_mem(c, b, WASM_VAL_I32));
     }
     for (i = 0; i < m->nglobals; ++i) {
       const WasmGlobal* g = &m->globals[i];
       if (g->is_import) continue;
       wasm_cg_push_instance_ptr(cg, &rt, instance_local);
       kit_cg_deref(cg, (int64_t)rt.global_offset[i]);
       if (g->type == WASM_VAL_F32 || g->type == WASM_VAL_F64)
         kit_cg_push_float(cg, g->init.fp, wasm_cg_type(c, b, g->type));
       else
         kit_cg_push_int(cg, (uint64_t)g->init.imm, wasm_cg_type(c, b, g->type));
       kit_cg_store(cg, wasm_cg_mem(c, b, g->type));
     }
     if (m->has_start)
       wasm_cg_call_func(c, cg, b, &m->funcs[m->start_func], &rt,
                         syms[m->start_func], func_types[m->start_func],
                         instance_local, m->start_func, 0, m->start_field_loc,
                         arena);
     kit_cg_ret(cg);
     kit_cg_func_end(cg);
   }
   /* Host-import resolution metadata: emit __kit_wasm_nimports always, plus
    * import descriptors and kind-specific type arrays when the module declares
    * imports. Function import signatures are deduplicated in __kit_wasm_types.
    * Read by kit_wasm_bind_host_imports. */
   wasm_cg_emit_host_import_metadata(c, cg, b, m, &rt, arena);
   wasm_cg_emit_runtime_layout_metadata(c, cg, b, m, &rt);
   for (i = 0; i < m->nfuncs; ++i) {
     const WasmFunc* f = &m->funcs[i];
     typedef struct WasmCgControl {
       uint8_t kind;
       int seen_else;
       KitCgLabel start;
       KitCgLabel end;
       KitCgLabel else_label;
     } WasmCgControl;
     /* Per-function arena-allocated locals and control stack. locals is
      * sized to nparams + nlocals; the control stack starts at 16 entries
      * and doubles in-place on overflow (arena pointers are stable, so
      * we use a side heap realloc instead). */
     KitCgLocal* locals =
         (f->nparams + f->nlocals)
             ? kit_arena_array(arena, KitCgLocal, f->nparams + f->nlocals)
             : NULL;
     KitHeap* heap = kit_compiler_context(c)->heap;
     uint32_t control_cap = 16u;
     WasmCgControl* control = (WasmCgControl*)heap->alloc(
         heap, sizeof(WasmCgControl) * control_cap, _Alignof(WasmCgControl));
     uint32_t ncontrol = 0;
     KitCgLocal instance_local;
     if (!control) wasm_error(c, f->loc, "wasm: out of memory");
     if (f->is_import) {
       heap->free(heap, control, sizeof(WasmCgControl) * control_cap);
       continue;
     }
     kit_cg_set_loc(cg, f->loc);
     kit_cg_func_begin(cg, syms[i]);
     {
       KitCgLocalAttrs attrs;
       memset(&attrs, 0, sizeof attrs);
       instance_local = kit_cg_param(cg, 0, rt.instance_ptr_ty, attrs);
     }
     for (j = 0; j < f->nparams; ++j) {
       KitCgLocalAttrs attrs;
       memset(&attrs, 0, sizeof attrs);
       locals[j] =
           kit_cg_param(cg, j + 1u, wasm_cg_type(c, b, f->params[j]), attrs);
     }
     for (j = 0; j < f->nlocals; ++j) {
       KitCgLocalAttrs attrs;
       memset(&attrs, 0, sizeof attrs);
       attrs.flags = KIT_CG_LOCAL_COMPILER_TEMP;
       locals[f->nparams + j] =
           kit_cg_local(cg, wasm_cg_type(c, b, f->locals[j]), attrs);
       kit_cg_push_local(cg, locals[f->nparams + j]);
       wasm_cg_push_zero(c, cg, b, f->locals[j]);
       kit_cg_store(cg, wasm_cg_mem(c, b, f->locals[j]));
     }
     for (j = 0; j < f->ninsns; ++j) {
       WasmInsn in = f->insns[j];
       kit_cg_set_loc(cg, in.loc);
       switch (in.kind) {
         case WASM_INSN_UNREACHABLE:
           wasm_cg_trap_unreachable(cg, &rt);
           break;
         case WASM_INSN_NOP:
           break;
         case WASM_INSN_BLOCK:
           if (ncontrol == control_cap) {
             uint32_t new_cap = control_cap * 2u;
             void* p = heap->realloc(
                 heap, control, sizeof(WasmCgControl) * control_cap,
                 sizeof(WasmCgControl) * new_cap, _Alignof(WasmCgControl));
             if (!p) wasm_error(c, in.loc, "wasm: out of memory");
             control = (WasmCgControl*)p;
             control_cap = new_cap;
           }
           memset(&control[ncontrol], 0, sizeof control[ncontrol]);
           control[ncontrol].kind = WASM_INSN_BLOCK;
           control[ncontrol].end = kit_cg_label_new(cg);
           ncontrol++;
           break;
         case WASM_INSN_LOOP:
           if (ncontrol == control_cap) {
             uint32_t new_cap = control_cap * 2u;
             void* p = heap->realloc(
                 heap, control, sizeof(WasmCgControl) * control_cap,
                 sizeof(WasmCgControl) * new_cap, _Alignof(WasmCgControl));
             if (!p) wasm_error(c, in.loc, "wasm: out of memory");
             control = (WasmCgControl*)p;
             control_cap = new_cap;
           }
           memset(&control[ncontrol], 0, sizeof control[ncontrol]);
           control[ncontrol].kind = WASM_INSN_LOOP;
           control[ncontrol].start = kit_cg_label_new(cg);
           control[ncontrol].end = kit_cg_label_new(cg);
           kit_cg_label_place(cg, control[ncontrol].start);
           ncontrol++;
           break;
         case WASM_INSN_IF:
           if (ncontrol == control_cap) {
             uint32_t new_cap = control_cap * 2u;
             void* p = heap->realloc(
                 heap, control, sizeof(WasmCgControl) * control_cap,
                 sizeof(WasmCgControl) * new_cap, _Alignof(WasmCgControl));
             if (!p) wasm_error(c, in.loc, "wasm: out of memory");
             control = (WasmCgControl*)p;
             control_cap = new_cap;
           }
           memset(&control[ncontrol], 0, sizeof control[ncontrol]);
           control[ncontrol].kind = WASM_INSN_IF;
           control[ncontrol].else_label = kit_cg_label_new(cg);
           control[ncontrol].end = kit_cg_label_new(cg);
           kit_cg_branch_false(cg, control[ncontrol].else_label);
           ncontrol++;
           break;
         case WASM_INSN_ELSE:
           if (!ncontrol || control[ncontrol - 1u].kind != WASM_INSN_IF)
             wasm_error(c, in.loc, "wasm: else without if");
           kit_cg_jump(cg, control[ncontrol - 1u].end);
           kit_cg_label_place(cg, control[ncontrol - 1u].else_label);
           control[ncontrol - 1u].seen_else = 1;
           break;
         case WASM_INSN_END:
           if (!ncontrol) wasm_error(c, in.loc, "wasm: end without block");
           ncontrol--;
           if (control[ncontrol].kind == WASM_INSN_IF &&
               !control[ncontrol].seen_else)
             kit_cg_label_place(cg, control[ncontrol].else_label);
           kit_cg_label_place(cg, control[ncontrol].end);
           break;
         case WASM_INSN_BR: {
           uint32_t depth = (uint32_t)in.imm;
           uint32_t idx;
           if (depth >= ncontrol)
             wasm_error(c, in.loc, "wasm: branch depth out of range");
           idx = ncontrol - 1u - depth;
           kit_cg_jump(cg, control[idx].kind == WASM_INSN_LOOP
                               ? control[idx].start
                               : control[idx].end);
           break;
         }
         case WASM_INSN_BR_IF: {
           uint32_t depth = (uint32_t)in.imm;
           uint32_t idx;
           if (depth >= ncontrol)
             wasm_error(c, in.loc, "wasm: branch depth out of range");
           idx = ncontrol - 1u - depth;
           kit_cg_branch_true(cg, control[idx].kind == WASM_INSN_LOOP
                                      ? control[idx].start
                                      : control[idx].end);
           break;
         }
         case WASM_INSN_BR_TABLE: {
           KitCgSwitch sw;
           if (in.ntargets == 0)
             wasm_error(c, in.loc, "wasm: bad br_table target count");
           KitCgSwitchCase* cases =
               kit_arena_zarray(arena, KitCgSwitchCase, in.ntargets);
           for (uint32_t k = 0; k + 1u < in.ntargets; ++k) {
             uint32_t idx;
             if (in.targets[k] >= ncontrol)
               wasm_error(c, in.loc, "wasm: branch depth out of range");
             idx = ncontrol - 1u - in.targets[k];
             cases[k].value = k;
             cases[k].label = control[idx].kind == WASM_INSN_LOOP
                                  ? control[idx].start
                                  : control[idx].end;
           }
           if (in.targets[in.ntargets - 1u] >= ncontrol)
             wasm_error(c, in.loc, "wasm: branch depth out of range");
           memset(&sw, 0, sizeof sw);
           sw.selector_type = b.id[KIT_CG_BUILTIN_I32];
           sw.cases = cases;
           sw.ncases = in.ntargets - 1u;
           sw.default_label =
               control[ncontrol - 1u - in.targets[in.ntargets - 1u]].kind ==
                       WASM_INSN_LOOP
                   ? control[ncontrol - 1u - in.targets[in.ntargets - 1u]].start
                   : control[ncontrol - 1u - in.targets[in.ntargets - 1u]].end;
           /* br_table is dense-by-construction (case values 0..N-1);
            * tell targets that care that a jump table is the natural
            * lowering. Targets that don't honour the hint fall back to
            * a cmp chain, which is still correct. */
           sw.hint = KIT_CG_SWITCH_JUMP_TABLE;
           kit_cg_switch(cg, sw);
           break;
         }
         case WASM_INSN_SELECT: {
           KitCgTypeId ty = wasm_cg_type(c, b, (WasmValType)in.type);
           KitCgMemAccess mem = wasm_cg_mem(c, b, (WasmValType)in.type);
           KitCgLocalAttrs attrs;
           KitCgLocal lhs, rhs, cond, result;
           KitCgLabel else_label = kit_cg_label_new(cg);
           KitCgLabel end_label = kit_cg_label_new(cg);
           memset(&attrs, 0, sizeof attrs);
           attrs.flags = KIT_CG_LOCAL_COMPILER_TEMP;
           lhs = kit_cg_local(cg, ty, attrs);
           rhs = kit_cg_local(cg, ty, attrs);
           result = kit_cg_local(cg, ty, attrs);
           cond = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I32], attrs);
 
           kit_cg_push_local(cg, cond);
           kit_cg_swap(cg);
           kit_cg_store(cg, wasm_cg_mem(c, b, WASM_VAL_I32));
           kit_cg_push_local(cg, rhs);
           kit_cg_swap(cg);
           kit_cg_store(cg, mem);
           kit_cg_push_local(cg, lhs);
           kit_cg_swap(cg);
           kit_cg_store(cg, mem);
 
           kit_cg_push_local(cg, cond);
           kit_cg_load(cg, wasm_cg_mem(c, b, WASM_VAL_I32));
           kit_cg_branch_false(cg, else_label);
           kit_cg_push_local(cg, result);
           kit_cg_push_local(cg, lhs);
           kit_cg_load(cg, mem);
           kit_cg_store(cg, mem);
           kit_cg_jump(cg, end_label);
           kit_cg_label_place(cg, else_label);
           kit_cg_push_local(cg, result);
           kit_cg_push_local(cg, rhs);
           kit_cg_load(cg, mem);
           kit_cg_store(cg, mem);
           kit_cg_label_place(cg, end_label);
           kit_cg_push_local(cg, result);
           kit_cg_load(cg, mem);
         } break;
         case WASM_INSN_I32_CONST:
           kit_cg_push_int(cg, (uint64_t)(uint32_t)in.imm,
                           b.id[KIT_CG_BUILTIN_I32]);
           break;
         case WASM_INSN_I64_CONST:
           kit_cg_push_int(cg, (uint64_t)in.imm, b.id[KIT_CG_BUILTIN_I64]);
           break;
         case WASM_INSN_F32_CONST:
           kit_cg_push_float(cg, in.fp, b.id[KIT_CG_BUILTIN_F32]);
           break;
         case WASM_INSN_F64_CONST:
           kit_cg_push_float(cg, in.fp, b.id[KIT_CG_BUILTIN_F64]);
           break;
         case WASM_INSN_LOCAL_GET: {
           uint32_t index = (uint32_t)in.imm;
           kit_cg_push_local(cg, locals[index]);
           kit_cg_load(cg, wasm_cg_mem(c, b, wasm_func_local_type(f, index)));
           break;
         }
         case WASM_INSN_LOCAL_SET: {
           uint32_t index = (uint32_t)in.imm;
           kit_cg_push_local(cg, locals[index]);
           kit_cg_swap(cg);
           kit_cg_store(cg, wasm_cg_mem(c, b, wasm_func_local_type(f, index)));
           break;
         }
         case WASM_INSN_LOCAL_TEE: {
           uint32_t index = (uint32_t)in.imm;
           kit_cg_dup(cg);
           kit_cg_push_local(cg, locals[index]);
           kit_cg_swap(cg);
           kit_cg_store(cg, wasm_cg_mem(c, b, wasm_func_local_type(f, index)));
           break;
         }
         case WASM_INSN_CALL:
         case WASM_INSN_RETURN_CALL:
           wasm_cg_call_func(c, cg, b, &m->funcs[in.imm], &rt, syms[in.imm],
                             func_types[in.imm], instance_local,
                             (uint32_t)in.imm, in.kind == WASM_INSN_RETURN_CALL,
                             in.loc, arena);
           break;
         case WASM_INSN_CALL_INDIRECT:
         case WASM_INSN_RETURN_CALL_INDIRECT: {
           const WasmFuncType* t = &m->types[in.imm];
           KitCgLocalAttrs attrs;
           KitCgLocal selector, callee, result = KIT_CG_LOCAL_NONE;
           KitCgLocal* args =
               t->nparams ? kit_arena_array(arena, KitCgLocal, t->nparams)
                          : NULL;
           KitCgLabel ok;
           KitCgMemAccess i32_mem = wasm_cg_mem(c, b, WASM_VAL_I32);
           KitCgFuncParam* indirect_params =
               kit_arena_zarray(arena, KitCgFuncParam, t->nparams + 1u);
           KitCgFuncSig indirect_sig;
           KitCgTypeId indirect_func_type;
           memset(&attrs, 0, sizeof attrs);
           attrs.flags = KIT_CG_LOCAL_COMPILER_TEMP;
           indirect_params[0].type = rt.instance_ptr_ty;
           for (uint32_t p = 0; p < t->nparams; ++p)
             indirect_params[p + 1u].type = wasm_cg_type(c, b, t->params[p]);
           memset(&indirect_sig, 0, sizeof indirect_sig);
           if (t->nresults)
             indirect_sig.result.type = wasm_cg_type(c, b, t->results[0]);
           indirect_sig.params = indirect_params;
           indirect_sig.nparams = t->nparams + 1u;
           indirect_sig.call_conv = KIT_CG_CC_TARGET_C;
           indirect_func_type = kit_cg_type_func(c, indirect_sig);
           selector = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I32], attrs);
           callee = kit_cg_local(cg, rt.void_ptr_ty, attrs);
           if (t->nresults)
             result = kit_cg_local(cg, wasm_cg_type(c, b, t->results[0]), attrs);
           kit_cg_push_local(cg, selector);
           kit_cg_swap(cg);
           kit_cg_store(cg, i32_mem);
           for (uint32_t p = 0; p < t->nparams; ++p) {
             uint32_t param = t->nparams - 1u - p;
             args[param] =
                 kit_cg_local(cg, wasm_cg_type(c, b, t->params[param]), attrs);
             kit_cg_push_local(cg, args[param]);
             kit_cg_swap(cg);
             kit_cg_store(cg, wasm_cg_mem(c, b, t->params[param]));
           }
 
           ok = kit_cg_label_new(cg);
           kit_cg_push_local(cg, selector);
           kit_cg_load(cg, i32_mem);
           {
             wasm_cg_push_instance_ptr(cg, &rt, instance_local);
             kit_cg_deref(
                 cg, (int64_t)(rt.table_offset[in.align] + rt.table_len_offset));
             kit_cg_load(cg, i32_mem);
           }
           kit_cg_int_cmp(cg, KIT_CG_INT_LT_U);
           kit_cg_branch_true(cg, ok);
           wasm_cg_trap_table(cg, &rt);
           kit_cg_label_place(cg, ok);
 
           ok = kit_cg_label_new(cg);
           kit_cg_push_local(cg, callee);
           wasm_cg_push_table_entry_ptr(cg, b, &rt, instance_local, in.align,
                                        selector, i32_mem);
           {
             kit_cg_deref(cg, (int64_t)rt.table_entry_fn_offset);
             kit_cg_load(cg, wasm_cg_mem_type(rt.void_ptr_ty));
           }
           kit_cg_store(cg, wasm_cg_mem_type(rt.void_ptr_ty));
           kit_cg_push_local(cg, callee);
           kit_cg_load(cg, wasm_cg_mem_type(rt.void_ptr_ty));
           kit_cg_push_null(cg, rt.void_ptr_ty);
           kit_cg_int_cmp(cg, KIT_CG_INT_NE);
           kit_cg_branch_true(cg, ok);
           wasm_cg_trap_table(cg, &rt);
           kit_cg_label_place(cg, ok);
 
           ok = kit_cg_label_new(cg);
           wasm_cg_push_table_entry_ptr(cg, b, &rt, instance_local, in.align,
                                        selector, i32_mem);
           {
             kit_cg_deref(cg, (int64_t)rt.table_entry_typeidx_offset);
             kit_cg_load(cg, i32_mem);
           }
           kit_cg_push_int(cg, (uint32_t)in.imm, b.id[KIT_CG_BUILTIN_I32]);
           kit_cg_int_cmp(cg, KIT_CG_INT_EQ);
           kit_cg_branch_true(cg, ok);
           wasm_cg_trap_signature(cg, &rt);
           kit_cg_label_place(cg, ok);
 
           kit_cg_push_local(cg, callee);
           kit_cg_load(cg, wasm_cg_mem_type(rt.void_ptr_ty));
           kit_cg_bitcast(cg, kit_cg_type_ptr(c, indirect_func_type, 0));
           kit_cg_push_local(cg, instance_local);
           kit_cg_load(cg, wasm_cg_mem_type(rt.instance_ptr_ty));
           for (uint32_t p = 0; p < t->nparams; ++p) {
             kit_cg_push_local(cg, args[p]);
             kit_cg_load(cg, wasm_cg_mem(c, b, t->params[p]));
           }
           kit_cg_call(
               cg, t->nparams + 1u, indirect_func_type,
               (KitCgCallAttrs){.tail = in.kind == WASM_INSN_RETURN_CALL_INDIRECT
                                            ? KIT_CG_TAIL_MUST
                                            : KIT_CG_TAIL_DEFAULT});
           if (in.kind == WASM_INSN_RETURN_CALL_INDIRECT) {
             if (t->nresults) wasm_cg_push_zero(c, cg, b, t->results[0]);
             kit_cg_unreachable(cg);
           } else if (t->nresults) {
             kit_cg_push_local(cg, result);
             kit_cg_swap(cg);
             kit_cg_store(cg, wasm_cg_mem(c, b, t->results[0]));
             kit_cg_push_local(cg, result);
             kit_cg_load(cg, wasm_cg_mem(c, b, t->results[0]));
           }
           break;
         }
         case WASM_INSN_REF_NULL:
           kit_cg_push_null(cg, rt.void_ptr_ty);
           break;
         case WASM_INSN_REF_FUNC:
           wasm_cg_push_instance_ptr(cg, &rt, instance_local);
           wasm_cg_ptr_add_offset(cg, b,
                                  rt.func_ref_entry_offset[(uint32_t)in.imm],
                                  rt.void_ptr_ty);
           break;
         case WASM_INSN_REF_IS_NULL:
           kit_cg_push_null(cg, rt.void_ptr_ty);
           kit_cg_int_cmp(cg, KIT_CG_INT_EQ);
           break;
         case WASM_INSN_CALL_REF:
         case WASM_INSN_RETURN_CALL_REF: {
           const WasmFuncType* t = &m->types[in.imm];
           KitCgLocalAttrs attrs;
           KitCgLocal callee_ref, callee, result = KIT_CG_LOCAL_NONE;
           KitCgLocal* args =
               t->nparams ? kit_arena_array(arena, KitCgLocal, t->nparams)
                          : NULL;
           KitCgLabel ok;
           KitCgMemAccess ref_mem = wasm_cg_mem_type(rt.void_ptr_ty);
           KitCgMemAccess i32_mem = wasm_cg_mem(c, b, WASM_VAL_I32);
           KitCgFuncParam* ref_params =
               kit_arena_zarray(arena, KitCgFuncParam, t->nparams + 1u);
           KitCgFuncSig ref_sig;
           KitCgTypeId ref_func_type;
           memset(&attrs, 0, sizeof attrs);
           attrs.flags = KIT_CG_LOCAL_COMPILER_TEMP;
           callee_ref = kit_cg_local(cg, rt.void_ptr_ty, attrs);
           callee = kit_cg_local(cg, rt.void_ptr_ty, attrs);
           kit_cg_push_local(cg, callee_ref);
           kit_cg_swap(cg);
           kit_cg_store(cg, ref_mem);
           for (uint32_t p = 0; p < t->nparams; ++p) {
             uint32_t param = t->nparams - 1u - p;
             args[param] =
                 kit_cg_local(cg, wasm_cg_type(c, b, t->params[param]), attrs);
             kit_cg_push_local(cg, args[param]);
             kit_cg_swap(cg);
             kit_cg_store(cg, wasm_cg_mem(c, b, t->params[param]));
           }
           ok = kit_cg_label_new(cg);
           kit_cg_push_local(cg, callee_ref);
           kit_cg_load(cg, ref_mem);
           kit_cg_push_null(cg, rt.void_ptr_ty);
           kit_cg_int_cmp(cg, KIT_CG_INT_NE);
           kit_cg_branch_true(cg, ok);
           wasm_cg_trap_table(cg, &rt);
           kit_cg_label_place(cg, ok);
 
           ok = kit_cg_label_new(cg);
           kit_cg_push_local(cg, callee_ref);
           kit_cg_load(cg, ref_mem);
           kit_cg_bitcast(cg, rt.table_entry_ptr_ty);
           {
             kit_cg_deref(cg, (int64_t)rt.table_entry_typeidx_offset);
             kit_cg_load(cg, i32_mem);
           }
           kit_cg_push_int(cg, (uint32_t)in.imm, b.id[KIT_CG_BUILTIN_I32]);
           kit_cg_int_cmp(cg, KIT_CG_INT_EQ);
           kit_cg_branch_true(cg, ok);
           wasm_cg_trap_signature(cg, &rt);
           kit_cg_label_place(cg, ok);
 
           kit_cg_push_local(cg, callee);
           kit_cg_push_local(cg, callee_ref);
           kit_cg_load(cg, ref_mem);
           kit_cg_bitcast(cg, rt.table_entry_ptr_ty);
           {
             kit_cg_deref(cg, (int64_t)rt.table_entry_fn_offset);
             kit_cg_load(cg, ref_mem);
           }
           kit_cg_store(cg, ref_mem);
 
           ref_params[0].type = rt.instance_ptr_ty;
           for (uint32_t p = 0; p < t->nparams; ++p)
             ref_params[p + 1u].type = wasm_cg_type(c, b, t->params[p]);
           memset(&ref_sig, 0, sizeof ref_sig);
           if (t->nresults) ref_sig.result.type = wasm_cg_type(c, b, t->results[0]);
           ref_sig.params = ref_params;
           ref_sig.nparams = t->nparams + 1u;
           ref_sig.call_conv = KIT_CG_CC_TARGET_C;
           ref_func_type = kit_cg_type_func(c, ref_sig);
           if (t->nresults)
             result = kit_cg_local(cg, wasm_cg_type(c, b, t->results[0]), attrs);
           kit_cg_push_local(cg, callee);
           kit_cg_load(cg, ref_mem);
           kit_cg_bitcast(cg, kit_cg_type_ptr(c, ref_func_type, 0));
           kit_cg_push_local(cg, instance_local);
           kit_cg_load(cg, wasm_cg_mem_type(rt.instance_ptr_ty));
           for (uint32_t p = 0; p < t->nparams; ++p) {
             kit_cg_push_local(cg, args[p]);
             kit_cg_load(cg, wasm_cg_mem(c, b, t->params[p]));
           }
           kit_cg_call(
               cg, t->nparams + 1u, ref_func_type,
               (KitCgCallAttrs){.tail = in.kind == WASM_INSN_RETURN_CALL_REF
                                            ? KIT_CG_TAIL_MUST
                                            : KIT_CG_TAIL_DEFAULT});
           if (in.kind == WASM_INSN_RETURN_CALL_REF) {
             if (t->nresults) wasm_cg_push_zero(c, cg, b, t->results[0]);
             kit_cg_unreachable(cg);
           } else if (t->nresults) {
             kit_cg_push_local(cg, result);
             kit_cg_swap(cg);
             kit_cg_store(cg, wasm_cg_mem(c, b, t->results[0]));
             kit_cg_push_local(cg, result);
             kit_cg_load(cg, wasm_cg_mem(c, b, t->results[0]));
           }
           break;
         }
         case WASM_INSN_GLOBAL_GET: {
           uint32_t index = (uint32_t)in.imm;
           wasm_cg_push_global_value_ptr(c, cg, b, &rt, instance_local, m,
                                         index);
           kit_cg_deref(cg, 0);
           kit_cg_load(cg, wasm_cg_mem(c, b, m->globals[index].type));
           break;
         }
         case WASM_INSN_GLOBAL_SET: {
           uint32_t index = (uint32_t)in.imm;
           wasm_cg_push_global_value_ptr(c, cg, b, &rt, instance_local, m,
                                         index);
           kit_cg_deref(cg, 0);
           kit_cg_swap(cg);
           kit_cg_store(cg, wasm_cg_mem(c, b, m->globals[index].type));
           break;
         }
         case WASM_INSN_RETURN:
           kit_cg_ret(cg);
           break;
         case WASM_INSN_DROP:
           kit_cg_drop(cg);
           break;
         case WASM_INSN_MEMORY_SIZE: {
           wasm_cg_push_instance_ptr(cg, &rt, instance_local);
           kit_cg_deref(cg, (int64_t)(rt.memory_offset[in.memidx] +
                                      rt.memory_pages_offset));
           kit_cg_load(cg, wasm_cg_mem(c, b, WASM_VAL_I64));
           if (!m->memories[in.memidx].is64)
             kit_cg_trunc(cg, b.id[KIT_CG_BUILTIN_I32]);
           break;
         }
         case WASM_INSN_MEMORY_GROW: {
           KitCgLocalAttrs attrs;
           KitCgLocal delta, old_pages, grow_result;
           KitCgLabel fail = kit_cg_label_new(cg);
           KitCgLabel done = kit_cg_label_new(cg);
           WasmValType page_vt =
               m->memories[in.memidx].is64 ? WASM_VAL_I64 : WASM_VAL_I32;
           KitCgTypeId page_ty = wasm_cg_type(c, b, page_vt);
           memset(&attrs, 0, sizeof attrs);
           attrs.flags = KIT_CG_LOCAL_COMPILER_TEMP;
           delta = kit_cg_local(cg, page_ty, attrs);
           old_pages = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
           grow_result = kit_cg_local(cg, page_ty, attrs);
           kit_cg_push_local(cg, delta);
           kit_cg_swap(cg);
           kit_cg_store(cg, wasm_cg_mem(c, b, page_vt));
           kit_cg_push_local(cg, old_pages);
           wasm_cg_push_instance_ptr(cg, &rt, instance_local);
           kit_cg_deref(cg, (int64_t)(rt.memory_offset[in.memidx] +
                                      rt.memory_pages_offset));
           kit_cg_load(cg, wasm_cg_mem(c, b, WASM_VAL_I64));
           kit_cg_store(cg, wasm_cg_mem(c, b, WASM_VAL_I64));
 
           kit_cg_push_local(cg, delta);
           kit_cg_load(cg, wasm_cg_mem(c, b, page_vt));
           if (!m->memories[in.memidx].is64)
             kit_cg_zext(cg, b.id[KIT_CG_BUILTIN_I64]);
           wasm_cg_push_instance_ptr(cg, &rt, instance_local);
           kit_cg_deref(cg, (int64_t)(rt.memory_offset[in.memidx] +
                                      rt.memory_max_pages_offset));
           kit_cg_load(cg, wasm_cg_mem(c, b, WASM_VAL_I64));
           kit_cg_push_local(cg, old_pages);
           kit_cg_load(cg, wasm_cg_mem(c, b, WASM_VAL_I64));
           kit_cg_int_binop(cg, KIT_CG_INT_SUB, 0);
           kit_cg_int_cmp(cg, KIT_CG_INT_LE_U);
           kit_cg_branch_false(cg, fail);
 
           wasm_cg_push_instance_ptr(cg, &rt, instance_local);
           kit_cg_deref(cg, (int64_t)(rt.memory_offset[in.memidx] +
                                      rt.memory_pages_offset));
           kit_cg_push_local(cg, old_pages);
           kit_cg_load(cg, wasm_cg_mem(c, b, WASM_VAL_I64));
           kit_cg_push_local(cg, delta);
           kit_cg_load(cg, wasm_cg_mem(c, b, page_vt));
           if (!m->memories[in.memidx].is64)
             kit_cg_zext(cg, b.id[KIT_CG_BUILTIN_I64]);
           kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
           kit_cg_store(cg, wasm_cg_mem(c, b, WASM_VAL_I64));
           kit_cg_push_local(cg, grow_result);
           kit_cg_push_local(cg, old_pages);
           kit_cg_load(cg, wasm_cg_mem(c, b, WASM_VAL_I64));
           if (!m->memories[in.memidx].is64)
             kit_cg_trunc(cg, b.id[KIT_CG_BUILTIN_I32]);
           kit_cg_store(cg, wasm_cg_mem(c, b, page_vt));
           kit_cg_jump(cg, done);
           kit_cg_label_place(cg, fail);
           kit_cg_push_local(cg, grow_result);
           kit_cg_push_int(cg, UINT64_MAX, page_ty);
           kit_cg_store(cg, wasm_cg_mem(c, b, page_vt));
           kit_cg_label_place(cg, done);
           kit_cg_push_local(cg, grow_result);
           kit_cg_load(cg, wasm_cg_mem(c, b, page_vt));
           break;
         }
         case WASM_INSN_MEMORY_COPY: {
           /* Stack on entry: [dst_addr, src_addr, n] (n at TOS). */
           uint32_t dst_memidx = in.memidx;
           uint32_t src_memidx = in.aux_idx;
           int dst_is64 = m->memories[dst_memidx].is64;
           int src_is64 = m->memories[src_memidx].is64;
           KitCgLocalAttrs attrs;
           KitCgLocal n_l, src_addr_l, dst_addr_l, src_base_l, dst_base_l;
           KitCgLocal src_size_l, dst_size_l;
           KitCgMemAccess i64_mem = wasm_cg_mem_type(b.id[KIT_CG_BUILTIN_I64]);
           KitCgMemAccess ptr_mem = wasm_cg_mem_type(rt.i8_ptr_ty);
           memset(&attrs, 0, sizeof attrs);
           attrs.flags = KIT_CG_LOCAL_COMPILER_TEMP;
           n_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
           src_addr_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
           dst_addr_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
           src_base_l = kit_cg_local(cg, rt.i8_ptr_ty, attrs);
           dst_base_l = kit_cg_local(cg, rt.i8_ptr_ty, attrs);
           src_size_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
           dst_size_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
           /* Pop n, src_addr, dst_addr into locals (zext if not memory64). */
           kit_cg_push_local(cg, n_l);
           kit_cg_swap(cg);
           if (!dst_is64) kit_cg_zext(cg, b.id[KIT_CG_BUILTIN_I64]);
           kit_cg_store(cg, i64_mem);
           kit_cg_push_local(cg, src_addr_l);
           kit_cg_swap(cg);
           if (!src_is64) kit_cg_zext(cg, b.id[KIT_CG_BUILTIN_I64]);
           kit_cg_store(cg, i64_mem);
           kit_cg_push_local(cg, dst_addr_l);
           kit_cg_swap(cg);
           if (!dst_is64) kit_cg_zext(cg, b.id[KIT_CG_BUILTIN_I64]);
           kit_cg_store(cg, i64_mem);
           /* Cache base and size for src and dst memories. */
           kit_cg_push_local(cg, src_base_l);
           wasm_cg_push_memory_data_ptr(cg, &rt, instance_local, src_memidx);
           kit_cg_store(cg, ptr_mem);
           kit_cg_push_local(cg, dst_base_l);
           wasm_cg_push_memory_data_ptr(cg, &rt, instance_local, dst_memidx);
           kit_cg_store(cg, ptr_mem);
           kit_cg_push_local(cg, src_size_l);
           wasm_cg_push_memory_pages_lvalue(cg, &rt, instance_local, src_memidx);
           kit_cg_load(cg, i64_mem);
           kit_cg_push_int(cg, 65536u, b.id[KIT_CG_BUILTIN_I64]);
           kit_cg_int_binop(cg, KIT_CG_INT_MUL, 0);
           kit_cg_store(cg, i64_mem);
           kit_cg_push_local(cg, dst_size_l);
           wasm_cg_push_memory_pages_lvalue(cg, &rt, instance_local, dst_memidx);
           kit_cg_load(cg, i64_mem);
           kit_cg_push_int(cg, 65536u, b.id[KIT_CG_BUILTIN_I64]);
           kit_cg_int_binop(cg, KIT_CG_INT_MUL, 0);
           kit_cg_store(cg, i64_mem);
           wasm_cg_bulk_bounds_check(cg, b, &rt, src_addr_l, n_l, src_size_l);
           wasm_cg_bulk_bounds_check(cg, b, &rt, dst_addr_l, n_l, dst_size_l);
           wasm_cg_emit_byte_copy_loop(cg, b, &rt, dst_base_l, src_base_l,
                                       dst_addr_l, src_addr_l, n_l);
           break;
         }
         case WASM_INSN_MEMORY_FILL: {
           /* Stack: [dst_addr, val, n] (n at TOS). */
           uint32_t memidx = in.memidx;
           int is64 = m->memories[memidx].is64;
           KitCgLocalAttrs attrs;
           KitCgLocal n_l, val_l, dst_addr_l, dst_base_l, dst_size_l;
           KitCgMemAccess i64_mem = wasm_cg_mem_type(b.id[KIT_CG_BUILTIN_I64]);
           KitCgMemAccess i32_mem = wasm_cg_mem_type(b.id[KIT_CG_BUILTIN_I32]);
           KitCgMemAccess ptr_mem = wasm_cg_mem_type(rt.i8_ptr_ty);
           memset(&attrs, 0, sizeof attrs);
           attrs.flags = KIT_CG_LOCAL_COMPILER_TEMP;
           n_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
           val_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I32], attrs);
           dst_addr_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
           dst_base_l = kit_cg_local(cg, rt.i8_ptr_ty, attrs);
           dst_size_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
           kit_cg_push_local(cg, n_l);
           kit_cg_swap(cg);
           if (!is64) kit_cg_zext(cg, b.id[KIT_CG_BUILTIN_I64]);
           kit_cg_store(cg, i64_mem);
           kit_cg_push_local(cg, val_l);
           kit_cg_swap(cg);
           kit_cg_store(cg, i32_mem);
           kit_cg_push_local(cg, dst_addr_l);
           kit_cg_swap(cg);
           if (!is64) kit_cg_zext(cg, b.id[KIT_CG_BUILTIN_I64]);
           kit_cg_store(cg, i64_mem);
           kit_cg_push_local(cg, dst_base_l);
           wasm_cg_push_memory_data_ptr(cg, &rt, instance_local, memidx);
           kit_cg_store(cg, ptr_mem);
           kit_cg_push_local(cg, dst_size_l);
           wasm_cg_push_memory_pages_lvalue(cg, &rt, instance_local, memidx);
           kit_cg_load(cg, i64_mem);
           kit_cg_push_int(cg, 65536u, b.id[KIT_CG_BUILTIN_I64]);
           kit_cg_int_binop(cg, KIT_CG_INT_MUL, 0);
           kit_cg_store(cg, i64_mem);
           wasm_cg_bulk_bounds_check(cg, b, &rt, dst_addr_l, n_l, dst_size_l);
           wasm_cg_emit_byte_fill_loop(cg, b, &rt, dst_base_l, dst_addr_l, val_l,
                                       n_l);
           break;
         }
         case WASM_INSN_MEMORY_INIT: {
           /* Stack: [dst_addr, src_addr, n]; in.imm = dataidx; in.memidx =
            * memidx */
           uint32_t dataidx = (uint32_t)in.imm;
           uint32_t memidx = in.memidx;
           int is64 = m->memories[memidx].is64;
           KitCgLocalAttrs attrs;
           KitCgLocal n_l, src_addr_l, dst_addr_l, src_base_l, dst_base_l;
           KitCgLocal src_size_l, dst_size_l;
           KitCgMemAccess i64_mem = wasm_cg_mem_type(b.id[KIT_CG_BUILTIN_I64]);
           KitCgMemAccess ptr_mem = wasm_cg_mem_type(rt.i8_ptr_ty);
           memset(&attrs, 0, sizeof attrs);
           attrs.flags = KIT_CG_LOCAL_COMPILER_TEMP;
           n_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
           src_addr_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
           dst_addr_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
           src_base_l = kit_cg_local(cg, rt.i8_ptr_ty, attrs);
           dst_base_l = kit_cg_local(cg, rt.i8_ptr_ty, attrs);
           src_size_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
           dst_size_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
           kit_cg_push_local(cg, n_l);
           kit_cg_swap(cg);
           kit_cg_zext(cg, b.id[KIT_CG_BUILTIN_I64]);
           kit_cg_store(cg, i64_mem);
           kit_cg_push_local(cg, src_addr_l);
           kit_cg_swap(cg);
           kit_cg_zext(cg, b.id[KIT_CG_BUILTIN_I64]);
           kit_cg_store(cg, i64_mem);
           kit_cg_push_local(cg, dst_addr_l);
           kit_cg_swap(cg);
           if (!is64) kit_cg_zext(cg, b.id[KIT_CG_BUILTIN_I64]);
           kit_cg_store(cg, i64_mem);
           /* src_base = inst->passive_data[dataidx].base */
           kit_cg_push_local(cg, src_base_l);
           wasm_cg_push_passive_data_lvalue(cg, &rt, instance_local, dataidx);
           kit_cg_field(cg, rt.passive_data_base_field);
           kit_cg_load(cg, ptr_mem);
           kit_cg_store(cg, ptr_mem);
           /* src_size = inst->passive_data[dataidx].len */
           kit_cg_push_local(cg, src_size_l);
           wasm_cg_push_passive_data_lvalue(cg, &rt, instance_local, dataidx);
           kit_cg_field(cg, rt.passive_data_len_field);
           kit_cg_load(cg, i64_mem);
           kit_cg_store(cg, i64_mem);
           kit_cg_push_local(cg, dst_base_l);
           wasm_cg_push_memory_data_ptr(cg, &rt, instance_local, memidx);
           kit_cg_store(cg, ptr_mem);
           kit_cg_push_local(cg, dst_size_l);
           wasm_cg_push_memory_pages_lvalue(cg, &rt, instance_local, memidx);
           kit_cg_load(cg, i64_mem);
           kit_cg_push_int(cg, 65536u, b.id[KIT_CG_BUILTIN_I64]);
           kit_cg_int_binop(cg, KIT_CG_INT_MUL, 0);
           kit_cg_store(cg, i64_mem);
           wasm_cg_bulk_bounds_check(cg, b, &rt, src_addr_l, n_l, src_size_l);
           wasm_cg_bulk_bounds_check(cg, b, &rt, dst_addr_l, n_l, dst_size_l);
           wasm_cg_emit_byte_copy_loop(cg, b, &rt, dst_base_l, src_base_l,
                                       dst_addr_l, src_addr_l, n_l);
           break;
         }
         case WASM_INSN_DATA_DROP: {
           uint32_t dataidx = (uint32_t)in.imm;
           wasm_cg_push_passive_data_lvalue(cg, &rt, instance_local, dataidx);
           kit_cg_field(cg, rt.passive_data_len_field);
           kit_cg_push_int(cg, 0, b.id[KIT_CG_BUILTIN_I64]);
           kit_cg_store(cg, wasm_cg_mem(c, b, WASM_VAL_I64));
           break;
         }
         case WASM_INSN_ELEM_DROP: {
           uint32_t elemidx = (uint32_t)in.imm;
           wasm_cg_push_passive_elem_lvalue(cg, &rt, instance_local, elemidx);
           kit_cg_field(cg, rt.passive_elem_length_field);
           kit_cg_push_int(cg, 0, b.id[KIT_CG_BUILTIN_I32]);
           kit_cg_store(cg, wasm_cg_mem(c, b, WASM_VAL_I32));
           break;
         }
         case WASM_INSN_TABLE_SIZE: {
           uint32_t tableidx = (uint32_t)in.imm;
           wasm_cg_push_table_lvalue(cg, &rt, instance_local, tableidx);
           kit_cg_field(cg, rt.table_len_field);
           kit_cg_load(cg, wasm_cg_mem(c, b, WASM_VAL_I32));
           break;
         }
         case WASM_INSN_TABLE_GROW: {
           /* Stack: [val, delta] -> [prev_size or -1 on failure].
            * Implementation: if current_len + delta <= max, write zeros into
            * new slots (we ignore val for funcref since slots are owned), then
            * bump len. We do honor val for funcref by writing the supplied
            * funcref into each new slot. */
           uint32_t tableidx = (uint32_t)in.imm;
           KitCgLocalAttrs attrs;
           KitCgLocal val_l, delta_l, old_len_l, result_l, idx_l;
           KitCgMemAccess i32_mem = wasm_cg_mem(c, b, WASM_VAL_I32);
           KitCgMemAccess fn_mem = wasm_cg_mem_type(rt.void_ptr_ty);
           KitCgLabel fail = kit_cg_label_new(cg);
           KitCgLabel done = kit_cg_label_new(cg);
           KitCgLabel fill_loop = kit_cg_label_new(cg);
           memset(&attrs, 0, sizeof attrs);
           attrs.flags = KIT_CG_LOCAL_COMPILER_TEMP;
           delta_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I32], attrs);
           val_l = kit_cg_local(cg, rt.void_ptr_ty, attrs);
           old_len_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I32], attrs);
           result_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I32], attrs);
           idx_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I32], attrs);
           kit_cg_push_local(cg, delta_l);
           kit_cg_swap(cg);
           kit_cg_store(cg, i32_mem);
           kit_cg_push_local(cg, val_l);
           kit_cg_swap(cg);
           kit_cg_store(cg, fn_mem);
           /* Cache val into fn/typeidx locals so the fill loop writes the
            * correct pair into each new slot. A null ref grows with null slots.
            */
           KitCgLocal val_fn_l = kit_cg_local(cg, rt.void_ptr_ty, attrs);
           KitCgLocal val_typeidx_l =
               kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I32], attrs);
           wasm_cg_cache_funcref_entry(c, cg, b, &rt, val_l, val_fn_l,
                                       val_typeidx_l, fn_mem, i32_mem);
           kit_cg_push_local(cg, old_len_l);
           wasm_cg_push_table_lvalue(cg, &rt, instance_local, tableidx);
           kit_cg_field(cg, rt.table_len_field);
           kit_cg_load(cg, i32_mem);
           kit_cg_store(cg, i32_mem);
           /* Check old_len + delta <= max. */
           wasm_cg_push_table_lvalue(cg, &rt, instance_local, tableidx);
           kit_cg_field(cg, rt.table_max_field);
           kit_cg_load(cg, i32_mem);
           kit_cg_push_local(cg, old_len_l);
           kit_cg_load(cg, i32_mem);
           kit_cg_int_cmp(cg, KIT_CG_INT_LT_U);
           kit_cg_branch_true(cg, fail);
           /* max - old_len >= delta */
           wasm_cg_push_table_lvalue(cg, &rt, instance_local, tableidx);
           kit_cg_field(cg, rt.table_max_field);
           kit_cg_load(cg, i32_mem);
           kit_cg_push_local(cg, old_len_l);
           kit_cg_load(cg, i32_mem);
           kit_cg_int_binop(cg, KIT_CG_INT_SUB, 0);
           kit_cg_push_local(cg, delta_l);
           kit_cg_load(cg, i32_mem);
           kit_cg_int_cmp(cg, KIT_CG_INT_LT_U);
           kit_cg_branch_true(cg, fail);
           /* Update length first. */
           wasm_cg_push_table_lvalue(cg, &rt, instance_local, tableidx);
           kit_cg_field(cg, rt.table_len_field);
           kit_cg_push_local(cg, old_len_l);
           kit_cg_load(cg, i32_mem);
           kit_cg_push_local(cg, delta_l);
           kit_cg_load(cg, i32_mem);
           kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
           kit_cg_store(cg, i32_mem);
           /* Initialize new slots [old_len, old_len+delta) to val. */
           kit_cg_push_local(cg, idx_l);
           kit_cg_push_local(cg, old_len_l);
           kit_cg_load(cg, i32_mem);
           kit_cg_store(cg, i32_mem);
           kit_cg_label_place(cg, fill_loop);
           kit_cg_push_local(cg, idx_l);
           kit_cg_load(cg, i32_mem);
           kit_cg_push_local(cg, old_len_l);
           kit_cg_load(cg, i32_mem);
           kit_cg_push_local(cg, delta_l);
           kit_cg_load(cg, i32_mem);
           kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
           kit_cg_int_cmp(cg, KIT_CG_INT_GE_U);
           {
             KitCgLabel fill_end = kit_cg_label_new(cg);
             kit_cg_branch_true(cg, fill_end);
             wasm_cg_push_table_entry_lvalue(cg, &rt, instance_local, tableidx,
                                             idx_l, i32_mem);
             kit_cg_field(cg, rt.table_entry_fn_field);
             kit_cg_push_local(cg, val_fn_l);
             kit_cg_load(cg, fn_mem);
             kit_cg_store(cg, fn_mem);
             wasm_cg_push_table_entry_lvalue(cg, &rt, instance_local, tableidx,
                                             idx_l, i32_mem);
             kit_cg_field(cg, rt.table_entry_typeidx_field);
             kit_cg_push_local(cg, val_typeidx_l);
             kit_cg_load(cg, i32_mem);
             kit_cg_store(cg, i32_mem);
             kit_cg_push_local(cg, idx_l);
             kit_cg_push_local(cg, idx_l);
             kit_cg_load(cg, i32_mem);
             kit_cg_push_int(cg, 1, b.id[KIT_CG_BUILTIN_I32]);
             kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
             kit_cg_store(cg, i32_mem);
             kit_cg_jump(cg, fill_loop);
             kit_cg_label_place(cg, fill_end);
           }
           kit_cg_push_local(cg, result_l);
           kit_cg_push_local(cg, old_len_l);
           kit_cg_load(cg, i32_mem);
           kit_cg_store(cg, i32_mem);
           kit_cg_jump(cg, done);
           kit_cg_label_place(cg, fail);
           kit_cg_push_local(cg, result_l);
           kit_cg_push_int(cg, UINT32_C(0xffffffff), b.id[KIT_CG_BUILTIN_I32]);
           kit_cg_store(cg, i32_mem);
           kit_cg_label_place(cg, done);
           kit_cg_push_local(cg, result_l);
           kit_cg_load(cg, i32_mem);
           break;
         }
         case WASM_INSN_TABLE_FILL: {
           /* Stack: [dst_idx, val, n]. `val` is a `KitWasmTableEntry*`
            * pointing at an instance-resident funcref slot (produced by
            * ref.func). To populate table slots correctly, we dereference
            * `val` once outside the loop into local fn/typeidx pairs and
            * then copy those into each slot. */
           uint32_t tableidx = (uint32_t)in.imm;
           KitCgLocalAttrs attrs;
           KitCgLocal n_l, val_l, dst_idx_l, len_l, idx_l;
           KitCgLocal val_fn_l, val_typeidx_l;
           KitCgMemAccess i32_mem = wasm_cg_mem(c, b, WASM_VAL_I32);
           KitCgMemAccess fn_mem = wasm_cg_mem_type(rt.void_ptr_ty);
           KitCgLabel loop = kit_cg_label_new(cg);
           KitCgLabel done = kit_cg_label_new(cg);
           KitCgLabel ok = kit_cg_label_new(cg);
           memset(&attrs, 0, sizeof attrs);
           attrs.flags = KIT_CG_LOCAL_COMPILER_TEMP;
           n_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I32], attrs);
           val_l = kit_cg_local(cg, rt.void_ptr_ty, attrs);
           dst_idx_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I32], attrs);
           len_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I32], attrs);
           idx_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I32], attrs);
           val_fn_l = kit_cg_local(cg, rt.void_ptr_ty, attrs);
           val_typeidx_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I32], attrs);
           kit_cg_push_local(cg, n_l);
           kit_cg_swap(cg);
           kit_cg_store(cg, i32_mem);
           kit_cg_push_local(cg, val_l);
           kit_cg_swap(cg);
           kit_cg_store(cg, fn_mem);
           kit_cg_push_local(cg, dst_idx_l);
           kit_cg_swap(cg);
           kit_cg_store(cg, i32_mem);
           /* Cache val into (val_fn, val_typeidx). Null refs fill with null
            * entries and are checked later by call_indirect/call_ref. */
           wasm_cg_cache_funcref_entry(c, cg, b, &rt, val_l, val_fn_l,
                                       val_typeidx_l, fn_mem, i32_mem);
           /* Bounds check: dst_idx + n <= len. */
           kit_cg_push_local(cg, len_l);
           wasm_cg_push_table_lvalue(cg, &rt, instance_local, tableidx);
           kit_cg_field(cg, rt.table_len_field);
           kit_cg_load(cg, i32_mem);
           kit_cg_store(cg, i32_mem);
           kit_cg_push_local(cg, len_l);
           kit_cg_load(cg, i32_mem);
           kit_cg_push_local(cg, dst_idx_l);
           kit_cg_load(cg, i32_mem);
           kit_cg_int_cmp(cg, KIT_CG_INT_GE_U);
           kit_cg_branch_true(cg, ok);
           wasm_cg_trap_bounds(cg, &rt);
           kit_cg_label_place(cg, ok);
           {
             KitCgLabel ok2 = kit_cg_label_new(cg);
             kit_cg_push_local(cg, len_l);
             kit_cg_load(cg, i32_mem);
             kit_cg_push_local(cg, dst_idx_l);
             kit_cg_load(cg, i32_mem);
             kit_cg_int_binop(cg, KIT_CG_INT_SUB, 0);
             kit_cg_push_local(cg, n_l);
             kit_cg_load(cg, i32_mem);
             kit_cg_int_cmp(cg, KIT_CG_INT_GE_U);
             kit_cg_branch_true(cg, ok2);
             wasm_cg_trap_bounds(cg, &rt);
             kit_cg_label_place(cg, ok2);
           }
           /* idx = 0; while idx < n: table[dst_idx+idx] = val; idx++ */
           kit_cg_push_local(cg, idx_l);
           kit_cg_push_int(cg, 0, b.id[KIT_CG_BUILTIN_I32]);
           kit_cg_store(cg, i32_mem);
           kit_cg_label_place(cg, loop);
           kit_cg_push_local(cg, idx_l);
           kit_cg_load(cg, i32_mem);
           kit_cg_push_local(cg, n_l);
           kit_cg_load(cg, i32_mem);
           kit_cg_int_cmp(cg, KIT_CG_INT_GE_U);
           kit_cg_branch_true(cg, done);
           {
             /* Compute slot = dst_idx + idx, store into a temp local for
              * indexing. */
             KitCgLocal slot_l =
                 kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I32], attrs);
             kit_cg_push_local(cg, slot_l);
             kit_cg_push_local(cg, dst_idx_l);
             kit_cg_load(cg, i32_mem);
             kit_cg_push_local(cg, idx_l);
             kit_cg_load(cg, i32_mem);
             kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
             kit_cg_store(cg, i32_mem);
             wasm_cg_push_table_entry_lvalue(cg, &rt, instance_local, tableidx,
                                             slot_l, i32_mem);
             kit_cg_field(cg, rt.table_entry_fn_field);
             kit_cg_push_local(cg, val_fn_l);
             kit_cg_load(cg, fn_mem);
             kit_cg_store(cg, fn_mem);
             wasm_cg_push_table_entry_lvalue(cg, &rt, instance_local, tableidx,
                                             slot_l, i32_mem);
             kit_cg_field(cg, rt.table_entry_typeidx_field);
             kit_cg_push_local(cg, val_typeidx_l);
             kit_cg_load(cg, i32_mem);
             kit_cg_store(cg, i32_mem);
           }
           kit_cg_push_local(cg, idx_l);
           kit_cg_push_local(cg, idx_l);
           kit_cg_load(cg, i32_mem);
           kit_cg_push_int(cg, 1, b.id[KIT_CG_BUILTIN_I32]);
           kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
           kit_cg_store(cg, i32_mem);
           kit_cg_jump(cg, loop);
           kit_cg_label_place(cg, done);
           break;
         }
         case WASM_INSN_TABLE_COPY: {
           /* Stack: [dst_idx, src_idx, n]; in.imm = dst_tableidx; in.aux_idx =
            * src */
           uint32_t dst_tbl = (uint32_t)in.imm;
           uint32_t src_tbl = in.aux_idx;
           KitCgLocalAttrs attrs;
           KitCgLocal n_l, src_idx_l, dst_idx_l, src_base_l, dst_base_l;
           KitCgLocal src_len_l, dst_len_l;
           KitCgMemAccess i64_mem = wasm_cg_mem_type(b.id[KIT_CG_BUILTIN_I64]);
           KitCgMemAccess i32_mem = wasm_cg_mem(c, b, WASM_VAL_I32);
           KitCgMemAccess ptr_mem = wasm_cg_mem_type(rt.table_entry_ptr_ty);
           memset(&attrs, 0, sizeof attrs);
           attrs.flags = KIT_CG_LOCAL_COMPILER_TEMP;
           n_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
           src_idx_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
           dst_idx_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
           src_base_l = kit_cg_local(cg, rt.table_entry_ptr_ty, attrs);
           dst_base_l = kit_cg_local(cg, rt.table_entry_ptr_ty, attrs);
           src_len_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
           dst_len_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
           kit_cg_push_local(cg, n_l);
           kit_cg_swap(cg);
           kit_cg_zext(cg, b.id[KIT_CG_BUILTIN_I64]);
           kit_cg_store(cg, i64_mem);
           kit_cg_push_local(cg, src_idx_l);
           kit_cg_swap(cg);
           kit_cg_zext(cg, b.id[KIT_CG_BUILTIN_I64]);
           kit_cg_store(cg, i64_mem);
           kit_cg_push_local(cg, dst_idx_l);
           kit_cg_swap(cg);
           kit_cg_zext(cg, b.id[KIT_CG_BUILTIN_I64]);
           kit_cg_store(cg, i64_mem);
           kit_cg_push_local(cg, src_base_l);
           wasm_cg_push_table_lvalue(cg, &rt, instance_local, src_tbl);
           kit_cg_field(cg, rt.table_entries_ptr_field);
           kit_cg_load(cg, ptr_mem);
           kit_cg_store(cg, ptr_mem);
           kit_cg_push_local(cg, dst_base_l);
           wasm_cg_push_table_lvalue(cg, &rt, instance_local, dst_tbl);
           kit_cg_field(cg, rt.table_entries_ptr_field);
           kit_cg_load(cg, ptr_mem);
           kit_cg_store(cg, ptr_mem);
           kit_cg_push_local(cg, src_len_l);
           wasm_cg_push_table_lvalue(cg, &rt, instance_local, src_tbl);
           kit_cg_field(cg, rt.table_len_field);
           kit_cg_load(cg, i32_mem);
           kit_cg_zext(cg, b.id[KIT_CG_BUILTIN_I64]);
           kit_cg_store(cg, i64_mem);
           kit_cg_push_local(cg, dst_len_l);
           wasm_cg_push_table_lvalue(cg, &rt, instance_local, dst_tbl);
           kit_cg_field(cg, rt.table_len_field);
           kit_cg_load(cg, i32_mem);
           kit_cg_zext(cg, b.id[KIT_CG_BUILTIN_I64]);
           kit_cg_store(cg, i64_mem);
           wasm_cg_bulk_bounds_check(cg, b, &rt, src_idx_l, n_l, src_len_l);
           wasm_cg_bulk_bounds_check(cg, b, &rt, dst_idx_l, n_l, dst_len_l);
           wasm_cg_emit_table_copy_loop(c, cg, b, &rt, dst_base_l, src_base_l,
                                        dst_idx_l, src_idx_l, n_l);
           break;
         }
         case WASM_INSN_TABLE_INIT: {
           /* Stack: [dst_idx, src_idx, n]; in.imm = elemidx; in.aux_idx =
            * tableidx */
           uint32_t elemidx = (uint32_t)in.imm;
           uint32_t tableidx = in.aux_idx;
           KitCgLocalAttrs attrs;
           KitCgLocal n_l, src_idx_l, dst_idx_l, src_base_l, dst_base_l;
           KitCgLocal src_len_l, dst_len_l;
           KitCgMemAccess i64_mem = wasm_cg_mem_type(b.id[KIT_CG_BUILTIN_I64]);
           KitCgMemAccess i32_mem = wasm_cg_mem(c, b, WASM_VAL_I32);
           KitCgMemAccess ptr_mem = wasm_cg_mem_type(rt.table_entry_ptr_ty);
           memset(&attrs, 0, sizeof attrs);
           attrs.flags = KIT_CG_LOCAL_COMPILER_TEMP;
           n_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
           src_idx_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
           dst_idx_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
           src_base_l = kit_cg_local(cg, rt.table_entry_ptr_ty, attrs);
           dst_base_l = kit_cg_local(cg, rt.table_entry_ptr_ty, attrs);
           src_len_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
           dst_len_l = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
           kit_cg_push_local(cg, n_l);
           kit_cg_swap(cg);
           kit_cg_zext(cg, b.id[KIT_CG_BUILTIN_I64]);
           kit_cg_store(cg, i64_mem);
           kit_cg_push_local(cg, src_idx_l);
           kit_cg_swap(cg);
           kit_cg_zext(cg, b.id[KIT_CG_BUILTIN_I64]);
           kit_cg_store(cg, i64_mem);
           kit_cg_push_local(cg, dst_idx_l);
           kit_cg_swap(cg);
           kit_cg_zext(cg, b.id[KIT_CG_BUILTIN_I64]);
           kit_cg_store(cg, i64_mem);
           kit_cg_push_local(cg, src_base_l);
           wasm_cg_push_passive_elem_lvalue(cg, &rt, instance_local, elemidx);
           kit_cg_field(cg, rt.passive_elem_entries_field);
           kit_cg_load(cg, ptr_mem);
           kit_cg_store(cg, ptr_mem);
           kit_cg_push_local(cg, dst_base_l);
           wasm_cg_push_table_lvalue(cg, &rt, instance_local, tableidx);
           kit_cg_field(cg, rt.table_entries_ptr_field);
           kit_cg_load(cg, ptr_mem);
           kit_cg_store(cg, ptr_mem);
           kit_cg_push_local(cg, src_len_l);
           wasm_cg_push_passive_elem_lvalue(cg, &rt, instance_local, elemidx);
           kit_cg_field(cg, rt.passive_elem_length_field);
           kit_cg_load(cg, i32_mem);
           kit_cg_zext(cg, b.id[KIT_CG_BUILTIN_I64]);
           kit_cg_store(cg, i64_mem);
           kit_cg_push_local(cg, dst_len_l);
           wasm_cg_push_table_lvalue(cg, &rt, instance_local, tableidx);
           kit_cg_field(cg, rt.table_len_field);
           kit_cg_load(cg, i32_mem);
           kit_cg_zext(cg, b.id[KIT_CG_BUILTIN_I64]);
           kit_cg_store(cg, i64_mem);
           wasm_cg_bulk_bounds_check(cg, b, &rt, src_idx_l, n_l, src_len_l);
           wasm_cg_bulk_bounds_check(cg, b, &rt, dst_idx_l, n_l, dst_len_l);
           wasm_cg_emit_table_copy_loop(c, cg, b, &rt, dst_base_l, src_base_l,
                                        dst_idx_l, src_idx_l, n_l);
           break;
         }
         case WASM_INSN_I32_TRUNC_SAT_F32_S:
         case WASM_INSN_I32_TRUNC_SAT_F64_S:
           kit_cg_float_to_sint(cg, b.id[KIT_CG_BUILTIN_I32],
                                KIT_CG_ROUND_TOWARD_ZERO);
           break;
         case WASM_INSN_I32_TRUNC_SAT_F32_U:
         case WASM_INSN_I32_TRUNC_SAT_F64_U:
           kit_cg_float_to_uint(cg, b.id[KIT_CG_BUILTIN_I32],
                                KIT_CG_ROUND_TOWARD_ZERO);
           break;
         case WASM_INSN_I64_TRUNC_SAT_F32_S:
         case WASM_INSN_I64_TRUNC_SAT_F64_S:
           kit_cg_float_to_sint(cg, b.id[KIT_CG_BUILTIN_I64],
                                KIT_CG_ROUND_TOWARD_ZERO);
           break;
         case WASM_INSN_I64_TRUNC_SAT_F32_U:
         case WASM_INSN_I64_TRUNC_SAT_F64_U:
           kit_cg_float_to_uint(cg, b.id[KIT_CG_BUILTIN_I64],
                                KIT_CG_ROUND_TOWARD_ZERO);
           break;
         case WASM_INSN_ATOMIC_FENCE:
           kit_cg_atomic_fence(cg, KIT_CG_MO_SEQ_CST);
           break;
         case WASM_INSN_I32_ATOMIC_LOAD:
         case WASM_INSN_I64_ATOMIC_LOAD:
         case WASM_INSN_I32_ATOMIC_LOAD8_U:
         case WASM_INSN_I32_ATOMIC_LOAD16_U:
         case WASM_INSN_I64_ATOMIC_LOAD8_U:
         case WASM_INSN_I64_ATOMIC_LOAD16_U:
         case WASM_INSN_I64_ATOMIC_LOAD32_U: {
           KitCgTypeId ty = wasm_cg_type(c, b, wasm_atomic_value_type(in.kind));
           KitCgMemAccess mem = wasm_cg_mem_type(ty);
           mem.align = in.align;
           wasm_cg_memory_check(c, cg, b, m, &rt, instance_local, &in);
           wasm_cg_memory_addr_from_tos(cg, b, &rt, m, instance_local, in.memidx,
                                        in.offset64);
           kit_cg_atomic_load(cg, mem, KIT_CG_MO_SEQ_CST);
           break;
         }
         case WASM_INSN_I32_ATOMIC_STORE:
         case WASM_INSN_I64_ATOMIC_STORE:
         case WASM_INSN_I32_ATOMIC_STORE8:
         case WASM_INSN_I32_ATOMIC_STORE16:
         case WASM_INSN_I64_ATOMIC_STORE8:
         case WASM_INSN_I64_ATOMIC_STORE16:
         case WASM_INSN_I64_ATOMIC_STORE32: {
           WasmValType vt = wasm_atomic_value_type(in.kind);
           KitCgTypeId ty = wasm_cg_type(c, b, vt);
           KitCgMemAccess mem = wasm_cg_mem_type(ty);
           KitCgLocalAttrs attrs;
           KitCgLocal value_tmp;
           memset(&attrs, 0, sizeof attrs);
           attrs.flags = KIT_CG_LOCAL_COMPILER_TEMP;
           mem.align = in.align;
           value_tmp = kit_cg_local(cg, ty, attrs);
           kit_cg_push_local(cg, value_tmp);
           kit_cg_swap(cg);
           kit_cg_store(cg, mem);
           wasm_cg_memory_check(c, cg, b, m, &rt, instance_local, &in);
           wasm_cg_memory_addr_from_tos(cg, b, &rt, m, instance_local, in.memidx,
                                        in.offset64);
           kit_cg_push_local(cg, value_tmp);
           kit_cg_load(cg, mem);
           kit_cg_atomic_store(cg, mem, KIT_CG_MO_SEQ_CST);
           break;
         }
         case WASM_INSN_I32_ATOMIC_RMW_ADD:
         case WASM_INSN_I64_ATOMIC_RMW_ADD:
         case WASM_INSN_I32_ATOMIC_RMW_SUB:
         case WASM_INSN_I64_ATOMIC_RMW_SUB:
         case WASM_INSN_I32_ATOMIC_RMW_AND:
         case WASM_INSN_I64_ATOMIC_RMW_AND:
         case WASM_INSN_I32_ATOMIC_RMW_OR:
         case WASM_INSN_I64_ATOMIC_RMW_OR:
         case WASM_INSN_I32_ATOMIC_RMW_XOR:
         case WASM_INSN_I64_ATOMIC_RMW_XOR:
         case WASM_INSN_I32_ATOMIC_RMW_XCHG:
         case WASM_INSN_I64_ATOMIC_RMW_XCHG: {
           WasmValType vt = wasm_atomic_value_type(in.kind);
           KitCgTypeId ty = wasm_cg_type(c, b, vt);
           KitCgMemAccess mem = wasm_cg_mem_type(ty);
           KitCgLocalAttrs attrs;
           KitCgLocal value_tmp;
           memset(&attrs, 0, sizeof attrs);
           attrs.flags = KIT_CG_LOCAL_COMPILER_TEMP;
           mem.align = in.align;
           value_tmp = kit_cg_local(cg, ty, attrs);
           kit_cg_push_local(cg, value_tmp);
           kit_cg_swap(cg);
           kit_cg_store(cg, mem);
           wasm_cg_memory_check(c, cg, b, m, &rt, instance_local, &in);
           wasm_cg_memory_addr_from_tos(cg, b, &rt, m, instance_local, in.memidx,
                                        in.offset64);
           kit_cg_push_local(cg, value_tmp);
           kit_cg_load(cg, mem);
           kit_cg_atomic_rmw(cg, mem, wasm_atomic_rmw_op(in.kind),
                             KIT_CG_MO_SEQ_CST);
           break;
         }
         case WASM_INSN_I32_ATOMIC_RMW_CMPXCHG:
         case WASM_INSN_I64_ATOMIC_RMW_CMPXCHG: {
           WasmValType vt = wasm_atomic_value_type(in.kind);
           KitCgTypeId ty = wasm_cg_type(c, b, vt);
           KitCgMemAccess mem = wasm_cg_mem_type(ty);
           KitCgLocalAttrs attrs;
           KitCgLocal expected_tmp, desired_tmp;
           memset(&attrs, 0, sizeof attrs);
           attrs.flags = KIT_CG_LOCAL_COMPILER_TEMP;
           mem.align = in.align;
           desired_tmp = kit_cg_local(cg, ty, attrs);
           expected_tmp = kit_cg_local(cg, ty, attrs);
           kit_cg_push_local(cg, desired_tmp);
           kit_cg_swap(cg);
           kit_cg_store(cg, mem);
           kit_cg_push_local(cg, expected_tmp);
           kit_cg_swap(cg);
           kit_cg_store(cg, mem);
           wasm_cg_memory_check(c, cg, b, m, &rt, instance_local, &in);
           wasm_cg_memory_addr_from_tos(cg, b, &rt, m, instance_local, in.memidx,
                                        in.offset64);
           kit_cg_push_local(cg, expected_tmp);
           kit_cg_load(cg, mem);
           kit_cg_push_local(cg, desired_tmp);
           kit_cg_load(cg, mem);
           kit_cg_atomic_cmpxchg(cg, mem, KIT_CG_MO_SEQ_CST, KIT_CG_MO_SEQ_CST,
                                 0);
           kit_cg_drop(cg);
           break;
         }
         case WASM_INSN_I32_ATOMIC_WAIT:
         case WASM_INSN_I64_ATOMIC_WAIT: {
           WasmValType vt = wasm_atomic_value_type(in.kind);
           KitCgTypeId ty = wasm_cg_type(c, b, vt);
           KitCgMemAccess mem = wasm_cg_mem_type(ty);
           KitCgLocalAttrs attrs;
           KitCgLocal timeout_tmp, expected_tmp, result_tmp;
           KitCgLabel equal = kit_cg_label_new(cg);
           KitCgLabel done = kit_cg_label_new(cg);
           memset(&attrs, 0, sizeof attrs);
           attrs.flags = KIT_CG_LOCAL_COMPILER_TEMP;
           mem.align = in.align;
           timeout_tmp = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I64], attrs);
           expected_tmp = kit_cg_local(cg, ty, attrs);
           result_tmp = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I32], attrs);
           kit_cg_push_local(cg, timeout_tmp);
           kit_cg_swap(cg);
           kit_cg_store(cg, wasm_cg_mem(c, b, WASM_VAL_I64));
           kit_cg_push_local(cg, expected_tmp);
           kit_cg_swap(cg);
           kit_cg_store(cg, mem);
           wasm_cg_memory_check(c, cg, b, m, &rt, instance_local, &in);
           wasm_cg_memory_addr_from_tos(cg, b, &rt, m, instance_local, in.memidx,
                                        in.offset64);
           kit_cg_atomic_load(cg, mem, KIT_CG_MO_SEQ_CST);
           kit_cg_push_local(cg, expected_tmp);
           kit_cg_load(cg, mem);
           kit_cg_int_cmp(cg, KIT_CG_INT_EQ);
           kit_cg_branch_true(cg, equal);
           kit_cg_push_local(cg, result_tmp);
           kit_cg_push_int(cg, 1, b.id[KIT_CG_BUILTIN_I32]);
           kit_cg_store(cg, wasm_cg_mem(c, b, WASM_VAL_I32));
           kit_cg_jump(cg, done);
           kit_cg_label_place(cg, equal);
           (void)timeout_tmp;
           kit_cg_push_local(cg, result_tmp);
           kit_cg_push_int(cg, 2, b.id[KIT_CG_BUILTIN_I32]);
           kit_cg_store(cg, wasm_cg_mem(c, b, WASM_VAL_I32));
           kit_cg_label_place(cg, done);
           kit_cg_push_local(cg, result_tmp);
           kit_cg_load(cg, wasm_cg_mem(c, b, WASM_VAL_I32));
           break;
         }
         case WASM_INSN_MEMORY_ATOMIC_NOTIFY: {
           KitCgMemAccess i32_mem = wasm_cg_mem(c, b, WASM_VAL_I32);
           KitCgLocalAttrs attrs;
           KitCgLocal count_tmp;
           memset(&attrs, 0, sizeof attrs);
           attrs.flags = KIT_CG_LOCAL_COMPILER_TEMP;
           count_tmp = kit_cg_local(cg, b.id[KIT_CG_BUILTIN_I32], attrs);
           kit_cg_push_local(cg, count_tmp);
           kit_cg_swap(cg);
           kit_cg_store(cg, i32_mem);
           wasm_cg_memory_check(c, cg, b, m, &rt, instance_local, &in);
           kit_cg_drop(cg);
           kit_cg_push_int(cg, 0, b.id[KIT_CG_BUILTIN_I32]);
           break;
         }
         case WASM_INSN_I32_LOAD:
         case WASM_INSN_I64_LOAD:
         case WASM_INSN_I32_LOAD8_S:
         case WASM_INSN_I32_LOAD8_U:
         case WASM_INSN_I32_LOAD16_S:
         case WASM_INSN_I32_LOAD16_U:
         case WASM_INSN_I64_LOAD8_S:
         case WASM_INSN_I64_LOAD8_U:
         case WASM_INSN_I64_LOAD16_S:
         case WASM_INSN_I64_LOAD16_U:
         case WASM_INSN_I64_LOAD32_S:
         case WASM_INSN_I64_LOAD32_U:
         case WASM_INSN_F32_LOAD:
         case WASM_INSN_F64_LOAD: {
           KitCgTypeId storage = wasm_load_storage_type(b, in.kind);
           KitCgTypeId result =
               wasm_cg_type(c, b, wasm_load_result_type(in.kind));
           KitCgMemAccess mem;
           memset(&mem, 0, sizeof mem);
           mem.type = storage;
           mem.align = in.align;
           wasm_cg_memory_check(c, cg, b, m, &rt, instance_local, &in);
           wasm_cg_memory_addr_from_tos(cg, b, &rt, m, instance_local, in.memidx,
                                        in.offset64);
           kit_cg_deref(cg, 0);
           kit_cg_load(cg, mem);
           if (storage != result) {
             if (in.kind == WASM_INSN_I32_LOAD8_S ||
                 in.kind == WASM_INSN_I32_LOAD16_S ||
                 in.kind == WASM_INSN_I64_LOAD8_S ||
                 in.kind == WASM_INSN_I64_LOAD16_S ||
                 in.kind == WASM_INSN_I64_LOAD32_S)
               kit_cg_sext(cg, result);
             else
               kit_cg_zext(cg, result);
           }
           break;
         }
         case WASM_INSN_I32_STORE:
         case WASM_INSN_I64_STORE:
         case WASM_INSN_I32_STORE8:
         case WASM_INSN_I32_STORE16:
         case WASM_INSN_I64_STORE8:
         case WASM_INSN_I64_STORE16:
         case WASM_INSN_I64_STORE32:
         case WASM_INSN_F32_STORE:
         case WASM_INSN_F64_STORE: {
           KitCgTypeId storage = wasm_store_storage_type(b, in.kind);
           KitCgTypeId value_type =
               wasm_cg_type(c, b, wasm_store_value_type(in.kind));
           KitCgMemAccess mem;
           KitCgLocalAttrs attrs;
           KitCgLocal addr_tmp, value_tmp;
           WasmValType addr_vt =
               m->memories[in.memidx].is64 ? WASM_VAL_I64 : WASM_VAL_I32;
           memset(&mem, 0, sizeof mem);
           mem.type = storage;
           mem.align = in.align;
           if (storage != value_type) kit_cg_trunc(cg, storage);
           memset(&attrs, 0, sizeof attrs);
           attrs.flags = KIT_CG_LOCAL_COMPILER_TEMP;
           value_tmp = kit_cg_local(cg, storage, attrs);
           addr_tmp = kit_cg_local(cg, wasm_cg_type(c, b, addr_vt), attrs);
           kit_cg_push_local(cg, value_tmp);
           kit_cg_swap(cg);
           kit_cg_store(cg, mem);
           wasm_cg_memory_check(c, cg, b, m, &rt, instance_local, &in);
           kit_cg_push_local(cg, addr_tmp);
           kit_cg_swap(cg);
           kit_cg_store(cg, wasm_cg_mem(c, b, addr_vt));
           kit_cg_push_local(cg, addr_tmp);
           kit_cg_load(cg, wasm_cg_mem(c, b, addr_vt));
           wasm_cg_memory_addr_from_tos(cg, b, &rt, m, instance_local, in.memidx,
                                        in.offset64);
           kit_cg_deref(cg, 0);
           kit_cg_push_local(cg, value_tmp);
           kit_cg_load(cg, mem);
           kit_cg_store(cg, mem);
           break;
         }
         case WASM_INSN_I32_ADD:
         case WASM_INSN_I64_ADD:
           kit_cg_int_binop(cg, KIT_CG_INT_ADD, 0);
           break;
         case WASM_INSN_I32_SUB:
         case WASM_INSN_I64_SUB:
           kit_cg_int_binop(cg, KIT_CG_INT_SUB, 0);
           break;
         case WASM_INSN_I32_MUL:
         case WASM_INSN_I64_MUL:
           kit_cg_int_binop(cg, KIT_CG_INT_MUL, 0);
           break;
         case WASM_INSN_I32_DIV_S:
           wasm_cg_checked_divrem(c, cg, b, &rt, WASM_VAL_I32, KIT_CG_INT_SDIV);
           break;
         case WASM_INSN_I32_DIV_U:
           wasm_cg_checked_divrem(c, cg, b, &rt, WASM_VAL_I32, KIT_CG_INT_UDIV);
           break;
         case WASM_INSN_I32_REM_S:
           wasm_cg_checked_divrem(c, cg, b, &rt, WASM_VAL_I32, KIT_CG_INT_SREM);
           break;
         case WASM_INSN_I32_REM_U:
           wasm_cg_checked_divrem(c, cg, b, &rt, WASM_VAL_I32, KIT_CG_INT_UREM);
           break;
         case WASM_INSN_I64_DIV_S:
           wasm_cg_checked_divrem(c, cg, b, &rt, WASM_VAL_I64, KIT_CG_INT_SDIV);
           break;
         case WASM_INSN_I64_DIV_U:
           wasm_cg_checked_divrem(c, cg, b, &rt, WASM_VAL_I64, KIT_CG_INT_UDIV);
           break;
         case WASM_INSN_I64_REM_S:
           wasm_cg_checked_divrem(c, cg, b, &rt, WASM_VAL_I64, KIT_CG_INT_SREM);
           break;
         case WASM_INSN_I64_REM_U:
           wasm_cg_checked_divrem(c, cg, b, &rt, WASM_VAL_I64, KIT_CG_INT_UREM);
           break;
         case WASM_INSN_I32_AND:
         case WASM_INSN_I64_AND:
           kit_cg_int_binop(cg, KIT_CG_INT_AND, 0);
           break;
         case WASM_INSN_I32_OR:
         case WASM_INSN_I64_OR:
           kit_cg_int_binop(cg, KIT_CG_INT_OR, 0);
           break;
         case WASM_INSN_I32_XOR:
         case WASM_INSN_I64_XOR:
           kit_cg_int_binop(cg, KIT_CG_INT_XOR, 0);
           break;
         case WASM_INSN_I32_SHL:
         case WASM_INSN_I64_SHL:
           kit_cg_int_binop(cg, KIT_CG_INT_SHL, 0);
           break;
         case WASM_INSN_I32_SHR_S:
         case WASM_INSN_I64_SHR_S:
           kit_cg_int_binop(cg, KIT_CG_INT_ASHR, 0);
           break;
         case WASM_INSN_I32_SHR_U:
         case WASM_INSN_I64_SHR_U:
           kit_cg_int_binop(cg, KIT_CG_INT_LSHR, 0);
           break;
         case WASM_INSN_I32_ROTL:
           wasm_cg_rotate(c, cg, b, WASM_VAL_I32, 0);
           break;
         case WASM_INSN_I32_ROTR:
           wasm_cg_rotate(c, cg, b, WASM_VAL_I32, 1);
           break;
         case WASM_INSN_I64_ROTL:
           wasm_cg_rotate(c, cg, b, WASM_VAL_I64, 0);
           break;
         case WASM_INSN_I64_ROTR:
           wasm_cg_rotate(c, cg, b, WASM_VAL_I64, 1);
           break;
         case WASM_INSN_I32_CLZ:
         case WASM_INSN_I64_CLZ:
           kit_cg_intrinsic(cg, KIT_CG_INTRIN_CLZ, 1,
                            in.kind == WASM_INSN_I32_CLZ
                                ? b.id[KIT_CG_BUILTIN_I32]
                                : b.id[KIT_CG_BUILTIN_I64]);
           break;
         case WASM_INSN_I32_CTZ:
         case WASM_INSN_I64_CTZ:
           kit_cg_intrinsic(cg, KIT_CG_INTRIN_CTZ, 1,
                            in.kind == WASM_INSN_I32_CTZ
                                ? b.id[KIT_CG_BUILTIN_I32]
                                : b.id[KIT_CG_BUILTIN_I64]);
           break;
         case WASM_INSN_I32_POPCNT:
         case WASM_INSN_I64_POPCNT:
           kit_cg_intrinsic(cg, KIT_CG_INTRIN_POPCOUNT, 1,
                            in.kind == WASM_INSN_I32_POPCNT
                                ? b.id[KIT_CG_BUILTIN_I32]
                                : b.id[KIT_CG_BUILTIN_I64]);
           break;
         case WASM_INSN_I32_EQZ:
           kit_cg_push_int(cg, 0, b.id[KIT_CG_BUILTIN_I32]);
           kit_cg_int_cmp(cg, KIT_CG_INT_EQ);
           kit_cg_zext(cg, b.id[KIT_CG_BUILTIN_I32]);
           break;
         case WASM_INSN_I64_EQZ:
           kit_cg_push_int(cg, 0, b.id[KIT_CG_BUILTIN_I64]);
           kit_cg_int_cmp(cg, KIT_CG_INT_EQ);
           kit_cg_zext(cg, b.id[KIT_CG_BUILTIN_I32]);
           break;
         case WASM_INSN_I32_WRAP_I64:
           kit_cg_trunc(cg, b.id[KIT_CG_BUILTIN_I32]);
           break;
         case WASM_INSN_I64_EXTEND_I32_S:
           kit_cg_sext(cg, b.id[KIT_CG_BUILTIN_I64]);
           break;
         case WASM_INSN_I64_EXTEND_I32_U:
           kit_cg_zext(cg, b.id[KIT_CG_BUILTIN_I64]);
           break;
         /* In-register sign-extension operators: narrow the value to the source
          * width, then sign-extend back to the result width. */
         case WASM_INSN_I32_EXTEND8_S:
           kit_cg_trunc(cg, b.id[KIT_CG_BUILTIN_I8]);
           kit_cg_sext(cg, b.id[KIT_CG_BUILTIN_I32]);
           break;
         case WASM_INSN_I32_EXTEND16_S:
           kit_cg_trunc(cg, b.id[KIT_CG_BUILTIN_I16]);
           kit_cg_sext(cg, b.id[KIT_CG_BUILTIN_I32]);
           break;
         case WASM_INSN_I64_EXTEND8_S:
           kit_cg_trunc(cg, b.id[KIT_CG_BUILTIN_I8]);
           kit_cg_sext(cg, b.id[KIT_CG_BUILTIN_I64]);
           break;
         case WASM_INSN_I64_EXTEND16_S:
           kit_cg_trunc(cg, b.id[KIT_CG_BUILTIN_I16]);
           kit_cg_sext(cg, b.id[KIT_CG_BUILTIN_I64]);
           break;
         case WASM_INSN_I64_EXTEND32_S:
           kit_cg_trunc(cg, b.id[KIT_CG_BUILTIN_I32]);
           kit_cg_sext(cg, b.id[KIT_CG_BUILTIN_I64]);
           break;
         case WASM_INSN_I32_TRUNC_F32_S:
         case WASM_INSN_I32_TRUNC_F64_S:
           wasm_cg_checked_trunc(c, cg, b, &rt,
                                 in.kind == WASM_INSN_I32_TRUNC_F32_S
                                     ? WASM_VAL_F32
                                     : WASM_VAL_F64,
                                 WASM_VAL_I32, 0);
           break;
         case WASM_INSN_I32_TRUNC_F32_U:
         case WASM_INSN_I32_TRUNC_F64_U:
           wasm_cg_checked_trunc(c, cg, b, &rt,
                                 in.kind == WASM_INSN_I32_TRUNC_F32_U
                                     ? WASM_VAL_F32
                                     : WASM_VAL_F64,
                                 WASM_VAL_I32, 1);
           break;
         case WASM_INSN_I64_TRUNC_F32_S:
         case WASM_INSN_I64_TRUNC_F64_S:
           wasm_cg_checked_trunc(c, cg, b, &rt,
                                 in.kind == WASM_INSN_I64_TRUNC_F32_S
                                     ? WASM_VAL_F32
                                     : WASM_VAL_F64,
                                 WASM_VAL_I64, 0);
           break;
         case WASM_INSN_I64_TRUNC_F32_U:
         case WASM_INSN_I64_TRUNC_F64_U:
           wasm_cg_checked_trunc(c, cg, b, &rt,
                                 in.kind == WASM_INSN_I64_TRUNC_F32_U
                                     ? WASM_VAL_F32
                                     : WASM_VAL_F64,
                                 WASM_VAL_I64, 1);
           break;
         case WASM_INSN_F32_CONVERT_I32_S:
         case WASM_INSN_F32_CONVERT_I64_S:
           kit_cg_sint_to_float(cg, b.id[KIT_CG_BUILTIN_F32],
                                KIT_CG_ROUND_DEFAULT);
           break;
         case WASM_INSN_F32_CONVERT_I32_U:
         case WASM_INSN_F32_CONVERT_I64_U:
           kit_cg_uint_to_float(cg, b.id[KIT_CG_BUILTIN_F32],
                                KIT_CG_ROUND_DEFAULT);
           break;
         case WASM_INSN_F64_CONVERT_I32_S:
         case WASM_INSN_F64_CONVERT_I64_S:
           kit_cg_sint_to_float(cg, b.id[KIT_CG_BUILTIN_F64],
                                KIT_CG_ROUND_DEFAULT);
           break;
         case WASM_INSN_F64_CONVERT_I32_U:
         case WASM_INSN_F64_CONVERT_I64_U:
           kit_cg_uint_to_float(cg, b.id[KIT_CG_BUILTIN_F64],
                                KIT_CG_ROUND_DEFAULT);
           break;
         case WASM_INSN_F32_DEMOTE_F64:
           kit_cg_fptrunc(cg, b.id[KIT_CG_BUILTIN_F32]);
           break;
         case WASM_INSN_F64_PROMOTE_F32:
           kit_cg_fpext(cg, b.id[KIT_CG_BUILTIN_F64]);
           break;
         case WASM_INSN_I32_REINTERPRET_F32:
           kit_cg_bitcast(cg, b.id[KIT_CG_BUILTIN_I32]);
           break;
         case WASM_INSN_I64_REINTERPRET_F64:
           kit_cg_bitcast(cg, b.id[KIT_CG_BUILTIN_I64]);
           break;
         case WASM_INSN_F32_REINTERPRET_I32:
           kit_cg_bitcast(cg, b.id[KIT_CG_BUILTIN_F32]);
           break;
         case WASM_INSN_F64_REINTERPRET_I64:
           kit_cg_bitcast(cg, b.id[KIT_CG_BUILTIN_F64]);
           break;
         default: {
           KitCgIntCmpOp cmp;
           KitCgFpBinOp fp_bin;
           KitCgFpCmpOp fp_cmp;
           if (wasm_int_cmp_op(in.kind, &cmp)) {
             kit_cg_int_cmp(cg, cmp);
             kit_cg_zext(cg, b.id[KIT_CG_BUILTIN_I32]);
           } else if (in.kind == WASM_INSN_F32_NEG ||
                      in.kind == WASM_INSN_F64_NEG) {
             kit_cg_fp_unop(cg, KIT_CG_FP_NEG, KIT_CG_FP_NONE);
           } else if (wasm_fp_binop(in.kind, &fp_bin)) {
             kit_cg_fp_binop(cg, fp_bin, KIT_CG_FP_NONE);
           } else if (wasm_fp_cmp_op(in.kind, &fp_cmp)) {
             kit_cg_fp_cmp(cg, fp_cmp);
             kit_cg_zext(cg, b.id[KIT_CG_BUILTIN_I32]);
           } else {
             wasm_error(c, in.loc, "wasm: unsupported instruction");
           }
           break;
         }
       }
     }
     /* If the body's final instruction was an explicit RETURN (or a
      * required-tail call), the SValue stack is already empty and the
      * function's prologue/epilogue is closed — emitting another ret here
      * underflows. Skip in that case; the wasm validator already proved
      * every reachable exit returns the right shape. */
     {
       uint32_t last = f->ninsns;
       int body_terminates;
       while (last > 0 && f->insns[last - 1u].kind == WASM_INSN_END) last--;
       body_terminates =
           last > 0 &&
           (f->insns[last - 1u].kind == WASM_INSN_RETURN ||
            f->insns[last - 1u].kind == WASM_INSN_RETURN_CALL ||
            f->insns[last - 1u].kind == WASM_INSN_RETURN_CALL_INDIRECT ||
            f->insns[last - 1u].kind == WASM_INSN_RETURN_CALL_REF ||
            f->insns[last - 1u].kind == WASM_INSN_UNREACHABLE);
       if (!body_terminates) kit_cg_ret(cg);
     }
     kit_cg_func_end(cg);
     heap->free(heap, control, sizeof(WasmCgControl) * control_cap);
   }
   kit_arena_free(arena);
 }
 
 void wasm_emit_cg(KitCompiler* c, const KitCodeOptions* code_opts,
                   KitObjBuilder* out, const WasmModule* m) {
   KitCg* cg = NULL;
   KitCgUnitOptions unit_opts;
   KitStatus cg_st = kit_cg_new(c, &cg);
   if (cg_st == KIT_OK) cg_st = kit_cg_begin(cg, out, code_opts);
   memset(&unit_opts, 0, sizeof unit_opts);
   if (cg_st == KIT_OK) cg_st = kit_cg_begin_unit(cg, &unit_opts);
   if (cg_st != KIT_OK || !cg)
     wasm_error(c, wasm_loc(0, 0), "wasm: failed to initialize codegen");
   wasm_emit_cg_into(c, cg, m);
   if (kit_cg_end_unit(cg) != KIT_OK || kit_cg_finish(cg, NULL) != KIT_OK ||
       kit_cg_detach(cg) != KIT_OK)
     wasm_error(c, wasm_loc(0, 0), "wasm: failed to finalize codegen");
   kit_cg_free(cg);
 }