kit

wat.c (74847B)

---

 #include <kit/source.h>
 
 #include "wasm/wasm.h"
 #include "wasm/wasm_insn_table.h"
 
 typedef struct WasmTok {
   const char* p;
   size_t len;
   uint32_t line;
   uint32_t col;
   uint8_t kind;
 } WasmTok;
 
 enum {
   WT_EOF = 0,
   WT_LPAREN,
   WT_RPAREN,
   WT_ATOM,
   WT_STRING,
 };
 
 /* One entry in the parser's lexical label scope. A block/loop/if pushes its
  * optional `$name` (a borrowed slice into the source text; NULL for anonymous
  * blocks); the matching end/close pops it. br/br_if/br_table resolve a `$name`
  * operand to the relative depth (0 = innermost enclosing). */
 typedef struct WatLabel {
   const char* name; /* NULL for an anonymous (unnamed) block */
   size_t name_len;
 } WatLabel;
 
 typedef struct WatParser {
   KitCompiler* c;
   const char* name;
   const char* src;
   size_t len;
   size_t pos;
   uint32_t line;
   uint32_t col;
   WasmTok tok;
   KitSrcLoc field_loc;
   WasmModule* module;
   /* Heap-grown label scope stack (no fixed cap, hangs off the parser). The
    * top of the stack is the innermost enclosing block, i.e. branch depth 0. */
   WatLabel* labels;
   uint32_t nlabels;
   uint32_t cap_labels;
 } WatParser;
 
 static KitSrcLoc wat_loc(WatParser* p, uint32_t line, uint32_t col) {
   KitSrcLoc loc = wasm_loc(line, col);
   if (p && p->module) loc.file_id = p->module->file_id;
   return loc;
 }
 
 static KitSrcLoc wat_tok_loc(WatParser* p, WasmTok t) {
   return wat_loc(p, t.line, t.col);
 }
 
 static int tok_is(WasmTok t, const char* s) {
   size_t n = kit_slice_cstr(s).len;
   return t.kind == WT_ATOM && t.len == n && memcmp(t.p, s, n) == 0;
 }
 
 static int wasm_name_eq(const char* name, WasmTok t) {
   size_t n;
   if (!name || t.kind != WT_ATOM) return 0;
   n = kit_slice_cstr(name).len;
   return t.len == n && memcmp(name, t.p, n) == 0;
 }
 
 static int wat_hex(char ch) {
   if (ch >= '0' && ch <= '9') return ch - '0';
   if (ch >= 'a' && ch <= 'f') return ch - 'a' + 10;
   if (ch >= 'A' && ch <= 'F') return ch - 'A' + 10;
   return -1;
 }
 
 static char* wat_dup_string(WatParser* p, WasmTok t, size_t* len_out) {
   char* out;
   size_t i, n = 0;
   if (t.kind != WT_STRING)
     wasm_error(p->c, wasm_loc(t.line, t.col), "wasm wat: expected string");
   out = (char*)p->module->heap->alloc(p->module->heap, t.len + 1u, 1);
   if (!out) wasm_error(p->c, wasm_loc(t.line, t.col), "wasm: out of memory");
   for (i = 0; i < t.len; ++i) {
     char ch = t.p[i];
     if (ch != '\\') {
       out[n++] = ch;
       continue;
     }
     if (++i >= t.len)
       wasm_error(p->c, wasm_loc(t.line, t.col),
                  "wasm wat: unterminated string escape");
     ch = t.p[i];
     switch (ch) {
       case 'n':
         out[n++] = '\n';
         break;
       case 'r':
         out[n++] = '\r';
         break;
       case 't':
         out[n++] = '\t';
         break;
       case '"':
       case '\'':
       case '\\':
         out[n++] = ch;
         break;
       default: {
         int hi = wat_hex(ch);
         int lo = (i + 1u < t.len) ? wat_hex(t.p[i + 1u]) : -1;
         if (hi < 0 || lo < 0)
           wasm_error(p->c, wasm_loc(t.line, t.col),
                      "wasm wat: unsupported string escape");
         out[n++] = (char)((hi << 4) | lo);
         i++;
         break;
       }
     }
   }
   out[n] = '\0';
   if (len_out) *len_out = n;
   return out;
 }
 
 static char* wat_dup_atom(WatParser* p, WasmTok t) {
   char* out = wasm_strdup(p->module->heap, t.p, t.len);
   if (!out) wasm_error(p->c, wasm_loc(t.line, t.col), "wasm: out of memory");
   return out;
 }
 
 static void wat_next(WatParser* p) {
   const char* s = p->src;
   while (p->pos < p->len) {
     char ch = s[p->pos];
     if (ch == '\n') {
       p->pos++;
       p->line++;
       p->col = 1;
       continue;
     }
     if (ch == ' ' || ch == '\t' || ch == '\r') {
       p->pos++;
       p->col++;
       continue;
     }
     if (ch == ';' && p->pos + 1u < p->len && s[p->pos + 1u] == ';') {
       while (p->pos < p->len && s[p->pos] != '\n') {
         p->pos++;
         p->col++;
       }
       continue;
     }
     if (ch == '(' && p->pos + 1u < p->len && s[p->pos + 1u] == ';') {
       uint32_t depth = 1;
       p->pos += 2u;
       p->col += 2u;
       while (depth && p->pos < p->len) {
         if (s[p->pos] == '\n') {
           p->pos++;
           p->line++;
           p->col = 1;
         } else if (s[p->pos] == '(' && p->pos + 1u < p->len &&
                    s[p->pos + 1u] == ';') {
           p->pos += 2u;
           p->col += 2u;
           depth++;
         } else if (s[p->pos] == ';' && p->pos + 1u < p->len &&
                    s[p->pos + 1u] == ')') {
           p->pos += 2u;
           p->col += 2u;
           depth--;
         } else {
           p->pos++;
           p->col++;
         }
       }
       if (depth)
         wasm_error(p->c, wasm_loc(p->line, p->col),
                    "wasm wat: unterminated block comment");
       continue;
     }
     break;
   }
   p->tok.p = s + p->pos;
   p->tok.len = 0;
   p->tok.line = p->line;
   p->tok.col = p->col;
   p->tok.kind = WT_EOF;
   if (p->pos >= p->len) return;
   if (s[p->pos] == '(') {
     p->tok.kind = WT_LPAREN;
     p->tok.len = 1;
     p->pos++;
     p->col++;
     return;
   }
   if (s[p->pos] == ')') {
     p->tok.kind = WT_RPAREN;
     p->tok.len = 1;
     p->pos++;
     p->col++;
     return;
   }
   if (s[p->pos] == '"') {
     size_t start = ++p->pos;
     p->col++;
     p->tok.kind = WT_STRING;
     p->tok.p = s + start;
     while (p->pos < p->len && s[p->pos] != '"') {
       if (s[p->pos] == '\\') {
         p->pos++;
         p->col++;
         if (p->pos >= p->len) break;
       } else if ((unsigned char)s[p->pos] < 0x20) {
         wasm_error(p->c, wasm_loc(p->line, p->col),
                    "wasm wat: unsupported string escape/control character");
       }
       p->pos++;
       p->col++;
     }
     if (p->pos >= p->len)
       wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                  "wasm wat: unterminated string");
     p->tok.len = (size_t)(s + p->pos - p->tok.p);
     p->pos++;
     p->col++;
     return;
   }
   p->tok.kind = WT_ATOM;
   while (p->pos < p->len) {
     char ch = s[p->pos];
     if (ch == '(' || ch == ')' || ch == ' ' || ch == '\t' || ch == '\r' ||
         ch == '\n')
       break;
     p->pos++;
     p->col++;
   }
   p->tok.len = (size_t)(s + p->pos - p->tok.p);
 }
 
 static void wat_expect(WatParser* p, uint8_t kind, const char* what) {
   if (p->tok.kind != kind)
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: expected %.*s", KIT_SLICE_ARG(kit_slice_cstr(what)));
   wat_next(p);
 }
 
 static int wat_parse_i64(WatParser* p, int64_t* out) {
   const char* s = p->tok.p;
   size_t n = p->tok.len, i = 0;
   uint64_t v = 0;
   int neg = 0;
   unsigned base = 10;
   if (p->tok.kind != WT_ATOM || n == 0) return 0;
   if (s[0] == '-' || s[0] == '+') {
     neg = s[0] == '-';
     i = 1;
   }
   if (i + 2u <= n && s[i] == '0' && (s[i + 1u] == 'x' || s[i + 1u] == 'X')) {
     base = 16;
     i += 2u;
   }
   if (i == n) return 0;
   for (; i < n; ++i) {
     int hd;
     unsigned d;
     if (s[i] == '_') continue;
     hd = wat_hex(s[i]);
     if (hd < 0 || (unsigned)hd >= base) return 0;
     d = (unsigned)hd;
     if (v > (UINT64_MAX - d) / base) return 0;
     v = v * base + d;
   }
   *out = neg ? -(int64_t)v : (int64_t)v;
   return 1;
 }
 
 static int wat_parse_f64(WatParser* p, double* out) {
   char buf[128];
   char* end = NULL;
   if (p->tok.kind != WT_ATOM || p->tok.len == 0 || p->tok.len >= sizeof buf)
     return 0;
   memcpy(buf, p->tok.p, p->tok.len);
   buf[p->tok.len] = '\0';
   *out = strtod(buf, &end);
   return end && *end == '\0';
 }
 
 static int wat_val_type(WasmTok t, WasmValType* out) {
   if (tok_is(t, "i32")) {
     *out = WASM_VAL_I32;
     return 1;
   }
   if (tok_is(t, "i64")) {
     *out = WASM_VAL_I64;
     return 1;
   }
   if (tok_is(t, "f32")) {
     *out = WASM_VAL_F32;
     return 1;
   }
   if (tok_is(t, "f64")) {
     *out = WASM_VAL_F64;
     return 1;
   }
   if (tok_is(t, "funcref")) {
     *out = WASM_VAL_FUNCREF;
     return 1;
   }
   if (tok_is(t, "externref")) {
     *out = WASM_VAL_EXTERNREF;
     return 1;
   }
   return 0;
 }
 
 static void wat_require_feature(WatParser* p, WasmFeatureSet feature,
                                 const char* feature_name, const char* what) {
   if (!wasm_feature_enabled(p->module, feature))
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: %.*s requires %.*s",
                KIT_SLICE_ARG(kit_slice_cstr(what)),
                KIT_SLICE_ARG(kit_slice_cstr(feature_name)));
 }
 
 static uint8_t wasm_export_kind_from_tok(WasmTok t) {
   if (tok_is(t, "func")) return 0;
   if (tok_is(t, "table")) return 1;
   if (tok_is(t, "memory")) return 2;
   if (tok_is(t, "global")) return 3;
   return 0xffu;
 }
 
 static void wat_skip_list(WatParser* p) {
   uint32_t depth = 1;
   while (depth && p->tok.kind != WT_EOF) {
     if (p->tok.kind == WT_LPAREN)
       depth++;
     else if (p->tok.kind == WT_RPAREN)
       depth--;
     wat_next(p);
   }
 }
 
 /* A handful of mnemonics have legacy spellings that map to the same kind as
  * their canonical WASM_INSN_TABLE name; the table carries only the canonical
  * spelling, so these aliases are checked separately. */
 static const struct {
   const char* mnemonic;
   WasmInsnKind kind;
 } WAT_INSN_ALIASES[] = {
     {"i32.atomic.wait", WASM_INSN_I32_ATOMIC_WAIT},
     {"i64.atomic.wait", WASM_INSN_I64_ATOMIC_WAIT},
     {"atomic.notify", WASM_INSN_MEMORY_ATOMIC_NOTIFY},
 };
 
 /* Resolve a WAT mnemonic token to its instruction kind via a linear scan over
  * the single WASM_INSN_TABLE map (same set, same semantics as the former
  * 200-arm strcmp chain), plus a tiny alias list. `has_imm` reflects whether the
  * grammar parses one trailing immediate token, derived from the operand
  * class. */
 static int wat_instr_kind(WasmTok t, WasmInsnKind* out, int* has_imm) {
   *has_imm = 0;
   if (t.kind != WT_ATOM) return 0;
   for (WasmInsnKind k = 0; k <= WASM_INSN_TABLE_FILL; ++k) {
     const WasmInsnInfo* info = wasm_insn_info(k);
     if (!info) continue;
     if (tok_is(t, info->mnemonic)) {
       *out = k;
       *has_imm =
           wasm_operand_class_has_imm((WasmOperandClass)info->operand_class);
       return 1;
     }
   }
   for (size_t i = 0; i < sizeof(WAT_INSN_ALIASES) / sizeof(WAT_INSN_ALIASES[0]);
        ++i) {
     if (tok_is(t, WAT_INSN_ALIASES[i].mnemonic)) {
       const WasmInsnInfo* info = wasm_insn_info(WAT_INSN_ALIASES[i].kind);
       *out = WAT_INSN_ALIASES[i].kind;
       *has_imm = info && wasm_operand_class_has_imm(
                              (WasmOperandClass)info->operand_class);
       return 1;
     }
   }
   return 0;
 }
 
 /* Push a block/loop/if onto the lexical label scope. `name`/`name_len` borrow
  * a `$name` slice from the source text (NULL for an anonymous block). The
  * stack grows on demand off the parser; it is never shrunk below 0 and is
  * popped in lockstep with WASM_INSN_END emission. */
 static void wat_label_push(WatParser* p, const char* name, size_t name_len) {
   KitHeap* h = p->module->heap;
   if (p->nlabels == p->cap_labels) {
     uint32_t nc = p->cap_labels ? p->cap_labels * 2u : 8u;
     WatLabel* nl = (WatLabel*)wasm_realloc(h, p->labels,
                                            sizeof(WatLabel) * p->cap_labels,
                                            sizeof(WatLabel) * nc);
     if (!nl) wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col), "wasm wat: oom");
     p->labels = nl;
     p->cap_labels = nc;
   }
   p->labels[p->nlabels].name = name;
   p->labels[p->nlabels].name_len = name_len;
   p->nlabels++;
 }
 
 static void wat_label_pop(WatParser* p) {
   if (p->nlabels) p->nlabels--;
 }
 
 /* Resolve a branch operand to a relative depth. A `$name` token is matched
  * against the enclosing label scope (innermost = 0); a numeric token is taken
  * literally. Fatal on an unknown name or a malformed immediate. */
 static void wat_parse_branch_depth(WatParser* p, int64_t* out) {
   if (p->tok.kind == WT_ATOM && p->tok.len && p->tok.p[0] == '$') {
     uint32_t i;
     for (i = 0; i < p->nlabels; ++i) {
       const WatLabel* l = &p->labels[p->nlabels - 1u - i];
       if (l->name && l->name_len == p->tok.len &&
           memcmp(l->name, p->tok.p, p->tok.len) == 0) {
         *out = (int64_t)i;
         return;
       }
     }
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: unknown label name");
   }
   if (!wat_parse_i64(p, out) || *out < 0)
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: expected branch depth");
 }
 
 static void wat_parse_func_index(WatParser* p, int64_t* out) {
   uint32_t i;
   if (p->tok.kind == WT_ATOM && p->tok.len && p->tok.p[0] == '$') {
     for (i = 0; i < p->module->nfuncs; ++i) {
       if (wasm_name_eq(p->module->funcs[i].name, p->tok)) {
         *out = i;
         return;
       }
     }
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: unknown function name");
   }
   if (!wat_parse_i64(p, out))
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: expected instruction immediate");
 }
 
 static void wat_parse_type_index(WatParser* p, int64_t* out) {
   uint32_t i;
   if (p->tok.kind == WT_ATOM && p->tok.len && p->tok.p[0] == '$') {
     for (i = 0; i < p->module->ntypes; ++i) {
       if (wasm_name_eq(p->module->types[i].name, p->tok)) {
         *out = i;
         return;
       }
     }
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: unknown type name");
   }
   if (!wat_parse_i64(p, out))
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: expected type index");
 }
 
 static void wat_parse_local_index(WatParser* p, WasmFunc* f, int64_t* out) {
   uint32_t i, nlocals = f->nparams + f->nlocals;
   if (p->tok.kind == WT_ATOM && p->tok.len && p->tok.p[0] == '$') {
     for (i = 0; i < nlocals; ++i) {
       if (wasm_name_eq(f->local_names[i], p->tok)) {
         *out = i;
         return;
       }
     }
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: unknown local name");
   }
   if (!wat_parse_i64(p, out))
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: expected instruction immediate");
 }
 
 static void wat_parse_global_index(WatParser* p, int64_t* out) {
   uint32_t i;
   if (p->tok.kind == WT_ATOM && p->tok.len && p->tok.p[0] == '$') {
     for (i = 0; i < p->module->nglobals; ++i) {
       if (wasm_name_eq(p->module->globals[i].name, p->tok)) {
         *out = i;
         return;
       }
     }
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: unknown global name");
   }
   if (!wat_parse_i64(p, out))
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: expected global index");
 }
 
 static void wat_parse_instr_imm(WatParser* p, WasmFunc* f, WasmInsnKind kind,
                                 int64_t* out) {
   switch (kind) {
     case WASM_INSN_CALL:
     case WASM_INSN_RETURN_CALL:
       wat_parse_func_index(p, out);
       break;
     case WASM_INSN_GLOBAL_GET:
     case WASM_INSN_GLOBAL_SET:
       wat_parse_global_index(p, out);
       break;
     case WASM_INSN_LOCAL_GET:
     case WASM_INSN_LOCAL_SET:
     case WASM_INSN_LOCAL_TEE:
       wat_parse_local_index(p, f, out);
       break;
     case WASM_INSN_BR:
     case WASM_INSN_BR_IF:
       wat_parse_branch_depth(p, out);
       break;
     default:
       if (!wat_parse_i64(p, out))
         wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                    "wasm wat: expected instruction immediate");
       break;
   }
 }
 
 static int wat_atom_prefix(WasmTok t, const char* prefix) {
   size_t n = kit_slice_cstr(prefix).len;
   return t.kind == WT_ATOM && t.len >= n && memcmp(t.p, prefix, n) == 0;
 }
 
 static int wat_parse_u32_atom(WasmTok t, uint32_t* out) {
   uint64_t v = 0;
   size_t i = 0;
   unsigned base = 10;
   if (t.kind != WT_ATOM || t.len == 0) return 0;
   if (i + 2u <= t.len && t.p[i] == '0' &&
       (t.p[i + 1u] == 'x' || t.p[i + 1u] == 'X')) {
     base = 16;
     i += 2u;
   }
   if (i == t.len) return 0;
   for (; i < t.len; ++i) {
     int hd;
     if (t.p[i] == '_') continue;
     hd = wat_hex(t.p[i]);
     if (hd < 0 || (unsigned)hd >= base) return 0;
     if (v > (UINT32_MAX - (uint32_t)hd) / base) return 0;
     v = v * base + (uint32_t)hd;
   }
   *out = (uint32_t)v;
   return 1;
 }
 
 static int wat_parse_mem_align_log2(WasmTok t, uint32_t* out) {
   uint32_t bytes;
   uint32_t lg = 0;
   if (!wat_parse_u32_atom(t, &bytes) || bytes == 0u) return 0;
   if (bytes & (bytes - 1u)) return 0;
   while (bytes > 1u) {
     bytes >>= 1u;
     lg++;
   }
   *out = lg;
   return 1;
 }
 
 static int wat_parse_u64_atom(WasmTok t, uint64_t* out) {
   uint64_t v = 0;
   size_t i = 0;
   unsigned base = 10;
   if (t.kind != WT_ATOM || t.len == 0) return 0;
   if (i + 2u <= t.len && t.p[i] == '0' &&
       (t.p[i + 1u] == 'x' || t.p[i + 1u] == 'X')) {
     base = 16;
     i += 2u;
   }
   if (i == t.len) return 0;
   for (; i < t.len; ++i) {
     int hd;
     if (t.p[i] == '_') continue;
     hd = wat_hex(t.p[i]);
     if (hd < 0 || (unsigned)hd >= base) return 0;
     if (v > (UINT64_MAX - (uint64_t)hd) / base) return 0;
     v = v * base + (uint64_t)hd;
   }
   *out = v;
   return 1;
 }
 
 static void wat_parse_memory_index(WatParser* p, uint32_t* out) {
   uint32_t i;
   int64_t idx;
   if (p->tok.kind == WT_ATOM && p->tok.len && p->tok.p[0] == '$') {
     for (i = 0; i < p->module->nmemories; ++i) {
       if (wasm_name_eq(p->module->memories[i].name, p->tok)) {
         *out = i;
         return;
       }
     }
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: unknown memory name");
   }
   if (!wat_parse_i64(p, &idx) || idx < 0 || idx > UINT32_MAX)
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: expected memory index");
   *out = (uint32_t)idx;
 }
 
 static void wat_parse_mem_attrs(WatParser* p, uint32_t* align, uint64_t* offset,
                                 uint32_t* memidx) {
   while (p->tok.kind == WT_ATOM) {
     WasmTok val;
     if (wat_atom_prefix(p->tok, "align=")) {
       val = p->tok;
       val.p += 6;
       val.len -= 6;
       if (!wat_parse_mem_align_log2(val, align))
         wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                    "wasm wat: bad memory alignment");
       wat_next(p);
     } else if (wat_atom_prefix(p->tok, "offset=")) {
       val = p->tok;
       val.p += 7;
       val.len -= 7;
       if (!wat_parse_u64_atom(val, offset))
         wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                    "wasm wat: bad memory offset");
       wat_next(p);
     } else if (wat_atom_prefix(p->tok, "memory=")) {
       val = p->tok;
       val.p += 7;
       val.len -= 7;
       if (!wat_parse_u32_atom(val, memidx))
         wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                    "wasm wat: bad memory index");
       wat_next(p);
     } else if (wat_atom_prefix(p->tok, "mem=")) {
       val = p->tok;
       val.p += 4;
       val.len -= 4;
       if (!wat_parse_u32_atom(val, memidx))
         wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                    "wasm wat: bad memory index");
       wat_next(p);
     } else {
       break;
     }
   }
 }
 
 /* Re-synthesize the current token as a '(' that was just consumed via
  * wat_next(), rewinding the lexer to just after it. Used when a one-token
  * lookahead past a '(' did not match the expected keyword and the '(' must be
  * handed to a nested instruction parse. Mirrors the inline idiom used
  * throughout this file. */
 static void wat_unget_to_lparen(WatParser* p, size_t lparen_pos,
                                 uint32_t lparen_line, uint32_t lparen_col) {
   p->pos = lparen_pos;
   p->line = lparen_line;
   p->col = lparen_col;
   p->tok.kind = WT_LPAREN;
   p->tok.p = p->src + p->pos - 1u;
   p->tok.len = 1;
   p->tok.line = lparen_line;
   p->tok.col = lparen_col - 1u;
 }
 
 /* Parse an optional `$label` for a block/loop/if header, pushing it onto the
  * label scope (anonymous push when absent). Returns nothing; the keyword has
  * already been consumed. */
 static void wat_block_push_label(WatParser* p) {
   if (p->tok.kind == WT_ATOM && p->tok.len && p->tok.p[0] == '$') {
     const char* nm = p->tok.p;
     size_t nlen = p->tok.len;
     wat_next(p);
     wat_label_push(p, nm, nlen);
   } else {
     wat_label_push(p, NULL, 0);
   }
 }
 
 /* The resolved blocktype of a block/loop/if header: either the void/single-
  * result shorthand (is_typeidx == 0; `result` is the value type, 0 == void) or
  * a function-type index (is_typeidx == 1; the multi-value form carrying block
  * params and/or multiple results). wat_parse_blocktype_apply stamps it onto a
  * freshly-added instruction. */
 typedef struct WatBlockType {
   int is_typeidx;
   uint32_t typeidx;
   WasmValType result;
 } WatBlockType;
 
 #define WAT_BLOCKTYPE_MAX 64u
 
 /* Parse an optional structured-control blocktype that follows a block/loop/if
  * header (after the label): `(type idx)? (param valtype*)* (result valtype*)*`.
  * The fidelity rule mirrors the binary blocktype: void or a single-`(result)`
  * with no params stays the scalar shorthand; any param or a second result (or
  * an explicit `(type idx)`) resolves to a function-type index, interned into
  * the type section when not named directly. `what` names the construct for
  * diagnostics. */
 static WatBlockType wat_parse_blocktype(WatParser* p, const char* what) {
   WatBlockType bt;
   WasmFunc sig;
   WasmValType pbuf[WAT_BLOCKTYPE_MAX];
   WasmValType rbuf[WAT_BLOCKTYPE_MAX];
   int64_t explicit_typeidx = -1;
   memset(&bt, 0, sizeof bt);
   memset(&sig, 0, sizeof sig);
   sig.params = pbuf;
   sig.cap_params = WAT_BLOCKTYPE_MAX;
   sig.results = rbuf;
   sig.cap_results = WAT_BLOCKTYPE_MAX;
   while (p->tok.kind == WT_LPAREN) {
     size_t lp = p->pos;
     uint32_t ll = p->line, lc = p->col;
     wat_next(p);
     if (tok_is(p->tok, "type")) {
       wat_next(p);
       wat_parse_type_index(p, &explicit_typeidx);
       if (explicit_typeidx < 0 ||
           (uint64_t)explicit_typeidx >= p->module->ntypes)
         wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                    "wasm wat: %.*s blocktype type index out of range",
                    KIT_SLICE_ARG(kit_slice_cstr(what)));
       wat_next(p);
       wat_expect(p, WT_RPAREN, "')'");
     } else if (tok_is(p->tok, "param")) {
       wat_next(p);
       while (p->tok.kind != WT_RPAREN && p->tok.kind != WT_EOF) {
         WasmValType vt;
         if (p->tok.kind == WT_ATOM && p->tok.len && p->tok.p[0] == '$') {
           wat_next(p);
           continue;
         }
         if (!wat_val_type(p->tok, &vt) || !wasm_is_frontend_value_type(vt))
           wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                      "wasm wat: unsupported %.*s param type",
                      KIT_SLICE_ARG(kit_slice_cstr(what)));
         if (sig.nparams >= WAT_BLOCKTYPE_MAX)
           wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                      "wasm wat: too many %.*s blocktype params",
                      KIT_SLICE_ARG(kit_slice_cstr(what)));
         sig.params[sig.nparams++] = vt;
         wat_next(p);
       }
       wat_expect(p, WT_RPAREN, "')'");
     } else if (tok_is(p->tok, "result")) {
       wat_next(p);
       while (p->tok.kind != WT_RPAREN && p->tok.kind != WT_EOF) {
         WasmValType vt;
         if (!wat_val_type(p->tok, &vt) || !wasm_is_frontend_value_type(vt))
           wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                      "wasm wat: unsupported %.*s result type",
                      KIT_SLICE_ARG(kit_slice_cstr(what)));
         if (sig.nresults >= WAT_BLOCKTYPE_MAX)
           wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                      "wasm wat: too many %.*s blocktype results",
                      KIT_SLICE_ARG(kit_slice_cstr(what)));
         sig.results[sig.nresults++] = vt;
         wat_next(p);
       }
       wat_expect(p, WT_RPAREN, "')'");
     } else {
       /* Not a blocktype clause — hand the '(' back to the body parser. */
       wat_unget_to_lparen(p, lp, ll, lc);
       break;
     }
   }
   if (explicit_typeidx >= 0) {
     const WasmFuncType* t = &p->module->types[explicit_typeidx];
     /* If param/result clauses also appear, they must match the named type. */
     if ((sig.nparams || sig.nresults) &&
         (sig.nparams != t->nparams || sig.nresults != t->nresults ||
          (sig.nparams &&
           memcmp(sig.params, t->params, sizeof(WasmValType) * sig.nparams)) ||
          (sig.nresults &&
           memcmp(sig.results, t->results, sizeof(WasmValType) * sig.nresults))))
       wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                  "wasm wat: %.*s blocktype does not match named type",
                  KIT_SLICE_ARG(kit_slice_cstr(what)));
     bt.is_typeidx = 1;
     bt.typeidx = (uint32_t)explicit_typeidx;
     return bt;
   }
   /* Shorthand: no params and at most one result stays a scalar blocktype for
    * round-trip fidelity; anything richer interns a function type. */
   if (sig.nparams == 0 && sig.nresults <= 1) {
     bt.result = sig.nresults ? sig.results[0] : 0;
     return bt;
   }
   bt.is_typeidx = 1;
   bt.typeidx = wasm_intern_func_type(p->c, p->module, &sig);
   return bt;
 }
 
 /* Stamp a parsed blocktype onto the most-recently-added instruction `in`. */
 static void wat_parse_blocktype_apply(WasmInsn* in, WatBlockType bt) {
   if (bt.is_typeidx)
     wasm_insn_set_blocktype_typeidx(in, bt.typeidx);
   else
     in->type = (uint8_t)bt.result;
 }
 
 static void wat_parse_instr(WatParser* p, WasmFunc* f);
 
 static void wat_check_instr_feature(WatParser* p, WasmInsnKind kind) {
   if (kind == WASM_INSN_RETURN_CALL || kind == WASM_INSN_RETURN_CALL_INDIRECT ||
       kind == WASM_INSN_RETURN_CALL_REF)
     wat_require_feature(p, WASM_FEATURE_TAIL_CALLS, "tail calls",
                         "tail-call instruction");
   if (kind == WASM_INSN_REF_NULL || kind == WASM_INSN_REF_FUNC ||
       kind == WASM_INSN_REF_IS_NULL || kind == WASM_INSN_CALL_REF ||
       kind == WASM_INSN_RETURN_CALL_REF)
     wat_require_feature(p, WASM_FEATURE_TYPED_FUNC_REFS,
                         "typed function references",
                         "typed-reference instruction");
   if (kind == WASM_INSN_ATOMIC_FENCE || wasm_insn_is_atomic_mem(kind))
     wat_require_feature(p, WASM_FEATURE_THREADS, "threads",
                         "atomic instruction");
 }
 
 static uint32_t wat_parse_call_indirect_type(WatParser* p) {
   int64_t typeidx;
   wat_expect(p, WT_LPAREN, "'('");
   if (!tok_is(p->tok, "type"))
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: expected call_indirect type");
   wat_next(p);
   wat_parse_type_index(p, &typeidx);
   if (typeidx < 0 || (uint64_t)typeidx >= p->module->ntypes)
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: type index out of range");
   wat_next(p);
   wat_expect(p, WT_RPAREN, "')'");
   return (uint32_t)typeidx;
 }
 
 static void wat_parse_ref_null_type(WatParser* p, int64_t* out) {
   if (tok_is(p->tok, "func") || tok_is(p->tok, "nofunc") ||
       tok_is(p->tok, "funcref")) {
     *out = WASM_VAL_FUNCREF;
     return;
   }
   if (tok_is(p->tok, "extern") || tok_is(p->tok, "noextern") ||
       tok_is(p->tok, "externref")) {
     *out = WASM_VAL_EXTERNREF;
     return;
   }
   wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
              "wasm wat: expected reference type");
 }
 
 /* True for bulk-memory and table.* ops whose WAT form takes one or two
  * index immediates rather than a memarg. */
 static int wat_is_bulk_op(WasmInsnKind k) {
   switch (k) {
     case WASM_INSN_MEMORY_INIT:
     case WASM_INSN_DATA_DROP:
     case WASM_INSN_MEMORY_COPY:
     case WASM_INSN_MEMORY_FILL:
     case WASM_INSN_TABLE_INIT:
     case WASM_INSN_ELEM_DROP:
     case WASM_INSN_TABLE_COPY:
     case WASM_INSN_TABLE_GROW:
     case WASM_INSN_TABLE_SIZE:
     case WASM_INSN_TABLE_FILL:
       return 1;
     default:
       return 0;
   }
 }
 
 /* Optionally parse a uleb-style index immediate. Accepts:
  *   - a numeric literal (any base/sign accepted by wat_parse_i64)
  *   - a `$name` resolved against the segment lookup callback `resolver`,
  *     which returns 1 with the resolved index in *out on success.
  * Returns 0 if the token isn't a recognizable immediate. */
 typedef int (*WatNameResolver)(WatParser*, WasmTok name, uint32_t* out);
 static int wat_try_parse_uleb_imm_named(WatParser* p, WatNameResolver resolver,
                                         uint32_t* out) {
   int64_t v;
   WasmInsnKind next_kind;
   int next_has_imm;
   if (p->tok.kind != WT_ATOM) return 0;
   if (p->tok.len && p->tok.p[0] == '$') {
     if (!resolver) return 0;
     if (!resolver(p, p->tok, out)) return 0;
     wat_next(p);
     return 1;
   }
   if (wat_instr_kind(p->tok, &next_kind, &next_has_imm)) return 0;
   if (!wat_parse_i64(p, &v) || v < 0 || v > UINT32_MAX) return 0;
   *out = (uint32_t)v;
   wat_next(p);
   return 1;
 }
 
 static int wat_resolve_data_name(WatParser* p, WasmTok name, uint32_t* out) {
   for (uint32_t i = 0; i < p->module->ndata; ++i) {
     if (wasm_name_eq(p->module->data[i].name, name)) {
       *out = i;
       return 1;
     }
   }
   return 0;
 }
 
 static int wat_resolve_elem_name(WatParser* p, WasmTok name, uint32_t* out) {
   for (uint32_t i = 0; i < p->module->nelems; ++i) {
     if (wasm_name_eq(p->module->elems[i].name, name)) {
       *out = i;
       return 1;
     }
   }
   return 0;
 }
 
 /* Emit a bulk-memory or table op, parsing 0..2 index immediates depending on
  * the kind. Caller has already consumed the opcode token. Named refs of the
  * form `$name` resolve against the appropriate segment table (data segments
  * for memory.init/data.drop, elem segments for table.init/elem.drop). */
 static void wat_emit_bulk_op(WatParser* p, WasmFunc* f, WasmInsnKind kind) {
   uint32_t a = 0, b = 0;
   int got_a, got_b;
   WatNameResolver primary = NULL;
   switch (kind) {
     case WASM_INSN_MEMORY_INIT:
     case WASM_INSN_DATA_DROP:
       primary = wat_resolve_data_name;
       break;
     case WASM_INSN_TABLE_INIT:
     case WASM_INSN_ELEM_DROP:
       primary = wat_resolve_elem_name;
       break;
     default:
       primary = NULL;
       break;
   }
   got_a = wat_try_parse_uleb_imm_named(p, primary, &a);
   got_b = wat_try_parse_uleb_imm_named(p, primary, &b);
   wasm_func_add_insn(p->c, p->module, f, kind, 0);
   switch (kind) {
     case WASM_INSN_MEMORY_INIT:
       /* memory.init [memidx] dataidx — single arg means dataidx, two args
        * is (memidx dataidx). */
       if (got_b) {
         f->insns[f->ninsns - 1u].memidx = a;
         f->insns[f->ninsns - 1u].imm = (int64_t)b;
       } else {
         f->insns[f->ninsns - 1u].memidx = 0;
         f->insns[f->ninsns - 1u].imm = (int64_t)a;
       }
       break;
     case WASM_INSN_DATA_DROP:
       f->insns[f->ninsns - 1u].imm = (int64_t)a;
       break;
     case WASM_INSN_MEMORY_COPY:
       /* memory.copy [dst src] — both default to 0. */
       f->insns[f->ninsns - 1u].memidx = got_a ? a : 0;
       f->insns[f->ninsns - 1u].aux_idx = got_b ? b : 0;
       break;
     case WASM_INSN_MEMORY_FILL:
       f->insns[f->ninsns - 1u].memidx = got_a ? a : 0;
       break;
     case WASM_INSN_TABLE_INIT:
       /* table.init [tableidx] elemidx. */
       if (got_b) {
         f->insns[f->ninsns - 1u].aux_idx = a;
         f->insns[f->ninsns - 1u].imm = (int64_t)b;
       } else {
         f->insns[f->ninsns - 1u].aux_idx = 0;
         f->insns[f->ninsns - 1u].imm = (int64_t)a;
       }
       break;
     case WASM_INSN_ELEM_DROP:
       f->insns[f->ninsns - 1u].imm = (int64_t)a;
       break;
     case WASM_INSN_TABLE_COPY:
       f->insns[f->ninsns - 1u].imm = (int64_t)(got_a ? a : 0);
       f->insns[f->ninsns - 1u].aux_idx = got_b ? b : 0;
       break;
     case WASM_INSN_TABLE_GROW:
     case WASM_INSN_TABLE_SIZE:
     case WASM_INSN_TABLE_FILL:
       f->insns[f->ninsns - 1u].imm = (int64_t)(got_a ? a : 0);
       break;
     default:
       break;
   }
 }
 
 static void wat_parse_instr_list(WatParser* p, WasmFunc* f) {
   WasmInsnKind kind;
   int has_imm;
   int64_t imm = 0;
   WasmTok head;
   wat_expect(p, WT_LPAREN, "'('");
   head = p->tok;
   p->module->current_loc = wat_tok_loc(p, head);
   if (tok_is(head, "block") || tok_is(head, "loop")) {
     WatBlockType blockty;
     kind = tok_is(head, "block") ? WASM_INSN_BLOCK : WASM_INSN_LOOP;
     wat_next(p);
     wat_block_push_label(p);
     blockty = wat_parse_blocktype(p, tok_is(head, "block") ? "block" : "loop");
     wasm_func_add_insn(p->c, p->module, f, kind, 0);
     wat_parse_blocktype_apply(&f->insns[f->ninsns - 1u], blockty);
     while (p->tok.kind != WT_RPAREN && p->tok.kind != WT_EOF)
       wat_parse_instr(p, f);
     wasm_func_add_insn(p->c, p->module, f, WASM_INSN_END, 0);
     wat_label_pop(p);
     wat_expect(p, WT_RPAREN, "')'");
     return;
   }
   if (tok_is(head, "if")) {
     WatBlockType blockty;
     wat_next(p);
     wat_block_push_label(p);
     blockty = wat_parse_blocktype(p, "if");
     while (p->tok.kind == WT_LPAREN) {
       WasmTok save_head;
       size_t save_pos = p->pos;
       uint32_t save_line = p->line, save_col = p->col;
       wat_next(p);
       save_head = p->tok;
       wat_unget_to_lparen(p, save_pos, save_line, save_col);
       if (tok_is(save_head, "then") || tok_is(save_head, "else")) break;
       wat_parse_instr(p, f);
     }
     wasm_func_add_insn(p->c, p->module, f, WASM_INSN_IF, 0);
     wat_parse_blocktype_apply(&f->insns[f->ninsns - 1u], blockty);
     if (p->tok.kind == WT_LPAREN) {
       wat_next(p);
       if (!tok_is(p->tok, "then"))
         wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                    "wasm wat: expected then");
       wat_next(p);
       while (p->tok.kind != WT_RPAREN && p->tok.kind != WT_EOF)
         wat_parse_instr(p, f);
       wat_expect(p, WT_RPAREN, "')'");
     }
     if (p->tok.kind == WT_LPAREN) {
       wat_next(p);
       if (!tok_is(p->tok, "else"))
         wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                    "wasm wat: expected else");
       wasm_func_add_insn(p->c, p->module, f, WASM_INSN_ELSE, 0);
       wat_next(p);
       while (p->tok.kind != WT_RPAREN && p->tok.kind != WT_EOF)
         wat_parse_instr(p, f);
       wat_expect(p, WT_RPAREN, "')'");
     }
     wasm_func_add_insn(p->c, p->module, f, WASM_INSN_END, 0);
     wat_label_pop(p);
     wat_expect(p, WT_RPAREN, "')'");
     return;
   }
   if (!wat_instr_kind(head, &kind, &has_imm))
     wasm_error(p->c, wasm_loc(head.line, head.col),
                "wasm wat: unsupported instruction");
   wat_check_instr_feature(p, kind);
   wat_next(p);
   if (wasm_insn_is_mem(kind)) {
     uint32_t align = 0, memidx = 0;
     uint64_t offset = 0;
     wat_parse_mem_attrs(p, &align, &offset, &memidx);
     while (p->tok.kind == WT_LPAREN) {
       wat_next(p);
       if (tok_is(p->tok, "memory")) {
         wat_next(p);
         wat_parse_memory_index(p, &memidx);
         wat_next(p);
         wat_expect(p, WT_RPAREN, "')'");
       } else {
         p->pos = (size_t)(p->tok.p - p->src);
         p->line = p->tok.line;
         p->col = p->tok.col;
         p->tok.kind = WT_LPAREN;
         p->tok.p = p->src + p->pos - 1u;
         p->tok.len = 1;
         p->tok.line = p->line;
         p->tok.col = p->col - 1u;
         wat_parse_instr(p, f);
       }
     }
     wasm_func_add_mem_insn(p->c, p->module, f, kind, align, offset, memidx);
     wat_expect(p, WT_RPAREN, "')'");
     return;
   }
   if (kind == WASM_INSN_BR_TABLE) {
     KitHeap* heap = p->module->heap;
     uint32_t n = 0, cap = 8u;
     uint32_t* tmp = (uint32_t*)heap->alloc(heap, sizeof(uint32_t) * cap,
                                            _Alignof(uint32_t));
     if (!tmp)
       wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col), "wasm wat: oom");
     while (p->tok.kind == WT_ATOM) {
       int64_t target;
       wat_parse_branch_depth(p, &target);
       if (target > UINT32_MAX)
         wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                    "wasm wat: bad br_table target");
       if (n == cap) {
         uint32_t nc = cap * 2u;
         tmp =
             (uint32_t*)heap->realloc(heap, tmp, sizeof(uint32_t) * cap,
                                      sizeof(uint32_t) * nc, _Alignof(uint32_t));
         if (!tmp)
           wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col), "wasm wat: oom");
         cap = nc;
       }
       tmp[n++] = (uint32_t)target;
       wat_next(p);
     }
     wasm_func_add_insn(p->c, p->module, f, WASM_INSN_BR_TABLE, 0);
     wasm_insn_set_targets(p->c, p->module, &f->insns[f->ninsns - 1u], tmp, n);
     heap->free(heap, tmp, sizeof(uint32_t) * cap);
     while (p->tok.kind == WT_LPAREN) wat_parse_instr(p, f);
     wat_expect(p, WT_RPAREN, "')'");
     return;
   }
   if (kind == WASM_INSN_CALL_INDIRECT ||
       kind == WASM_INSN_RETURN_CALL_INDIRECT) {
     uint32_t typeidx = wat_parse_call_indirect_type(p);
     while (p->tok.kind == WT_LPAREN) wat_parse_instr(p, f);
     wasm_func_add_insn(p->c, p->module, f, kind, typeidx);
     wat_expect(p, WT_RPAREN, "')'");
     return;
   }
   if (kind == WASM_INSN_CALL_REF || kind == WASM_INSN_RETURN_CALL_REF) {
     uint32_t typeidx = wat_parse_call_indirect_type(p);
     while (p->tok.kind == WT_LPAREN) wat_parse_instr(p, f);
     wasm_func_add_insn(p->c, p->module, f, kind, typeidx);
     wat_expect(p, WT_RPAREN, "')'");
     return;
   }
   if (kind == WASM_INSN_REF_NULL) {
     wat_parse_ref_null_type(p, &imm);
     wat_next(p);
     wasm_func_add_insn(p->c, p->module, f, kind, imm);
     wat_expect(p, WT_RPAREN, "')'");
     return;
   }
   if (kind == WASM_INSN_REF_FUNC) {
     wat_parse_func_index(p, &imm);
     wat_next(p);
     wasm_func_add_insn(p->c, p->module, f, kind, imm);
     wat_expect(p, WT_RPAREN, "')'");
     return;
   }
   if (wat_is_bulk_op(kind)) {
     wat_emit_bulk_op(p, f, kind);
     while (p->tok.kind == WT_LPAREN) wat_parse_instr(p, f);
     wat_expect(p, WT_RPAREN, "')'");
     return;
   }
   if (kind == WASM_INSN_MEMORY_SIZE || kind == WASM_INSN_MEMORY_GROW) {
     uint32_t memidx = 0;
     while (p->tok.kind == WT_LPAREN) {
       wat_next(p);
       if (!tok_is(p->tok, "memory")) {
         p->pos = (size_t)(p->tok.p - p->src);
         p->line = p->tok.line;
         p->col = p->tok.col;
         p->tok.kind = WT_LPAREN;
         p->tok.p = p->src + p->pos - 1u;
         p->tok.len = 1;
         p->tok.line = p->line;
         p->tok.col = p->col - 1u;
         wat_parse_instr(p, f);
         continue;
       }
       wat_next(p);
       wat_parse_memory_index(p, &memidx);
       wat_next(p);
       wat_expect(p, WT_RPAREN, "')'");
     }
     {
       WasmInsnKind next_kind;
       int next_has_imm;
       if (p->tok.kind == WT_ATOM &&
           !wat_instr_kind(p->tok, &next_kind, &next_has_imm)) {
         wat_parse_memory_index(p, &memidx);
         wat_next(p);
       }
     }
     wasm_func_add_insn(p->c, p->module, f,
                        kind == WASM_INSN_MEMORY_GROW ? WASM_INSN_MEMORY_GROW
                                                      : WASM_INSN_MEMORY_SIZE,
                        0);
     f->insns[f->ninsns - 1u].memidx = memidx;
     wat_expect(p, WT_RPAREN, "')'");
     return;
   }
   if (has_imm) {
     if (kind == WASM_INSN_F32_CONST || kind == WASM_INSN_F64_CONST) {
       double fv;
       if (!wat_parse_f64(p, &fv))
         wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                    "wasm wat: expected float immediate");
       wat_next(p);
       while (p->tok.kind == WT_LPAREN) wat_parse_instr(p, f);
       wasm_func_add_fp_insn(p->c, p->module, f, kind, fv);
       wat_expect(p, WT_RPAREN, "')'");
       return;
     } else {
       wat_parse_instr_imm(p, f, kind, &imm);
       wat_next(p);
     }
   }
   while (p->tok.kind == WT_LPAREN) wat_parse_instr(p, f);
   wasm_func_add_insn(p->c, p->module, f, kind, imm);
   wat_expect(p, WT_RPAREN, "')'");
 }
 
 static void wat_parse_instr(WatParser* p, WasmFunc* f) {
   WasmInsnKind kind;
   int has_imm;
   if (p->tok.kind == WT_LPAREN) {
     wat_parse_instr_list(p, f);
     return;
   }
   if (!wat_instr_kind(p->tok, &kind, &has_imm))
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: unsupported instruction");
   p->module->current_loc = wat_tok_loc(p, p->tok);
   wat_check_instr_feature(p, kind);
   wat_next(p);
   if (kind == WASM_INSN_BLOCK || kind == WASM_INSN_LOOP ||
       kind == WASM_INSN_IF) {
     /* Flat-form structured control: `block $l (result T) ... end`. Parse the
      * optional label and blocktype, push the label scope, and record the
      * resolved blocktype on the instruction. */
     WatBlockType blockty;
     wat_block_push_label(p);
     blockty = wat_parse_blocktype(
         p, kind == WASM_INSN_IF ? "if" : (kind == WASM_INSN_LOOP ? "loop"
                                                                  : "block"));
     wasm_func_add_insn(p->c, p->module, f, kind, 0);
     wat_parse_blocktype_apply(&f->insns[f->ninsns - 1u], blockty);
     return;
   }
   if (kind == WASM_INSN_END) {
     wat_label_pop(p);
     wasm_func_add_insn(p->c, p->module, f, kind, 0);
     return;
   }
   if (wasm_insn_is_mem(kind)) {
     uint32_t align = 0, memidx = 0;
     uint64_t offset = 0;
     wat_parse_mem_attrs(p, &align, &offset, &memidx);
     wasm_func_add_mem_insn(p->c, p->module, f, kind, align, offset, memidx);
     return;
   }
   if (kind == WASM_INSN_BR_TABLE) {
     KitHeap* heap = p->module->heap;
     uint32_t n = 0, cap = 8u;
     uint32_t* tmp = (uint32_t*)heap->alloc(heap, sizeof(uint32_t) * cap,
                                            _Alignof(uint32_t));
     if (!tmp)
       wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col), "wasm wat: oom");
     wasm_func_add_insn(p->c, p->module, f, WASM_INSN_BR_TABLE, 0);
     while (p->tok.kind == WT_ATOM) {
       WasmInsnKind next_kind;
       int next_has_imm;
       int64_t target;
       int is_label = p->tok.len && p->tok.p[0] == '$';
       if (!is_label && wat_instr_kind(p->tok, &next_kind, &next_has_imm)) break;
       wat_parse_branch_depth(p, &target);
       if (target > UINT32_MAX)
         wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                    "wasm wat: bad br_table target");
       if (n == cap) {
         uint32_t nc = cap * 2u;
         tmp =
             (uint32_t*)heap->realloc(heap, tmp, sizeof(uint32_t) * cap,
                                      sizeof(uint32_t) * nc, _Alignof(uint32_t));
         if (!tmp)
           wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col), "wasm wat: oom");
         cap = nc;
       }
       tmp[n++] = (uint32_t)target;
       wat_next(p);
     }
     wasm_insn_set_targets(p->c, p->module, &f->insns[f->ninsns - 1u], tmp, n);
     heap->free(heap, tmp, sizeof(uint32_t) * cap);
     return;
   }
   if (kind == WASM_INSN_CALL_INDIRECT ||
       kind == WASM_INSN_RETURN_CALL_INDIRECT) {
     uint32_t typeidx = wat_parse_call_indirect_type(p);
     wasm_func_add_insn(p->c, p->module, f, kind, typeidx);
     return;
   }
   if (kind == WASM_INSN_CALL_REF || kind == WASM_INSN_RETURN_CALL_REF) {
     uint32_t typeidx = wat_parse_call_indirect_type(p);
     wasm_func_add_insn(p->c, p->module, f, kind, typeidx);
     return;
   }
   if (kind == WASM_INSN_REF_NULL) {
     int64_t imm;
     wat_parse_ref_null_type(p, &imm);
     wasm_func_add_insn(p->c, p->module, f, kind, imm);
     wat_next(p);
     return;
   }
   if (kind == WASM_INSN_REF_FUNC) {
     int64_t imm;
     wat_parse_func_index(p, &imm);
     wasm_func_add_insn(p->c, p->module, f, kind, imm);
     wat_next(p);
     return;
   }
   if (kind == WASM_INSN_MEMORY_SIZE || kind == WASM_INSN_MEMORY_GROW) {
     uint32_t memidx = 0;
     WasmInsnKind next_kind;
     int next_has_imm;
     if (p->tok.kind == WT_ATOM &&
         !wat_instr_kind(p->tok, &next_kind, &next_has_imm)) {
       wat_parse_memory_index(p, &memidx);
       wat_next(p);
     }
     wasm_func_add_insn(p->c, p->module, f, kind, 0);
     f->insns[f->ninsns - 1u].memidx = memidx;
     return;
   }
   if (wat_is_bulk_op(kind)) {
     wat_emit_bulk_op(p, f, kind);
     return;
   }
   if (has_imm) {
     if (kind == WASM_INSN_F32_CONST || kind == WASM_INSN_F64_CONST) {
       double fv;
       if (!wat_parse_f64(p, &fv))
         wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                    "wasm wat: expected float immediate");
       wasm_func_add_fp_insn(p->c, p->module, f, kind, fv);
       wat_next(p);
     } else {
       int64_t imm;
       wat_parse_instr_imm(p, f, kind, &imm);
       wasm_func_add_insn(p->c, p->module, f, kind, imm);
       wat_next(p);
     }
   } else {
     wasm_func_add_insn(p->c, p->module, f, kind, 0);
   }
 }
 
 static void wat_parse_func(WatParser* p) {
   WasmFunc* f = wasm_add_func(p->c, p->module);
   uint32_t checked_params = 0;
   uint32_t checked_results = 0;
   /* The label scope is per-function; a well-formed body balances its pushes
    * and pops, but reset defensively so a malformed prior function can't leak
    * stale labels into this one. */
   p->nlabels = 0;
   f->loc = p->field_loc;
   wat_expect(p, WT_LPAREN, "'('");
   if (!tok_is(p->tok, "func"))
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: expected func");
   wat_next(p);
   if (p->tok.kind == WT_ATOM && p->tok.len > 0 && p->tok.p[0] == '$') {
     f->name = wat_dup_atom(p, p->tok);
     wat_next(p);
   }
   while (p->tok.kind != WT_RPAREN && p->tok.kind != WT_EOF) {
     if (p->tok.kind == WT_LPAREN) {
       wat_next(p);
       if (tok_is(p->tok, "export")) {
         wat_next(p);
         if (p->tok.kind != WT_STRING)
           wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                      "wasm wat: expected export string");
         f->export_name = wat_dup_string(p, p->tok, NULL);
         {
           WasmExport* ex = wasm_add_export(p->c, p->module);
           ex->name = wasm_strdup(p->module->heap, f->export_name,
                                  kit_slice_cstr(f->export_name).len);
           if (!ex->name)
             wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                        "wasm: out of memory");
           ex->kind = 0;
           ex->index = p->module->nfuncs - 1u;
         }
         wat_next(p);
         wat_expect(p, WT_RPAREN, "')'");
       } else if (tok_is(p->tok, "type")) {
         int64_t typeidx;
         wat_next(p);
         wat_parse_type_index(p, &typeidx);
         if (typeidx < 0 || (uint64_t)typeidx >= p->module->ntypes)
           wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                      "wasm wat: type index out of range");
         f->typeidx = (uint32_t)typeidx;
         f->has_typeidx = 1;
         wasm_func_set_params(p->c, p->module, f,
                              p->module->types[typeidx].params,
                              p->module->types[typeidx].nparams);
         wasm_func_set_results(p->c, p->module, f,
                               p->module->types[typeidx].results,
                               p->module->types[typeidx].nresults);
         wat_next(p);
         wat_expect(p, WT_RPAREN, "')'");
       } else if (tok_is(p->tok, "param")) {
         WasmTok pending_name;
         int have_name = 0;
         memset(&pending_name, 0, sizeof pending_name);
         wat_next(p);
         while (p->tok.kind != WT_RPAREN && p->tok.kind != WT_EOF) {
           WasmValType vt;
           if (p->tok.kind == WT_ATOM && p->tok.len && p->tok.p[0] == '$') {
             pending_name = p->tok;
             have_name = 1;
             wat_next(p);
             continue;
           }
           if (!wat_val_type(p->tok, &vt))
             wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                        "wasm wat: expected parameter type");
           if (!wasm_is_frontend_value_type(vt))
             wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                        "wasm wat: unsupported parameter type");
           if (f->has_typeidx) {
             if (checked_params >= f->nparams || f->params[checked_params] != vt)
               wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                          "wasm wat: parameter does not match type");
             if (have_name) {
               wasm_func_set_local_name(p->c, p->module, f, checked_params,
                                        pending_name.p, pending_name.len);
               have_name = 0;
             }
             checked_params++;
             wat_next(p);
             continue;
           }
           if (have_name) {
             wasm_func_set_local_name(p->c, p->module, f, f->nparams,
                                      pending_name.p, pending_name.len);
             have_name = 0;
           }
           wasm_func_push_param(p->c, p->module, f, vt);
           wat_next(p);
         }
         wat_expect(p, WT_RPAREN, "')'");
       } else if (tok_is(p->tok, "result")) {
         wat_next(p);
         while (p->tok.kind != WT_RPAREN && p->tok.kind != WT_EOF) {
           WasmValType vt;
           if (!wat_val_type(p->tok, &vt))
             wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                        "wasm wat: expected result type");
           if (!wasm_is_frontend_value_type(vt))
             wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                        "wasm wat: unsupported result type");
           if (f->has_typeidx) {
             if (checked_results >= f->nresults ||
                 f->results[checked_results] != vt)
               wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                          "wasm wat: result does not match type");
             checked_results++;
           } else {
             wasm_func_push_result(p->c, p->module, f, vt);
           }
           wat_next(p);
         }
         wat_expect(p, WT_RPAREN, "')'");
       } else if (tok_is(p->tok, "local")) {
         WasmTok pending_name;
         int have_name = 0;
         memset(&pending_name, 0, sizeof pending_name);
         wat_next(p);
         while (p->tok.kind != WT_RPAREN && p->tok.kind != WT_EOF) {
           WasmValType vt;
           if (p->tok.kind == WT_ATOM && p->tok.len && p->tok.p[0] == '$') {
             pending_name = p->tok;
             have_name = 1;
             wat_next(p);
             continue;
           }
           if (!wat_val_type(p->tok, &vt))
             wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                        "wasm wat: expected local type");
           if (have_name) {
             uint32_t index = f->nparams + f->nlocals;
             wasm_func_set_local_name(p->c, p->module, f, index, pending_name.p,
                                      pending_name.len);
             have_name = 0;
           }
           wasm_func_push_local(p->c, p->module, f, vt);
           wat_next(p);
         }
         wat_expect(p, WT_RPAREN, "')'");
       } else {
         p->pos = (size_t)(p->tok.p - p->src);
         p->line = p->tok.line;
         p->col = p->tok.col;
         p->tok.kind = WT_LPAREN;
         p->tok.p = p->src + p->pos - 1u;
         p->tok.len = 1;
         p->tok.line = p->line;
         p->tok.col = p->col - 1u;
         wat_parse_instr(p, f);
       }
     } else {
       wat_parse_instr(p, f);
     }
   }
   if (!f->has_typeidx)
     f->typeidx = wasm_intern_func_type(p->c, p->module, f);
   else if (checked_params && checked_params != f->nparams)
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: parameter list does not match type");
   wat_expect(p, WT_RPAREN, "')'");
 }
 
 static void wat_parse_type_field(WatParser* p) {
   WasmFuncType* t = wasm_add_type(p->c, p->module);
   if (p->tok.kind == WT_ATOM && p->tok.len && p->tok.p[0] == '$') {
     t->name = wat_dup_atom(p, p->tok);
     wat_next(p);
   }
   wat_expect(p, WT_LPAREN, "'('");
   if (!tok_is(p->tok, "func"))
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: expected func type");
   wat_next(p);
   while (p->tok.kind != WT_RPAREN && p->tok.kind != WT_EOF) {
     wat_expect(p, WT_LPAREN, "'('");
     if (tok_is(p->tok, "param")) {
       wat_next(p);
       while (p->tok.kind != WT_RPAREN && p->tok.kind != WT_EOF) {
         WasmValType vt;
         if (p->tok.kind == WT_ATOM && p->tok.len && p->tok.p[0] == '$') {
           wat_next(p);
           continue;
         }
         if (!wat_val_type(p->tok, &vt) || !wasm_is_num_type(vt))
           wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                      "wasm wat: expected parameter type");
         wasm_type_push_param(p->c, p->module, t, vt);
         wat_next(p);
       }
     } else if (tok_is(p->tok, "result")) {
       wat_next(p);
       while (p->tok.kind != WT_RPAREN && p->tok.kind != WT_EOF) {
         WasmValType vt;
         if (!wat_val_type(p->tok, &vt) || !wasm_is_num_type(vt))
           wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                      "wasm wat: expected result type");
         wasm_type_push_result(p->c, p->module, t, vt);
         wat_next(p);
       }
     } else {
       wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                  "wasm wat: expected type field");
     }
     wat_expect(p, WT_RPAREN, "')'");
   }
   wat_expect(p, WT_RPAREN, "')'");
   wat_expect(p, WT_RPAREN, "')'");
 }
 
 static void wat_parse_export_field(WatParser* p) {
   char* name;
   int64_t idx = 0;
   uint8_t kind;
   if (p->tok.kind != WT_STRING)
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: expected export string");
   name = wat_dup_string(p, p->tok, NULL);
   wat_next(p);
   wat_expect(p, WT_LPAREN, "'('");
   kind = wasm_export_kind_from_tok(p->tok);
   if (kind == 0xffu)
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: expected export kind");
   wat_next(p);
   if (kind == 0)
     wat_parse_func_index(p, &idx);
   else if (!wat_parse_i64(p, &idx))
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: expected export index");
   if (idx < 0 || idx > UINT32_MAX)
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: export index out of range");
   {
     WasmExport* ex = wasm_add_export(p->c, p->module);
     ex->name = name;
     ex->kind = kind;
     ex->index = (uint32_t)idx;
   }
   if (kind == 0 && (uint64_t)idx < p->module->nfuncs) {
     wasm_free_str(p->module->heap, &p->module->funcs[idx].export_name);
     p->module->funcs[idx].export_name =
         wasm_strdup(p->module->heap, name, kit_slice_cstr(name).len);
   } else if (kind == 2 && (uint64_t)idx < p->module->nmemories) {
     wasm_free_str(p->module->heap,
                   &p->module->memories[(uint32_t)idx].export_name);
     p->module->memories[(uint32_t)idx].export_name =
         wasm_strdup(p->module->heap, name, kit_slice_cstr(name).len);
   } else if (kind == 1 && (uint64_t)idx < p->module->ntables) {
     wasm_free_str(p->module->heap, &p->module->tables[idx].export_name);
     p->module->tables[idx].export_name =
         wasm_strdup(p->module->heap, name, kit_slice_cstr(name).len);
   } else if (kind == 3 && (uint64_t)idx < p->module->nglobals) {
     wasm_free_str(p->module->heap, &p->module->globals[idx].export_name);
     p->module->globals[idx].export_name =
         wasm_strdup(p->module->heap, name, kit_slice_cstr(name).len);
   }
   wat_next(p);
   wat_expect(p, WT_RPAREN, "')'");
   wat_expect(p, WT_RPAREN, "')'");
 }
 
 static void wat_parse_memory_field(WatParser* p) {
   int64_t min_pages, max_pages;
   WasmMemory* mem = wasm_add_memory(p->c, p->module);
   if (p->tok.kind == WT_ATOM && p->tok.len && p->tok.p[0] == '$') {
     mem->name = wat_dup_atom(p, p->tok);
     wat_next(p);
   }
   while (p->tok.kind == WT_ATOM &&
          (tok_is(p->tok, "i64") || tok_is(p->tok, "shared"))) {
     if (tok_is(p->tok, "i64"))
       mem->is64 = 1;
     else
       mem->shared = 1;
     wat_next(p);
   }
   if (!wat_parse_i64(p, &min_pages) || min_pages < 0)
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: expected memory minimum");
   mem->min_pages = (uint64_t)min_pages;
   wat_next(p);
   while (p->tok.kind == WT_ATOM &&
          (tok_is(p->tok, "i64") || tok_is(p->tok, "shared"))) {
     if (tok_is(p->tok, "i64"))
       mem->is64 = 1;
     else
       mem->shared = 1;
     wat_next(p);
   }
   if (p->tok.kind != WT_RPAREN) {
     if (!wat_parse_i64(p, &max_pages) || max_pages < min_pages)
       wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                  "wasm wat: bad memory maximum");
     mem->has_max = 1;
     mem->max_pages = (uint64_t)max_pages;
     wat_next(p);
     while (p->tok.kind == WT_ATOM &&
            (tok_is(p->tok, "i64") || tok_is(p->tok, "shared"))) {
       if (tok_is(p->tok, "i64"))
         mem->is64 = 1;
       else
         mem->shared = 1;
       wat_next(p);
     }
   }
   (void)mem;
   wat_expect(p, WT_RPAREN, "')'");
 }
 
 static void wat_parse_memory_limits(WatParser* p, WasmMemory* mem) {
   int64_t lo, hi;
   while (p->tok.kind == WT_ATOM &&
          (tok_is(p->tok, "i64") || tok_is(p->tok, "shared"))) {
     if (tok_is(p->tok, "i64"))
       mem->is64 = 1;
     else
       mem->shared = 1;
     wat_next(p);
   }
   if (!wat_parse_i64(p, &lo) || lo < 0)
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: expected memory minimum");
   mem->min_pages = (uint64_t)lo;
   wat_next(p);
   while (p->tok.kind == WT_ATOM &&
          (tok_is(p->tok, "i64") || tok_is(p->tok, "shared"))) {
     if (tok_is(p->tok, "i64"))
       mem->is64 = 1;
     else
       mem->shared = 1;
     wat_next(p);
   }
   if (p->tok.kind != WT_RPAREN) {
     if (!wat_parse_i64(p, &hi) || hi < lo)
       wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                  "wasm wat: bad memory maximum");
     mem->has_max = 1;
     mem->max_pages = (uint64_t)hi;
     wat_next(p);
     while (p->tok.kind == WT_ATOM &&
            (tok_is(p->tok, "i64") || tok_is(p->tok, "shared"))) {
       if (tok_is(p->tok, "i64"))
         mem->is64 = 1;
       else
         mem->shared = 1;
       wat_next(p);
     }
   }
 }
 
 static void wat_parse_table_limits_and_type(WatParser* p, WasmTable* t) {
   int64_t lo, hi;
   if (!wat_parse_i64(p, &lo) || lo < 0 || lo > UINT32_MAX)
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: expected table minimum");
   t->min = (uint32_t)lo;
   wat_next(p);
   if (p->tok.kind == WT_ATOM) {
     WasmValType maybe_type;
     if (!wat_val_type(p->tok, &maybe_type)) {
       if (!wat_parse_i64(p, &hi) || hi < lo || hi > UINT32_MAX)
         wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                    "wasm wat: bad table maximum");
       t->has_max = 1;
       t->max = (uint32_t)hi;
       wat_next(p);
     }
   }
   if (!wat_val_type(p->tok, &t->elem_type))
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: expected table element type");
   wat_next(p);
 }
 
 static void wat_parse_import_field(WatParser* p) {
   char *mod, *name;
   if (p->tok.kind != WT_STRING)
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: expected import module string");
   mod = wat_dup_string(p, p->tok, NULL);
   wat_next(p);
   if (p->tok.kind != WT_STRING)
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: expected import name string");
   name = wat_dup_string(p, p->tok, NULL);
   wat_next(p);
   wat_expect(p, WT_LPAREN, "'('");
   if (tok_is(p->tok, "func")) {
     WasmFunc* f = wasm_add_func(p->c, p->module);
     f->is_import = 1;
     f->import_module = mod;
     f->import_name = name;
     wat_next(p);
     if (p->tok.kind == WT_ATOM && p->tok.len && p->tok.p[0] == '$') {
       f->name = wat_dup_atom(p, p->tok);
       wat_next(p);
     }
     while (p->tok.kind != WT_RPAREN && p->tok.kind != WT_EOF) {
       wat_expect(p, WT_LPAREN, "'('");
       if (tok_is(p->tok, "type")) {
         int64_t typeidx;
         wat_next(p);
         wat_parse_type_index(p, &typeidx);
         if (typeidx < 0 || (uint64_t)typeidx >= p->module->ntypes)
           wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                      "wasm wat: type index out of range");
         f->typeidx = (uint32_t)typeidx;
         f->has_typeidx = 1;
         wasm_func_set_params(p->c, p->module, f,
                              p->module->types[typeidx].params,
                              p->module->types[typeidx].nparams);
         wasm_func_set_results(p->c, p->module, f,
                               p->module->types[typeidx].results,
                               p->module->types[typeidx].nresults);
         wat_next(p);
       } else if (tok_is(p->tok, "param")) {
         wat_next(p);
         while (p->tok.kind != WT_RPAREN && p->tok.kind != WT_EOF) {
           WasmValType vt;
           if (p->tok.kind == WT_ATOM && p->tok.len && p->tok.p[0] == '$') {
             wat_next(p);
             continue;
           }
           if (!wat_val_type(p->tok, &vt) || !wasm_is_frontend_value_type(vt))
             wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                        "wasm wat: expected parameter type");
           wasm_func_push_param(p->c, p->module, f, vt);
           wat_next(p);
         }
       } else if (tok_is(p->tok, "result")) {
         wat_next(p);
         while (p->tok.kind != WT_RPAREN && p->tok.kind != WT_EOF) {
           WasmValType vt;
           if (!wat_val_type(p->tok, &vt) || !wasm_is_frontend_value_type(vt))
             wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                        "wasm wat: expected result type");
           wasm_func_push_result(p->c, p->module, f, vt);
           wat_next(p);
         }
       } else {
         wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                    "wasm wat: expected import func type field");
       }
       wat_expect(p, WT_RPAREN, "')'");
     }
     if (!f->has_typeidx) f->typeidx = wasm_intern_func_type(p->c, p->module, f);
   } else if (tok_is(p->tok, "memory")) {
     WasmMemory* mem = wasm_add_memory(p->c, p->module);
     mem->is_import = 1;
     mem->import_module = mod;
     mem->import_name = name;
     wat_next(p);
     if (p->tok.kind == WT_ATOM && p->tok.len && p->tok.p[0] == '$') {
       mem->name = wat_dup_atom(p, p->tok);
       wat_next(p);
     }
     wat_parse_memory_limits(p, mem);
   } else if (tok_is(p->tok, "table")) {
     WasmTable* t = wasm_add_table(p->c, p->module);
     t->is_import = 1;
     t->import_module = mod;
     t->import_name = name;
     wat_next(p);
     if (p->tok.kind == WT_ATOM && p->tok.len && p->tok.p[0] == '$') {
       t->name = wat_dup_atom(p, p->tok);
       wat_next(p);
     }
     wat_parse_table_limits_and_type(p, t);
   } else if (tok_is(p->tok, "global")) {
     WasmGlobal* g = wasm_add_global(p->c, p->module);
     g->is_import = 1;
     g->import_module = mod;
     g->import_name = name;
     wat_next(p);
     if (p->tok.kind == WT_ATOM && p->tok.len && p->tok.p[0] == '$') {
       g->name = wat_dup_atom(p, p->tok);
       wat_next(p);
     }
     if (p->tok.kind == WT_LPAREN) {
       wat_next(p);
       if (!tok_is(p->tok, "mut"))
         wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                    "wasm wat: expected mut");
       g->mutable_ = 1;
       wat_next(p);
       if (!wat_val_type(p->tok, &g->type) || !wasm_is_num_type(g->type))
         wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                    "wasm wat: expected global type");
       wat_next(p);
       wat_expect(p, WT_RPAREN, "')'");
     } else {
       if (!wat_val_type(p->tok, &g->type) || !wasm_is_num_type(g->type))
         wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                    "wasm wat: expected global type");
       wat_next(p);
     }
   } else {
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: unsupported import kind");
   }
   wat_expect(p, WT_RPAREN, "')'");
   wat_expect(p, WT_RPAREN, "')'");
 }
 
 static void wat_parse_table_field(WatParser* p) {
   WasmTable* t = wasm_add_table(p->c, p->module);
   if (p->tok.kind == WT_ATOM && p->tok.len && p->tok.p[0] == '$') {
     t->name = wat_dup_atom(p, p->tok);
     wat_next(p);
   }
   wat_parse_table_limits_and_type(p, t);
   wat_expect(p, WT_RPAREN, "')'");
 }
 
 static void wat_parse_const_expr(WatParser* p, WasmInsn* out) {
   WasmInsnKind kind;
   int has_imm;
   memset(out, 0, sizeof *out);
   wat_expect(p, WT_LPAREN, "'('");
   if (!wat_instr_kind(p->tok, &kind, &has_imm) ||
       (kind != WASM_INSN_I32_CONST && kind != WASM_INSN_I64_CONST &&
        kind != WASM_INSN_F32_CONST && kind != WASM_INSN_F64_CONST))
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: expected constant expression");
   out->kind = (uint8_t)kind;
   out->loc = wat_tok_loc(p, p->tok);
   wat_next(p);
   if (kind == WASM_INSN_F32_CONST || kind == WASM_INSN_F64_CONST) {
     if (!wat_parse_f64(p, &out->fp))
       wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                  "wasm wat: expected float immediate");
   } else if (!wat_parse_i64(p, &out->imm)) {
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: expected integer immediate");
   }
   wat_next(p);
   wat_expect(p, WT_RPAREN, "')'");
 }
 
 static void wat_parse_global_field(WatParser* p) {
   WasmGlobal* g = wasm_add_global(p->c, p->module);
   g->loc = p->field_loc;
   if (p->tok.kind == WT_ATOM && p->tok.len && p->tok.p[0] == '$') {
     g->name = wat_dup_atom(p, p->tok);
     wat_next(p);
   }
   if (p->tok.kind == WT_LPAREN) {
     wat_next(p);
     if (tok_is(p->tok, "export")) {
       wat_next(p);
       if (p->tok.kind != WT_STRING)
         wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                    "wasm wat: expected export string");
       g->export_name = wat_dup_string(p, p->tok, NULL);
       {
         WasmExport* ex = wasm_add_export(p->c, p->module);
         ex->name = wasm_strdup(p->module->heap, g->export_name,
                                kit_slice_cstr(g->export_name).len);
         if (!ex->name)
           wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                      "wasm: out of memory");
         ex->kind = 3;
         ex->index = p->module->nglobals - 1u;
       }
       wat_next(p);
       wat_expect(p, WT_RPAREN, "')'");
     } else if (tok_is(p->tok, "mut")) {
       g->mutable_ = 1;
       wat_next(p);
       if (!wat_val_type(p->tok, &g->type) || !wasm_is_num_type(g->type))
         wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                    "wasm wat: expected global type");
       wat_next(p);
       wat_expect(p, WT_RPAREN, "')'");
     } else {
       wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                  "wasm wat: expected global type");
     }
   } else {
     if (!wat_val_type(p->tok, &g->type) || !wasm_is_num_type(g->type))
       wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                  "wasm wat: expected global type");
     wat_next(p);
   }
   if (!g->type)
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: missing global type");
   wat_parse_const_expr(p, &g->init);
   wat_expect(p, WT_RPAREN, "')'");
 }
 
 static void wat_parse_elem_field(WatParser* p) {
   WasmElemSegment* e = wasm_add_elem(p->c, p->module);
   int has_offset = 0;
   int is_declarative = 0;
   e->tableidx = 0;
   e->elem_type = WASM_VAL_FUNCREF;
   /* Optional `$name` or "declare" keyword for declarative segments. */
   if (p->tok.kind == WT_ATOM && p->tok.len && p->tok.p[0] == '$') {
     e->name = wasm_strdup(p->module->heap, p->tok.p, p->tok.len);
     wat_next(p);
   }
   if (tok_is(p->tok, "declare")) {
     is_declarative = 1;
     wat_next(p);
   } else if (p->tok.kind == WT_ATOM) {
     int64_t tableidx;
     if (wat_parse_i64(p, &tableidx)) {
       if (tableidx < 0 || tableidx > UINT32_MAX)
         wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                    "wasm wat: table index out of range");
       e->tableidx = (uint32_t)tableidx;
       wat_next(p);
     }
   }
   if (!is_declarative && p->tok.kind == WT_LPAREN) {
     WasmInsn off;
     wat_parse_const_expr(p, &off);
     if (off.kind != WASM_INSN_I32_CONST || off.imm < 0)
       wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                  "wasm wat: element offset must be i32.const");
     e->offset = off.imm;
     has_offset = 1;
   }
   if (is_declarative)
     e->mode = WASM_SEG_DECLARATIVE;
   else if (has_offset)
     e->mode = WASM_SEG_ACTIVE;
   else
     e->mode = WASM_SEG_PASSIVE;
   if (tok_is(p->tok, "func")) wat_next(p);
   while (p->tok.kind != WT_RPAREN && p->tok.kind != WT_EOF) {
     int64_t idx;
     wat_parse_func_index(p, &idx);
     if (idx < 0 || idx > UINT32_MAX)
       wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                  "wasm wat: element function index out of range");
     wasm_elem_push_func(p->c, p->module, e, (uint32_t)idx);
     wat_next(p);
   }
   wat_expect(p, WT_RPAREN, "')'");
 }
 
 static void wat_parse_start_field(WatParser* p) {
   int64_t idx;
   p->module->start_field_loc = p->field_loc;
   wat_parse_func_index(p, &idx);
   if (idx < 0 || idx > UINT32_MAX)
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: start function index out of range");
   p->module->has_start = 1;
   p->module->start_func = (uint32_t)idx;
   wat_next(p);
   wat_expect(p, WT_RPAREN, "')'");
 }
 
 static void wat_parse_custom_field(WatParser* p) {
   WasmCustom* cs;
   size_t n = 0;
   char* bytes;
   if (p->tok.kind != WT_STRING)
     wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                "wasm wat: expected custom section name");
   cs = wasm_add_custom(p->c, p->module);
   cs->name = wat_dup_string(p, p->tok, NULL);
   if (kit_slice_eq_cstr(kit_slice_cstr(cs->name), "target_features") ||
       kit_slice_eq_cstr(kit_slice_cstr(cs->name), "target-feature"))
     p->module->has_target_features = 1;
   wat_next(p);
   if (p->tok.kind == WT_STRING) {
     bytes = wat_dup_string(p, p->tok, &n);
     cs->data = (uint8_t*)p->module->heap->alloc(p->module->heap, n ? n : 1u, 1);
     if (!cs->data)
       wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                  "wasm: out of memory");
     memcpy(cs->data, bytes, n);
     cs->len = (uint32_t)n;
     p->module->heap->free(p->module->heap, bytes, p->tok.len + 1u);
     wat_next(p);
   }
   wat_expect(p, WT_RPAREN, "')'");
 }
 
 static void wat_parse_data_field(WatParser* p) {
   int64_t offset = 0;
   uint32_t memidx = 0;
   int has_offset = 0;
   char* bytes = NULL;
   size_t nbytes = 0;
   size_t alloc_len = 0;
   char* seg_name = NULL;
   WasmDataSegment* d;
   if (p->tok.kind == WT_ATOM) {
     WasmInsnKind next_kind;
     int next_has_imm;
     if (p->tok.len && p->tok.p[0] == '$') {
       seg_name = wasm_strdup(p->module->heap, p->tok.p, p->tok.len);
       wat_next(p);
     } else if (!wat_instr_kind(p->tok, &next_kind, &next_has_imm)) {
       wat_parse_memory_index(p, &memidx);
       wat_next(p);
     } else {
       wat_next(p);
     }
   }
   if (p->tok.kind == WT_LPAREN) {
     size_t save_pos = p->pos;
     uint32_t save_line = p->line, save_col = p->col;
     wat_next(p);
     if (tok_is(p->tok, "memory")) {
       wat_next(p);
       wat_parse_memory_index(p, &memidx);
       wat_next(p);
       wat_expect(p, WT_RPAREN, "')'");
     } else {
       p->pos = save_pos;
       p->line = save_line;
       p->col = save_col;
       p->tok.kind = WT_LPAREN;
       p->tok.p = p->src + p->pos - 1u;
       p->tok.len = 1;
       p->tok.line = save_line;
       p->tok.col = save_col - 1u;
     }
   }
   /* The offset expression `(i32.const N)` is optional: when absent the
    * segment is passive. */
   if (p->tok.kind == WT_LPAREN) {
     wat_next(p);
     if (!tok_is(p->tok, "i32.const") && !tok_is(p->tok, "i64.const"))
       wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                  "wasm wat: expected const data offset");
     wat_next(p);
     if (!wat_parse_i64(p, &offset) || offset < 0 || offset > UINT32_MAX)
       wasm_error(p->c, wasm_loc(p->tok.line, p->tok.col),
                  "wasm wat: bad data offset");
     wat_next(p);
     wat_expect(p, WT_RPAREN, "')'");
     has_offset = 1;
   }
   if (p->tok.kind == WT_STRING) {
     alloc_len = p->tok.len + 1u;
     bytes = wat_dup_string(p, p->tok, &nbytes);
     wat_next(p);
   }
   d = wasm_add_data(p->c, p->module);
   d->name = seg_name;
   if (has_offset) {
     d->mode = WASM_SEG_ACTIVE;
     d->memidx = memidx;
     d->offset = offset;
   } else {
     d->mode = WASM_SEG_PASSIVE;
   }
   if (nbytes)
     wasm_data_set_bytes(p->c, p->module, d, (const uint8_t*)bytes, nbytes);
   if (bytes) p->module->heap->free(p->module->heap, bytes, alloc_len);
   wat_expect(p, WT_RPAREN, "')'");
 }
 
 void wasm_parse_wat_body(KitCompiler* c, WasmModule* m, WasmFunc* f,
                          const char* src, size_t len, KitSrcLoc loc) {
   WatParser p;
   memset(&p, 0, sizeof p);
   p.c = c;
   p.name = "<asm template>";
   p.src = src;
   p.len = len;
   p.line = loc.line ? loc.line : 1u;
   p.col = loc.col ? loc.col : 1u;
   p.module = m;
   p.field_loc = loc;
   wat_next(&p);
   while (p.tok.kind != WT_EOF) wat_parse_instr(&p, f);
   if (p.labels)
     m->heap->free(m->heap, p.labels, sizeof(WatLabel) * p.cap_labels);
 }
 
 void wasm_parse_wat(KitCompiler* c, KitSlice name, const KitSlice* input,
                     WasmModule* out) {
   WatParser p;
   memset(&p, 0, sizeof p);
   if (name.s) (void)kit_source_add_memory(c, name, &out->file_id);
   p.c = c;
   p.name = name.s;
   p.src = input->s;
   p.len = input->len;
   p.line = 1;
   p.col = 1;
   p.module = out;
   wat_next(&p);
   wat_expect(&p, WT_LPAREN, "'('");
   if (!tok_is(p.tok, "module"))
     wasm_error(c, wasm_loc(p.tok.line, p.tok.col), "wasm wat: expected module");
   wat_next(&p);
   while (p.tok.kind != WT_RPAREN && p.tok.kind != WT_EOF) {
     if (p.tok.kind != WT_LPAREN)
       wasm_error(c, wasm_loc(p.tok.line, p.tok.col),
                  "wasm wat: expected module field");
     wat_next(&p);
     p.field_loc = wat_tok_loc(&p, p.tok);
     if (tok_is(p.tok, "type")) {
       wat_next(&p);
       wat_parse_type_field(&p);
     } else if (tok_is(p.tok, "func")) {
       p.pos = (size_t)(p.tok.p - p.src);
       p.line = p.tok.line;
       p.col = p.tok.col;
       p.tok.kind = WT_LPAREN;
       p.tok.p = p.src + p.pos - 1u;
       p.tok.len = 1;
       p.tok.line = p.line;
       p.tok.col = p.col - 1u;
       wat_parse_func(&p);
     } else if (tok_is(p.tok, "export")) {
       wat_next(&p);
       wat_parse_export_field(&p);
     } else if (tok_is(p.tok, "memory")) {
       wat_next(&p);
       wat_parse_memory_field(&p);
     } else if (tok_is(p.tok, "data")) {
       wat_next(&p);
       wat_parse_data_field(&p);
     } else if (tok_is(p.tok, "import")) {
       wat_next(&p);
       wat_parse_import_field(&p);
     } else if (tok_is(p.tok, "table")) {
       wat_next(&p);
       wat_parse_table_field(&p);
     } else if (tok_is(p.tok, "global")) {
       wat_next(&p);
       wat_parse_global_field(&p);
     } else if (tok_is(p.tok, "elem")) {
       wat_next(&p);
       wat_parse_elem_field(&p);
     } else if (tok_is(p.tok, "start")) {
       wat_next(&p);
       wat_parse_start_field(&p);
     } else if (tok_is(p.tok, "custom") || tok_is(p.tok, "@custom")) {
       wat_next(&p);
       wat_parse_custom_field(&p);
     } else {
       wat_skip_list(&p);
     }
   }
   wat_expect(&p, WT_RPAREN, "')'");
   if (p.tok.kind != WT_EOF)
     wasm_error(c, wasm_loc(p.tok.line, p.tok.col),
                "wasm wat: trailing tokens after module");
   if (p.labels)
     out->heap->free(out->heap, p.labels, sizeof(WatLabel) * p.cap_labels);
 }