kit

arith.c (84814B)

---

 #include "cg/internal.h"
 
 static int api_try_fold_int_convert(KitCg* g, ConvKind ck, KitCgTypeId sty,
                                     KitCgTypeId dty, i64 in, i64* out) {
   u32 sw;
   u32 dw;
   u64 r;
   if (!g || !out || !api_foldable_int_like_type(g->c, sty, &sw) ||
       !api_foldable_int_like_type(g->c, dty, &dw)) {
     return 0;
   }
   switch (ck) {
     case CV_SEXT:
       r = (u64)api_sign_extend_width((u64)in, sw);
       break;
     case CV_ZEXT:
       r = api_mask_width((u64)in, sw);
       break;
     case CV_TRUNC:
       r = api_mask_width((u64)in, dw);
       break;
     default:
       return 0;
   }
   *out = api_fold_result(g->c, dty, r, dw);
   return 1;
 }
 
 void api_cg_binop(KitCg* g, BinOp iop, u32 flags) {
   ApiSValue b, a;
   ApiConstValue cb, ca, result_const;
   CgTarget* T;
   KitCgTypeId ty;
   Operand ra, rb;
   CGLocal rr;
   Operand dst;
   ApiSValue folded_sv;
   i64 folded;
   int can_delay;
   if (!g) return;
   cb = api_const_at(g, 0);
   ca = api_const_at(g, 1);
   T = g->target;
   b = api_pop(g);
   a = api_pop(g);
   ty = a.type ? a.type : b.type;
   api_const_fold_binop(g, iop, ty, ca, cb, flags, &result_const);
   if (api_unevaluated(g)) {
     api_release(g, &a);
     api_release(g, &b);
     api_push(g, api_uneval_value(g, ty));
     api_const_set_top(g, result_const);
     return;
   }
   /* Delayability is a pure function of (ty, flags), neither of which changes
    * below (strength-reduce rewrites the op and operands, not the type), so
    * classify the foldable int once instead of re-deriving it at each of the
    * three delay gates. */
   can_delay = api_can_delay_int_arith(g, ty, flags);
 
   if (!flags && api_sv_op_is(&a, OPK_IMM) && api_sv_op_is(&b, OPK_IMM) &&
       api_try_fold_int_binop(g, iop, ty, a.op.v.imm, b.op.v.imm, &folded)) {
     api_release(g, &a);
     api_release(g, &b);
     api_push(g, api_make_sv(api_op_imm(folded, ty), ty));
     api_const_set_top(g, result_const);
     return;
   }
 
   /* Strength-reduce mul/udiv/urem by a power of two into shift/and. Rewrites
    * iop and the operands in place; the result flows through the same delay /
    * identity / fallback machinery as any other shift or and. */
   if (!flags) api_try_strength_reduce(g, &iop, ty, &a, &b);
 
   if (can_delay && api_try_fold_arith_chain(g, iop, ty, &a, &b, &folded_sv)) {
     api_release(g, &a);
     api_release(g, &b);
     api_push(g, folded_sv);
     api_const_set_top(g, result_const);
     return;
   }
 
   if (api_type_is_float(g->c, ty)) {
     ra = api_force_local(g, &a, ty);
     rb = api_force_local(g, &b, ty);
   } else {
     ra = api_force_local_unless_imm(g, &a, ty);
     rb = api_force_local_unless_imm(g, &b, ty);
   }
 
   if (can_delay &&
       api_try_collapse_binop_identity(g, iop, ty, &a, &b, &folded_sv)) {
     api_release(g, &a);
     api_release(g, &b);
     api_push(g, folded_sv);
     api_const_set_top(g, result_const);
     return;
   }
 
   if (can_delay && (ra.kind == OPK_LOCAL || rb.kind == OPK_LOCAL) &&
       (ra.kind == OPK_LOCAL || ra.kind == OPK_IMM) &&
       (rb.kind == OPK_LOCAL || rb.kind == OPK_IMM)) {
     int a_owned = api_sv_owns_operand_local(&a, &ra);
     int b_owned = api_sv_owns_operand_local(&b, &rb);
     api_push(g, api_make_arith_binop(g, iop, ra, rb, ty, a_owned, b_owned));
     if (a_owned) api_sv_set_res(&a, RES_INHERENT);
     if (b_owned) api_sv_set_res(&b, RES_INHERENT);
     api_release(g, &a);
     api_release(g, &b);
     api_const_set_top(g, result_const);
     return;
   }
 
   rr = api_alloc_temp_local(g, ty);
   dst = api_op_local(rr, ty);
   /* Flag dead-transient operands so the -O0 backend drops them after the op
    * instead of spilling them at the next barrier (eager dead-operand drop). */
   ra = api_op_kill_if_dead(g, ra, dst);
   rb = api_op_kill_if_dead(g, rb, dst);
   T->binop(T, iop, dst, ra, rb);
   api_release(g, &a);
   api_release(g, &b);
   api_push(g, api_make_sv(dst, ty));
   api_const_set_top(g, result_const);
 }
 
 void api_cg_unop(KitCg* g, UnOp iop, u32 flags) {
   ApiSValue a;
   ApiConstValue ca, result_const;
   CgTarget* T;
   KitCgTypeId ty;
   Operand ra;
   CGLocal rr;
   Operand dst;
   ApiSValue folded_sv;
   i64 folded;
   int can_delay;
   if (!g) return;
   ca = api_const_at(g, 0);
   T = g->target;
   a = api_pop(g);
   ty = a.type ? a.type : a.op.type;
   api_const_fold_unop(g, iop, ty, ca, flags, &result_const);
   if (api_unevaluated(g)) {
     api_release(g, &a);
     api_push(g, api_uneval_value(g, ty));
     api_const_set_top(g, result_const);
     return;
   }
   /* Pure function of (ty, flags); classify the foldable int once for both delay
    * gates below. */
   can_delay = api_can_delay_int_arith(g, ty, flags);
 
   if (iop == UO_FNEG) {
     CG_REQUIRE(g, api_type_is_float(g->c, ty),
                "KitCg: FP negation requires floating operand");
     ra = api_force_local(g, &a, ty);
     rr = api_alloc_temp_local(g, ty);
     dst = api_op_local(rr, ty);
     ra = api_op_kill_if_dead(g, ra, dst);
     T->unop(T, iop, dst, ra);
     api_release(g, &a);
     api_push(g, api_make_sv(dst, ty));
     api_const_set_top(g, result_const);
     return;
   }
 
   /* Logical NOT of a delayed compare stays delayed: invert the predicate in
    * place. For FP this flips ordered<->unordered as well as the relation (via
    * api_invert_cmp), so `!(a<b)` becomes UGE (NaN -> true), matching IEEE
    * negation. The inverted compare keeps the same i32 result type. */
   if (iop == UO_NOT && api_sv_kind(&a) == SV_CMP) {
     a.delayed->cmp.op = api_invert_cmp(a.delayed->cmp.op);
     api_push(g, a);
     api_const_set_top(g, result_const);
     return;
   }
 
   if (!flags && api_sv_op_is(&a, OPK_IMM) &&
       api_try_fold_int_unop(g, iop, ty, a.op.v.imm, &folded)) {
     api_release(g, &a);
     api_push(g, api_make_sv(api_op_imm(folded, ty), ty));
     api_const_set_top(g, result_const);
     return;
   }
 
   if (can_delay && api_try_fold_unary_chain(&a, iop, ty, &folded_sv)) {
     api_release(g, &a);
     api_push(g, folded_sv);
     api_const_set_top(g, result_const);
     return;
   }
 
   ra = api_force_local_unless_imm(g, &a, ty);
   if (can_delay && ra.kind == OPK_LOCAL) {
     int a_owned = api_sv_owns_operand_local(&a, &ra);
     api_push(g, api_make_arith_unop(g, iop, ra, ty, a_owned));
     if (a_owned) api_sv_set_res(&a, RES_INHERENT);
     api_release(g, &a);
     api_const_set_top(g, result_const);
     return;
   }
   rr = api_alloc_temp_local(g, ty);
   dst = api_op_local(rr, ty);
   ra = api_op_kill_if_dead(g, ra, dst);
   T->unop(T, iop, dst, ra);
   api_release(g, &a);
   api_push(g, api_make_sv(dst, ty));
   api_const_set_top(g, result_const);
 }
 
 void api_cg_cmp(KitCg* g, CmpOp cop) {
   ApiSValue b, a;
   ApiConstValue cb, ca, result_const;
   KitCgTypeId opty;
   KitCgTypeId i32;
   Operand ra, rb;
   i64 folded;
   if (!g) return;
   cb = api_const_at(g, 0);
   ca = api_const_at(g, 1);
   b = api_pop(g);
   a = api_pop(g);
   opty = a.type ? a.type : b.type;
   i32 = builtin_id(KIT_CG_BUILTIN_I32);
   api_const_fold_cmp(g, cop, ca, cb, &result_const);
   if (api_unevaluated(g)) {
     api_release(g, &a);
     api_release(g, &b);
     api_push(g, api_uneval_value(g, i32));
     api_const_set_top(g, result_const);
     return;
   }
 
   if (api_sv_op_is(&a, OPK_IMM) && api_sv_op_is(&b, OPK_IMM) &&
       api_try_fold_int_cmp(g, cop, opty, a.op.v.imm, b.op.v.imm, &folded)) {
     api_release(g, &a);
     api_release(g, &b);
     api_push(g, api_make_sv(api_op_imm(folded, i32), i32));
     api_const_set_top(g, result_const);
     return;
   }
 
   ra = api_force_local_unless_imm(g, &a, opty);
   rb = api_force_local_unless_imm(g, &b, opty);
   /* Both integer and FP compares are produced as delayed SV_CMP values.
    * Delaying is what lets api_branch_if (and api_cg_unop's UO_NOT) invert
    * the compare via api_invert_cmp, reaching the unordered FP duals
    * (UGE/UGT/ULE/ULT/UEQ/UNE) from `!(a<b)` etc. with NaN-correct semantics.
    * If the compare instead escapes into value context it is materialized
    * unchanged via api_materialize_cmp_to, which calls T->cmp with the same
    * opcode the eager path used to. */
   api_push(g,
            api_make_cmp(g, cop, ra, rb, i32, api_sv_owns_operand_local(&a, &ra),
                         api_sv_owns_operand_local(&b, &rb)));
   api_const_set_top(g, result_const);
 }
 
 int api_try_i128_convert(KitCg* g, ConvKind ck, KitCgTypeId sty,
                          KitCgTypeId dty, ApiSValue* v);
 int api_try_wide8_convert(KitCg* g, ConvKind ck, KitCgTypeId sty,
                           KitCgTypeId dty, ApiSValue* v);
 
 void api_cg_convert_kind(KitCg* g, KitCgTypeId dst_type, ConvKind ck) {
   ApiSValue v;
   ApiConstValue in_const, result_const;
   CgTarget* T;
   KitCgTypeId sty;
   KitCgTypeId dty;
   Operand src;
   CGLocal rr;
   Operand dst;
   if (!g) return;
   T = g->target;
   /* Keep the return-before-pop order on an invalid dst_type. */
   dty = resolve_type(g->c, dst_type);
   if (!dty) return;
   in_const = api_const_at(g, 0);
   v = api_pop(g);
   sty = resolve_type(g->c, v.type ? v.type : v.op.type);
   if (!sty) {
     api_release(g, &v);
     return;
   }
   api_const_fold_convert(g, ck, sty, dty, in_const, &result_const);
   if (api_unevaluated(g)) {
     api_release(g, &v);
     api_push(g, api_uneval_value(g, dty));
     api_const_set_top(g, result_const);
     return;
   }
   if (sty == dty) {
     v.type = dty;
     v.op.type = dty;
     api_push(g, v);
     api_const_set_top(g, result_const);
     return;
   }
   if (api_sv_op_is(&v, OPK_IMM)) {
     i64 folded;
     if (api_try_fold_int_convert(g, ck, sty, dty, v.op.v.imm, &folded)) {
       api_release(g, &v);
       /* A folded split-lane 8-byte result must be memory-resident, not a bare
        * i64 immediate the backend would truncate. */
       if (api_is_wide8_scalar_type(g->c, dty))
         api_push(g, api_make_wide8_int_const(g, folded, dty));
       else
         api_push(g, api_make_sv(api_op_imm(folded, dty), dty));
       api_const_set_top(g, result_const);
       return;
     }
   }
   if (api_try_i128_convert(g, ck, sty, dty, &v)) {
     api_const_set_top(g, result_const);
     return;
   }
   if (api_try_wide8_convert(g, ck, sty, dty, &v)) {
     api_const_set_top(g, result_const);
     return;
   }
   /* A bitcast between two narrow types of equal width and register class —
    * pointer<->pointer-width integer (the frontend's ptr_to_int/int_to_ptr that
    * brackets every field-offset add) or pointer<->pointer — reinterprets the
    * bits without moving them, so retype the value in place instead of forcing
    * it to a fresh temp and emitting a register copy. int<->float of equal width
    * is excluded: it crosses the GPR/FPR banks and needs a real fmov. The wide
    * (i128 / split-lane) cases are already handled above. */
   /* On a machine-word target, a same-width / same-register-class bitcast moves
    * no bits, so make the result value *be* the source, retyped — no temp, no
    * register copy. This is a value-identity rewrite (the source is dead: it is
    * released just below), which only the cg layer can do; the backend sees
    * convert(dst, src) as two independent locals and cannot know src is
    * substitutable. untyped_values reports the one backend-specific fact it
    * needs — typed backends (the C emitter, the optimizer recorder) leave it 0
    * and keep the explicit convert. OPK_IMM is excluded: a bitcast immediate
    * that becomes an address base must reach a register (an INDIRECT base cannot
    * be an immediate), an invariant the aggregate-arg lowering relies on. */
   if (ck == CV_BITCAST && T->untyped_values && v.op.kind != OPK_IMM) {
     /* sty/dty are already unaliased; decode each id's predicate bitset once and
      * read the aggregate/float bits locally instead of three fresh decodes. */
     u8 sty_bits = api_type_pred_bits(g->c, sty);
     u8 dty_bits = api_type_pred_bits(g->c, dty);
     if (!(sty_bits & API_PRED_AGGREGATE) && !(dty_bits & API_PRED_AGGREGATE) &&
         abi_cg_sizeof(g->c->abi, sty) == abi_cg_sizeof(g->c->abi, dty) &&
         !(sty_bits & API_PRED_FLOAT) == !(dty_bits & API_PRED_FLOAT)) {
       v.type = dty;
       v.op.type = dty;
       api_push(g, v);
       api_const_set_top(g, result_const);
       return;
     }
   }
   if (ck == CV_BITCAST && abi_cg_sizeof(g->c->abi, sty) == 16 &&
       abi_cg_sizeof(g->c->abi, dty) == 16 &&
       (api_is_f128_type(g->c, sty) || api_is_f128_type(g->c, dty))) {
     CGLocal local = api_f128_temp_local(g, dty);
     Operand dst_lv = api_op_local(local, dty);
     if (api_is_lvalue_sv(&v) || v.op.kind == OPK_LOCAL ||
         v.op.kind == OPK_INDIRECT || v.op.kind == OPK_GLOBAL) {
       KitCgTypeId ptr_ty = cg_type_ptr_to(g->c, dty);
       ApiSValue src_lv = v;
       Operand dst_addr;
       Operand src_addr;
       AggregateAccess agg;
       ApiSValue dst_place = api_make_lv(dst_lv, dty);
       api_sv_set_lvalue(&src_lv, 1);
       dst_addr = api_lvalue_addr(g, &dst_place, ptr_ty);
       src_addr = api_lvalue_addr(g, &src_lv, cg_type_ptr_to(g->c, sty));
       memset(&agg, 0, sizeof agg);
       agg.size = 16;
       agg.align = 16;
       g->target->copy_bytes(g->target, dst_addr, src_addr, agg);
     } else if (v.op.kind == OPK_LOCAL) {
       g->target->store(g->target, dst_lv, v.op,
                        api_mem_for_lvalue(g, &dst_lv, sty));
     } else if (v.op.kind == OPK_IMM) {
       u8 bytes[16];
       u64 lo = (u64)v.op.v.imm;
       memset(bytes, 0, sizeof bytes);
       for (u32 i = 0; i < 8; ++i) {
         u32 idx = g->c->target.big_endian ? 15u - i : i;
         bytes[idx] = (u8)(lo >> (i * 8u));
       }
       api_store_f128_bytes(g, local, dty, bytes);
     } else {
       CG_BUG(g, "KitCg: unsupported 16-byte bitcast source");
     }
     api_release(g, &v);
     api_push(g, api_make_lv(dst_lv, dty));
     api_const_set_top(g, result_const);
     return;
   }
 
   src = api_force_local(g, &v, sty);
   rr = api_alloc_temp_local(g, dty);
   dst = api_op_local(rr, dty);
   /* If the source operand is a transient with no other live value-stack
    * reference (v, its last reference, was already popped), flag it dead so the
    * -O0 backend can convert in place into the source's register instead of
    * routing through a fresh one — and drop a redundant extend entirely. */
   if (api_coalesce_on(g) && src.kind == OPK_LOCAL &&
       api_temp_dead(g, src.v.local))
     src.flags |= OPK_FLAG_KILL;
   T->convert(T, ck, dst, src);
   api_release(g, &v);
   api_push(g, api_make_sv(dst, dty));
   api_const_set_top(g, result_const);
 }
 
 /* ============================================================
  * 128-bit integer lowering
  *
  * i128/u128 are 16-byte memory-resident scalars (see api_is_wide16
  * and src/cg/wide.c). The native backends only model <=64-bit
  * register ops, so every i128 arithmetic/compare/convert is lowered
  * here to a compiler-rt-style runtime call (rt/lib/int64). This
  * mirrors the f128 dispatch in kit_cg_fp_*.
  * ============================================================ */
 
 /* The wide / soft-float dispatch class cached on the value-stack node at
  * `depth` below TOS (api_push stamps it from api_type_class). The five
  * stack-top predicates below are now a bounds check plus a tag compare; the
  * type-get + alias-chase gauntlet that used to run per operand happens once at
  * push instead. WK_NARROW for an out-of-range depth keeps every predicate false
  * exactly as the old bounds guards did. */
 static u8 api_stack_wide_kind(KitCg* g, u32 depth) {
   if (!g || g->sp <= depth) return WK_NARROW;
   return g->stack[g->sp - 1u - depth].bitfield.wide_kind;
 }
 
 int api_i128_stack_top(KitCg* g, u32 depth) {
   return api_stack_wide_kind(g, depth) == WK_I128;
 }
 
 /* 64-bit integer split into two 32-bit lanes by the selected ABI. The native
  * backend handles add/sub/and/or/xor on such values as register pairs, but
  * mul/div/shift must be lowered to a __*di3 runtime call (see
  * api_wideint64_binop). i128 routes through its own ti3 path (api_i128_*), so
  * the WK_WIDE8 class excludes it. */
 static int api_wide64_stack_top(KitCg* g, u32 depth) {
   return api_stack_wide_kind(g, depth) == WK_WIDE8;
 }
 
 static int api_binop_is_shift(BinOp iop) {
   return iop == BO_SHL || iop == BO_SHR_U || iop == BO_SHR_S;
 }
 
 static int api_is_bool_type(Compiler* c, KitCgTypeId ty) {
   const CgType* cg = cg_type_get(c, ty);
   return cg && cg->kind == KIT_CG_TYPE_BOOL;
 }
 
 /* Materialize an i128 value as an lvalue and return a pointer local to it. */
 static Operand api_i128_addr(KitCg* g, ApiSValue* v) {
   KitCgTypeId i128 = builtin_id(KIT_CG_BUILTIN_I128);
   ApiSValue lv = api_wide16_materialize_lvalue(g, v, i128);
   return api_lvalue_addr(g, &lv, cg_type_ptr_to(g->c, i128));
 }
 
 /* Load a 64-bit lane of an i128 (addressed by `addr`) into a fresh i64. */
 static Operand api_i128_load_lane(KitCg* g, Operand addr, i32 off) {
   KitCgTypeId i64 = builtin_id(KIT_CG_BUILTIN_I64);
   CGLocal rr = api_alloc_temp_local(g, i64);
   Operand dst = api_op_local(rr, i64);
   MemAccess ma;
   memset(&ma, 0, sizeof ma);
   ma.type = i64;
   ma.size = 8;
   ma.align = 8;
   g->target->load(g->target, dst, api_op_indirect(addr.v.local, off, i64), ma);
   return dst;
 }
 
 static void api_i128_binop(KitCg* g, BinOp iop) {
   KitCgTypeId i128 = builtin_id(KIT_CG_BUILTIN_I128);
   KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32);
   const char* name = api_i128_binop_helper(iop);
   KitCgTypeId ps[2];
   ApiSValue args[2];
   if (!name) {
     compiler_panic(g->c, g->cur_loc, "KitCg: unsupported i128 binop");
     return;
   }
   args[1] = api_pop(g);
   args[0] = api_pop(g);
   ps[0] = i128;
   ps[1] = api_binop_is_shift(iop) ? i32 : i128;
   api_runtime_call_values(g, name, i128, ps, 2, args);
 }
 
 /* Runtime helper name for a 64-bit-integer mul/div/rem/shift on a 32-bit
  * target. Mirrors api_i128_binop_helper but with the compiler-rt *di3 names.
  * Returns NULL for ops the inline backend handles (add/sub/and/or/xor). */
 static const char* api_wideint64_binop_helper(BinOp op) {
   switch (op) {
     case BO_IMUL:
       return "__muldi3";
     case BO_SDIV:
       return "__divdi3";
     case BO_UDIV:
       return "__udivdi3";
     case BO_SREM:
       return "__moddi3";
     case BO_UREM:
       return "__umoddi3";
     case BO_SHL:
       return "__ashldi3";
     case BO_SHR_U:
       return "__lshrdi3";
     case BO_SHR_S:
       return "__ashrdi3";
     default:
       return NULL;
   }
 }
 
 /* Lower a 64-bit mul/div/rem/shift to a runtime call. Mirrors api_i128_binop
  * but ret/params are builtin i64; the shift-count param is i32 (the __ashldi3
  * family takes (i64 value, i32 count) per compiler-rt). */
 static void api_wideint64_binop(KitCg* g, BinOp iop) {
   KitCgTypeId i64 = builtin_id(KIT_CG_BUILTIN_I64);
   KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32);
   const char* name = api_wideint64_binop_helper(iop);
   KitCgTypeId ps[2];
   ApiSValue args[2];
   if (!name) {
     compiler_panic(g->c, g->cur_loc, "KitCg: unsupported wide i64 binop");
     return;
   }
   args[1] = api_pop(g);
   args[0] = api_pop(g);
   ps[0] = i64;
   ps[1] = api_binop_is_shift(iop) ? i32 : i64;
   api_runtime_call_values(g, name, i64, ps, 2, args);
 }
 
 /* ============================================================
  * wide8 inline 2-word lane arithmetic
  *
  * Some 32-bit ABIs represent a 64-bit integer as a memory-resident 8-byte
  * scalar split into two 32-bit lanes. add/sub/and/or/xor/neg/not and compares
  * have no compiler-rt helper (they would recurse), so they are emitted INLINE
  * here as lane ops. mul/div/rem/shift route to __*di3 (api_wideint64_*).
  * ============================================================ */
 
 static i32 wide8_lo_off(KitCg* g) { return g->c->target.big_endian ? 4 : 0; }
 static i32 wide8_hi_off(KitCg* g) { return g->c->target.big_endian ? 0 : 4; }
 
 /* Emit one i32 binop into a fresh temp and return it. */
 static Operand wide8_i32_binop(KitCg* g, BinOp op, Operand a, Operand b) {
   KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32);
   CGLocal r = api_alloc_temp_local(g, i32);
   Operand d = api_op_local(r, i32);
   g->target->binop(g->target, op, d, a, b);
   return d;
 }
 
 /* Emit one i32 compare (0/1 result) into a fresh temp and return it. */
 static Operand wide8_i32_cmp(KitCg* g, CmpOp op, Operand a, Operand b) {
   KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32);
   CGLocal r = api_alloc_temp_local(g, i32);
   Operand d = api_op_local(r, i32);
   g->target->cmp(g->target, op, d, a, b);
   return d;
 }
 
 /* (lo | hi) of the 8-byte value `v` as an i32, for a truthiness test. Consumes
  * nothing on the value stack (caller owns *v). */
 Operand api_wide8_or_lanes(KitCg* g, ApiSValue* v, KitCgTypeId ty) {
   Operand addr = api_wide8_addr(g, v, ty);
   Operand lo = api_wide8_load_lane(g, addr, wide8_lo_off(g));
   Operand hi = api_wide8_load_lane(g, addr, wide8_hi_off(g));
   return wide8_i32_binop(g, BO_OR, lo, hi);
 }
 
 /* add/sub/and/or/xor on two 8-byte ints, result pushed as a fresh 8-byte value.
  * add/sub carry/borrow through the high lane via an sltu (CMP_LT_U). */
 static void api_wide64_binop_inline(KitCg* g, BinOp iop) {
   ApiSValue b = api_pop(g);
   ApiSValue a = api_pop(g);
   KitCgTypeId ty = a.type ? a.type : b.type;
   int lo = wide8_lo_off(g), hi = wide8_hi_off(g);
   Operand aa = api_wide8_addr(g, &a, ty);
   Operand ab = api_wide8_addr(g, &b, ty);
   Operand alo = api_wide8_load_lane(g, aa, lo);
   Operand ahi = api_wide8_load_lane(g, aa, hi);
   Operand blo = api_wide8_load_lane(g, ab, lo);
   Operand bhi = api_wide8_load_lane(g, ab, hi);
   CGLocal res = api_wide8_temp_local(g, ty);
   ApiSValue res_lv = api_make_lv(api_op_local(res, ty), ty);
   Operand ar = api_lvalue_addr(g, &res_lv, cg_type_ptr_to(g->c, ty));
   Operand rlo;
   Operand rhi;
   switch (iop) {
     case BO_AND:
     case BO_OR:
     case BO_XOR:
       rlo = wide8_i32_binop(g, iop, alo, blo);
       rhi = wide8_i32_binop(g, iop, ahi, bhi);
       break;
     case BO_IADD: {
       Operand carry;
       rlo = wide8_i32_binop(g, BO_IADD, alo, blo);
       carry = wide8_i32_cmp(g, CMP_LT_U, rlo, alo); /* unsigned wrap -> carry */
       rhi = wide8_i32_binop(g, BO_IADD, ahi, bhi);
       rhi = wide8_i32_binop(g, BO_IADD, rhi, carry);
       break;
     }
     case BO_ISUB: {
       Operand borrow = wide8_i32_cmp(g, CMP_LT_U, alo, blo);
       rlo = wide8_i32_binop(g, BO_ISUB, alo, blo);
       rhi = wide8_i32_binop(g, BO_ISUB, ahi, bhi);
       rhi = wide8_i32_binop(g, BO_ISUB, rhi, borrow);
       break;
     }
     default:
       compiler_panic(g->c, g->cur_loc,
                      "KitCg: unsupported wide i64 inline binop");
       return;
   }
   api_wide8_store_lane(g, ar, lo, rlo);
   api_wide8_store_lane(g, ar, hi, rhi);
   api_release(g, &a);
   api_release(g, &b);
   api_push(g, api_make_sv(api_op_local(res, ty), ty));
 }
 
 /* neg / bnot on an 8-byte int. NEG is two's complement: lo = 0-lo with borrow
  * into hi = 0-hi-borrow. BNOT is lane-wise xor -1. */
 static void api_wide64_unop_inline(KitCg* g, UnOp iop) {
   KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32);
   ApiSValue a = api_pop(g);
   KitCgTypeId ty = a.type ? a.type : a.op.type;
   int lo = wide8_lo_off(g), hi = wide8_hi_off(g);
   Operand aa = api_wide8_addr(g, &a, ty);
   Operand alo = api_wide8_load_lane(g, aa, lo);
   Operand ahi = api_wide8_load_lane(g, aa, hi);
   CGLocal res = api_wide8_temp_local(g, ty);
   ApiSValue res_lv = api_make_lv(api_op_local(res, ty), ty);
   Operand ar = api_lvalue_addr(g, &res_lv, cg_type_ptr_to(g->c, ty));
   Operand rlo;
   Operand rhi;
   if (iop == UO_BNOT) {
     rlo = wide8_i32_binop(g, BO_XOR, alo, api_op_imm(-1, i32));
     rhi = wide8_i32_binop(g, BO_XOR, ahi, api_op_imm(-1, i32));
   } else { /* UO_NEG: 0 - value */
     Operand zero = api_op_imm(0, i32);
     Operand borrow = wide8_i32_cmp(g, CMP_LT_U, zero, alo); /* 0<lo -> borrow */
     rlo = wide8_i32_binop(g, BO_ISUB, zero, alo);
     rhi = wide8_i32_binop(g, BO_ISUB, zero, ahi);
     rhi = wide8_i32_binop(g, BO_ISUB, rhi, borrow);
   }
   api_wide8_store_lane(g, ar, lo, rlo);
   api_wide8_store_lane(g, ar, hi, rhi);
   api_release(g, &a);
   api_push(g, api_make_sv(api_op_local(res, ty), ty));
 }
 
 /* a < b over 8-byte lanes: (a_hi <{s,u} b_hi) | (a_hi==b_hi & a_lo <u b_lo).
  * The high lane uses the signed/unsigned relation; the low lane is always
  * unsigned. Returns an i32 0/1. */
 static Operand wide8_lt(KitCg* g, int is_signed, Operand alo, Operand ahi,
                         Operand blo, Operand bhi) {
   Operand hi_lt = wide8_i32_cmp(g, is_signed ? CMP_LT_S : CMP_LT_U, ahi, bhi);
   Operand hi_eq = wide8_i32_cmp(g, CMP_EQ, ahi, bhi);
   Operand lo_lt = wide8_i32_cmp(g, CMP_LT_U, alo, blo);
   Operand t = wide8_i32_binop(g, BO_AND, hi_eq, lo_lt);
   return wide8_i32_binop(g, BO_OR, hi_lt, t);
 }
 
 static Operand wide8_eq(KitCg* g, Operand alo, Operand ahi, Operand blo,
                         Operand bhi) {
   KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32);
   Operand dlo = wide8_i32_binop(g, BO_XOR, alo, blo);
   Operand dhi = wide8_i32_binop(g, BO_XOR, ahi, bhi);
   Operand diff = wide8_i32_binop(g, BO_OR, dlo, dhi);
   return wide8_i32_cmp(g, CMP_EQ, diff, api_op_imm(0, i32));
 }
 
 static int cmp_is_signed_rel(CmpOp op) {
   return op == CMP_LT_S || op == CMP_LE_S || op == CMP_GT_S || op == CMP_GE_S;
 }
 
 /* 8-byte int compare -> eager i32 0/1 value (not a delayed SV_CMP). */
 static void api_wide64_cmp_inline(KitCg* g, CmpOp cop) {
   KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32);
   int sg = cmp_is_signed_rel(cop);
   ApiSValue b = api_pop(g);
   ApiSValue a = api_pop(g);
   KitCgTypeId ty = a.type ? a.type : b.type;
   int lo = wide8_lo_off(g), hi = wide8_hi_off(g);
   Operand aa = api_wide8_addr(g, &a, ty);
   Operand ab = api_wide8_addr(g, &b, ty);
   Operand alo = api_wide8_load_lane(g, aa, lo);
   Operand ahi = api_wide8_load_lane(g, aa, hi);
   Operand blo = api_wide8_load_lane(g, ab, lo);
   Operand bhi = api_wide8_load_lane(g, ab, hi);
   Operand one = api_op_imm(1, i32);
   Operand res;
   switch (cop) {
     case CMP_EQ:
       res = wide8_eq(g, alo, ahi, blo, bhi);
       break;
     case CMP_NE:
       res = wide8_i32_binop(g, BO_XOR, wide8_eq(g, alo, ahi, blo, bhi), one);
       break;
     case CMP_LT_S:
     case CMP_LT_U:
       res = wide8_lt(g, sg, alo, ahi, blo, bhi);
       break;
     case CMP_GT_S:
     case CMP_GT_U: /* a>b  ==  b<a */
       res = wide8_lt(g, sg, blo, bhi, alo, ahi);
       break;
     case CMP_LE_S:
     case CMP_LE_U: /* a<=b ==  !(b<a) */
       res =
           wide8_i32_binop(g, BO_XOR, wide8_lt(g, sg, blo, bhi, alo, ahi), one);
       break;
     case CMP_GE_S:
     case CMP_GE_U: /* a>=b ==  !(a<b) */
       res =
           wide8_i32_binop(g, BO_XOR, wide8_lt(g, sg, alo, ahi, blo, bhi), one);
       break;
     default:
       compiler_panic(g->c, g->cur_loc, "KitCg: unsupported wide i64 compare");
       return;
   }
   api_release(g, &a);
   api_release(g, &b);
   api_push(g, api_make_sv(res, i32));
 }
 
 /* ============================================================
  * wide64 __builtin_*_overflow on split-lane 64-bit values
  *
  * The native backends only model single-register overflow, so a 64-bit
  * operand traps there. Here we legalize the 6 overflow intrinsics for a
  * 64-bit operand pair into 32-bit lane ops, computing both the
  * 64-bit wrapped value (stored to a fresh 8-byte temp) and the boolean
  * overflow flag, then pushing [value, ok] exactly as the native path does.
  * add/sub reuse the carry/borrow lane logic; mul builds the full 128-bit
  * product from 32x32->64 partials (no MULHU opcode exists, so each partial
  * is itself synthesized from 16-bit halves).
  * ============================================================ */
 
 /* Unsigned 32x32 -> 64 product of i32 lanes a,b, returned as (*plo,*phi) i32
  * via the 16-bit-halves schoolbook method (the target has no high-multiply
  * opcode, and a plain BO_IMUL only yields the low 32 bits).
  *
  *   a = ah*2^16 + al,  b = bh*2^16 + bl
  *   a*b = ah*bh*2^32 + (ah*bl + al*bh)*2^16 + al*bl
  */
 static void wide8_umul32(KitCg* g, Operand a, Operand b, Operand* plo,
                          Operand* phi) {
   KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32);
   Operand mask = api_op_imm(0xffff, i32);
   Operand sh16 = api_op_imm(16, i32);
   Operand al = wide8_i32_binop(g, BO_AND, a, mask);
   Operand ah = wide8_i32_binop(g, BO_SHR_U, a, sh16);
   Operand bl = wide8_i32_binop(g, BO_AND, b, mask);
   Operand bh = wide8_i32_binop(g, BO_SHR_U, b, sh16);
   Operand ll =
       wide8_i32_binop(g, BO_IMUL, al, bl); /* bits 0..31  (<=32 bits) */
   Operand lh = wide8_i32_binop(g, BO_IMUL, al, bh); /* bits 16..47 */
   Operand hl = wide8_i32_binop(g, BO_IMUL, ah, bl); /* bits 16..47 */
   Operand hh = wide8_i32_binop(g, BO_IMUL, ah, bh); /* bits 32..63 */
   /* mid = lh + hl + (ll >> 16); a 33-bit sum -> track carry into bit 32. */
   Operand ll_hi = wide8_i32_binop(g, BO_SHR_U, ll, sh16);
   Operand mid = wide8_i32_binop(g, BO_IADD, lh, hl);
   /* carry out of (lh+hl) into bit 48 (i.e. +2^32 in the high word). */
   Operand c0 = wide8_i32_cmp(g, CMP_LT_U, mid, lh);
   Operand mid2 = wide8_i32_binop(g, BO_IADD, mid, ll_hi);
   Operand c1 = wide8_i32_cmp(g, CMP_LT_U, mid2, mid);
   Operand carry32 = wide8_i32_binop(g, BO_IADD, c0, c1); /* into high word */
   /* lo = (mid2 << 16) | (ll & 0xffff) */
   Operand mid2_lo = wide8_i32_binop(g, BO_AND, mid2, mask);
   Operand mid2_loshift = wide8_i32_binop(g, BO_SHL, mid2_lo, sh16);
   Operand ll_lo = wide8_i32_binop(g, BO_AND, ll, mask);
   *plo = wide8_i32_binop(g, BO_OR, mid2_loshift, ll_lo);
   /* hi = hh + (mid2 >> 16) + carry32*2^16 */
   Operand mid2_hi = wide8_i32_binop(g, BO_SHR_U, mid2, sh16);
   Operand carry_word = wide8_i32_binop(g, BO_SHL, carry32, sh16);
   Operand hi = wide8_i32_binop(g, BO_IADD, hh, mid2_hi);
   *phi = wide8_i32_binop(g, BO_IADD, hi, carry_word);
 }
 
 /* Add three i32 columns acc += addend, threading carry: returns the new sum and
  * adds the unsigned-wrap carry (0/1) into *carry. */
 static Operand wide8_addc(KitCg* g, Operand acc, Operand addend,
                           Operand* carry) {
   Operand sum = wide8_i32_binop(g, BO_IADD, acc, addend);
   Operand c = wide8_i32_cmp(g, CMP_LT_U, sum, acc);
   *carry = wide8_i32_binop(g, BO_IADD, *carry, c);
   return sum;
 }
 
 /* The 6 __builtin_*_overflow intrinsics for a split-lane wide64 operand pair.
  * Pops the two 8-byte args, computes the wrapped 64-bit value into a fresh
  * 8-byte temp and the bool overflow flag into an i32, then pushes [value, ok]
  * matching the contract of the native overflow path. */
 static void api_wide64_overflow_inline(KitCg* g, KitCgIntrinsic intrin) {
   KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32);
   KitCgTypeId bool_ty = builtin_id(KIT_CG_BUILTIN_BOOL);
   Operand sh31 = api_op_imm(31, i32);
   ApiSValue b = api_pop(g);
   ApiSValue a = api_pop(g);
   KitCgTypeId ty = a.type ? a.type : b.type;
   int lo = wide8_lo_off(g), hi = wide8_hi_off(g);
   Operand aa = api_wide8_addr(g, &a, ty);
   Operand ab = api_wide8_addr(g, &b, ty);
   Operand alo = api_wide8_load_lane(g, aa, lo);
   Operand ahi = api_wide8_load_lane(g, aa, hi);
   Operand blo = api_wide8_load_lane(g, ab, lo);
   Operand bhi = api_wide8_load_lane(g, ab, hi);
   CGLocal res = api_wide8_temp_local(g, ty);
   ApiSValue res_lv = api_make_lv(api_op_local(res, ty), ty);
   Operand ar = api_lvalue_addr(g, &res_lv, cg_type_ptr_to(g->c, ty));
   Operand rlo;
   Operand rhi;
   Operand ok;
   switch (intrin) {
     case KIT_CG_INTRIN_UADD_OVERFLOW:
     case KIT_CG_INTRIN_SADD_OVERFLOW: {
       Operand carry;
       rlo = wide8_i32_binop(g, BO_IADD, alo, blo);
       carry = wide8_i32_cmp(g, CMP_LT_U, rlo, alo);
       rhi = wide8_i32_binop(g, BO_IADD, ahi, bhi);
       /* carry-out of the high lane = (rhi<ahi) before +carry, OR wrap on
        * +carry. Compute rhi step by step so we can detect the final carry-out.
        */
       Operand c_hi0 = wide8_i32_cmp(g, CMP_LT_U, rhi, ahi);
       rhi = wide8_i32_binop(g, BO_IADD, rhi, carry);
       Operand c_hi1 = wide8_i32_cmp(g, CMP_LT_U, rhi, carry);
       if (intrin == KIT_CG_INTRIN_UADD_OVERFLOW) {
         /* unsigned: ok = carry-out of the high lane */
         ok = wide8_i32_binop(g, BO_OR, c_hi0, c_hi1);
       } else {
         /* signed: ok = ((a_hi ^ r_hi) & (b_hi ^ r_hi)) sign bit (bit 31) */
         Operand ar_x = wide8_i32_binop(g, BO_XOR, ahi, rhi);
         Operand br_x = wide8_i32_binop(g, BO_XOR, bhi, rhi);
         Operand both = wide8_i32_binop(g, BO_AND, ar_x, br_x);
         ok = wide8_i32_binop(g, BO_SHR_U, both, sh31);
       }
       break;
     }
     case KIT_CG_INTRIN_USUB_OVERFLOW:
     case KIT_CG_INTRIN_SSUB_OVERFLOW: {
       Operand borrow = wide8_i32_cmp(g, CMP_LT_U, alo, blo);
       rlo = wide8_i32_binop(g, BO_ISUB, alo, blo);
       Operand t = wide8_i32_binop(g, BO_ISUB, ahi, bhi);
       /* high-lane borrow-out: (ahi < bhi) OR (t < borrow after subtracting). */
       Operand b_hi0 = wide8_i32_cmp(g, CMP_LT_U, ahi, bhi);
       Operand b_hi1 = wide8_i32_cmp(g, CMP_LT_U, t, borrow);
       rhi = wide8_i32_binop(g, BO_ISUB, t, borrow);
       if (intrin == KIT_CG_INTRIN_USUB_OVERFLOW) {
         ok = wide8_i32_binop(g, BO_OR, b_hi0, b_hi1);
       } else {
         /* signed: ok = ((a_hi ^ b_hi) & (a_hi ^ r_hi)) sign bit (bit 31) */
         Operand ab_x = wide8_i32_binop(g, BO_XOR, ahi, bhi);
         Operand ar_x = wide8_i32_binop(g, BO_XOR, ahi, rhi);
         Operand both = wide8_i32_binop(g, BO_AND, ab_x, ar_x);
         ok = wide8_i32_binop(g, BO_SHR_U, both, sh31);
       }
       break;
     }
     case KIT_CG_INTRIN_UMUL_OVERFLOW:
     case KIT_CG_INTRIN_SMUL_OVERFLOW: {
       int is_signed = (intrin == KIT_CG_INTRIN_SMUL_OVERFLOW);
       /* For signed, compute |a|,|b| as unsigned 64-bit, do the unsigned 128-bit
        * product, then apply the result sign. Overflow tests below use the
        * unsigned magnitude product plus the expected sign. */
       Operand ua_lo = alo, ua_hi = ahi, ub_lo = blo, ub_hi = bhi;
       Operand sgn = (Operand){0};
       if (is_signed) {
         /* a_sign = ahi >> 31 (0 or 1 in i32, but as a mask we want -1/0). */
         Operand am = wide8_i32_binop(g, BO_SHR_S, ahi, sh31); /* 0 or -1 */
         Operand bm = wide8_i32_binop(g, BO_SHR_S, bhi, sh31);
         /* |a| = (a ^ am) - am  (two's-complement abs), lane-wise w/ borrow. */
         Operand axl = wide8_i32_binop(g, BO_XOR, alo, am);
         Operand axh = wide8_i32_binop(g, BO_XOR, ahi, am);
         Operand brwa = wide8_i32_cmp(g, CMP_LT_U, axl, am);
         ua_lo = wide8_i32_binop(g, BO_ISUB, axl, am);
         Operand tah = wide8_i32_binop(g, BO_ISUB, axh, am);
         ua_hi = wide8_i32_binop(g, BO_ISUB, tah, brwa);
         Operand bxl = wide8_i32_binop(g, BO_XOR, blo, bm);
         Operand bxh = wide8_i32_binop(g, BO_XOR, bhi, bm);
         Operand brwb = wide8_i32_cmp(g, CMP_LT_U, bxl, bm);
         ub_lo = wide8_i32_binop(g, BO_ISUB, bxl, bm);
         Operand tbh = wide8_i32_binop(g, BO_ISUB, bxh, bm);
         ub_hi = wide8_i32_binop(g, BO_ISUB, tbh, brwb);
         sgn = wide8_i32_binop(g, BO_XOR, am, bm); /* result sign mask -1/0 */
       }
       /* Unsigned 128-bit product of (ua_hi:ua_lo) * (ub_hi:ub_lo).
        *   P00 = ua_lo*ub_lo  -> columns 0,1
        *   P01 = ua_lo*ub_hi  -> columns 1,2
        *   P10 = ua_hi*ub_lo  -> columns 1,2
        *   P11 = ua_hi*ub_hi  -> columns 2,3 */
       Operand p00l, p00h, p01l, p01h, p10l, p10h, p11l, p11h;
       wide8_umul32(g, ua_lo, ub_lo, &p00l, &p00h);
       wide8_umul32(g, ua_lo, ub_hi, &p01l, &p01h);
       wide8_umul32(g, ua_hi, ub_lo, &p10l, &p10h);
       wide8_umul32(g, ua_hi, ub_hi, &p11l, &p11h);
       Operand zero = api_op_imm(0, i32);
       /* column 0 */
       Operand r0 = p00l;
       /* column 1 = p00h + p01l + p10l */
       Operand c1 = zero;
       Operand r1 = p00h;
       r1 = wide8_addc(g, r1, p01l, &c1);
       r1 = wide8_addc(g, r1, p10l, &c1);
       /* column 2 = p01h + p10h + p11l + c1 */
       Operand c2 = zero;
       Operand r2 = p01h;
       r2 = wide8_addc(g, r2, p10h, &c2);
       r2 = wide8_addc(g, r2, p11l, &c2);
       r2 = wide8_addc(g, r2, c1, &c2);
       /* column 3 = p11h + c2 */
       Operand r3 = wide8_i32_binop(g, BO_IADD, p11h, c2);
       /* low 64 bits = (r1:r0); high 64 bits = (r3:r2). */
       Operand mlo = r0, mhi = r1;
       Operand hi_lo = r2, hi_hi = r3;
       if (is_signed) {
         /* Apply result sign: negate the 128-bit magnitude if sgn==-1.
          * negated = (x ^ sgn) - sgn across all 4 words with borrow. */
         Operand w0 = wide8_i32_binop(g, BO_XOR, mlo, sgn);
         Operand w1 = wide8_i32_binop(g, BO_XOR, mhi, sgn);
         Operand w2 = wide8_i32_binop(g, BO_XOR, hi_lo, sgn);
         Operand w3 = wide8_i32_binop(g, BO_XOR, hi_hi, sgn);
         Operand bor0 = wide8_i32_cmp(g, CMP_LT_U, w0, sgn);
         mlo = wide8_i32_binop(g, BO_ISUB, w0, sgn);
         Operand t1 = wide8_i32_binop(g, BO_ISUB, w1, sgn);
         Operand bor1a = wide8_i32_cmp(g, CMP_LT_U, w1, sgn);
         Operand bor1b = wide8_i32_cmp(g, CMP_LT_U, t1, bor0);
         mhi = wide8_i32_binop(g, BO_ISUB, t1, bor0);
         Operand bor1 = wide8_i32_binop(g, BO_OR, bor1a, bor1b);
         Operand t2 = wide8_i32_binop(g, BO_ISUB, w2, sgn);
         Operand bor2a = wide8_i32_cmp(g, CMP_LT_U, w2, sgn);
         Operand bor2b = wide8_i32_cmp(g, CMP_LT_U, t2, bor1);
         hi_lo = wide8_i32_binop(g, BO_ISUB, t2, bor1);
         Operand bor2 = wide8_i32_binop(g, BO_OR, bor2a, bor2b);
         Operand t3 = wide8_i32_binop(g, BO_ISUB, w3, sgn);
         hi_hi = wide8_i32_binop(g, BO_ISUB, t3, bor2);
       }
       rlo = mlo;
       rhi = mhi;
       if (!is_signed) {
         /* unsigned overflow: high 64 bits nonzero. */
         Operand t = wide8_i32_binop(g, BO_OR, hi_lo, hi_hi);
         ok = wide8_i32_cmp(g, CMP_NE, t, zero);
       } else {
         /* signed overflow: the 128-bit result is not the sign-extension of its
          * low 64 bits. sext = (rhi >> 31) replicated; overflow if
          * (hi_lo != sext) | (hi_hi != sext) where sext = arithmetic sign of
          * the signed low-64 result (bit 63 = rhi sign). */
         Operand sext = wide8_i32_binop(g, BO_SHR_S, rhi, sh31); /* 0 or -1 */
         Operand d2 = wide8_i32_binop(g, BO_XOR, hi_lo, sext);
         Operand d3 = wide8_i32_binop(g, BO_XOR, hi_hi, sext);
         Operand d = wide8_i32_binop(g, BO_OR, d2, d3);
         ok = wide8_i32_cmp(g, CMP_NE, d, zero);
       }
       break;
     }
     default:
       compiler_panic(g->c, g->cur_loc,
                      "KitCg: unsupported wide i64 overflow intrinsic");
       api_release(g, &a);
       api_release(g, &b);
       return;
   }
   api_wide8_store_lane(g, ar, lo, rlo);
   api_wide8_store_lane(g, ar, hi, rhi);
   api_release(g, &a);
   api_release(g, &b);
   /* Materialize ok as a fresh bool temp so it has a stable home. */
   {
     CGLocal okl = api_alloc_temp_local(g, bool_ty);
     Operand okd = api_op_local(okl, bool_ty);
     g->target->binop(g->target, BO_AND, okd, ok, api_op_imm(1, i32));
     api_push(g, api_make_sv(api_op_local(res, ty), ty));
     api_push(g, api_make_sv(okd, bool_ty));
   }
 }
 
 /* int<->split-i64 conversions (sext/zext/trunc/bitcast across the 4<->8
  * boundary, and i64->bool). Returns 1 if it handled (and consumed) *v. The
  * i64<->float conversions are routed to libcalls in kit_cg_*_to_float /
  * kit_cg_float_to_* and never reach here. */
 int api_try_wide8_convert(KitCg* g, ConvKind ck, KitCgTypeId sty,
                           KitCgTypeId dty, ApiSValue* v) {
   KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32);
   int s_wide = api_is_wide8_scalar_type(g->c, sty);
   int d_wide = api_is_wide8_scalar_type(g->c, dty);
   int lo = wide8_lo_off(g), hi = wide8_hi_off(g);
   if (!s_wide && !d_wide) return 0;
   if (s_wide && d_wide) {
     /* i64<->soft-double reinterpret (same 8-byte layout) or i64<->u64. */
     v->type = dty;
     v->op.type = dty;
     api_push(g, *v);
     return 1;
   }
   if (d_wide) {
     /* narrower int -> i64: low lane is the (converted-to-i32) source; high lane
      * is the sign-extension (CV_SEXT) or zero (CV_ZEXT/CV_BITCAST of a ptr). */
     int sext = (ck == CV_SEXT);
     Operand src32;
     CGLocal res;
     ApiSValue res_lv;
     Operand ar;
     Operand hival;
     if (sty != i32) {
       api_push(g, *v);
       api_cg_convert_kind(g, i32, ck == CV_SEXT ? CV_SEXT : CV_ZEXT);
       *v = api_pop(g);
     }
     src32 = api_force_local(g, v, i32);
     res = api_wide8_temp_local(g, dty);
     res_lv = api_make_lv(api_op_local(res, dty), dty);
     ar = api_lvalue_addr(g, &res_lv, cg_type_ptr_to(g->c, dty));
     api_wide8_store_lane(g, ar, lo, src32);
     if (sext)
       hival = wide8_i32_binop(g, BO_SHR_S, src32, api_op_imm(31, i32));
     else
       hival = api_op_imm(0, i32);
     api_wide8_store_lane(g, ar, hi, hival);
     api_release(g, v);
     api_push(g, api_make_sv(api_op_local(res, dty), dty));
     return 1;
   }
   /* s_wide: i64 -> narrower. _Bool is "any bit set"; else take the low lane and
    * truncate/convert further. */
   if (api_is_bool_type(g->c, dty)) {
     Operand orl = api_wide8_or_lanes(g, v, sty);
     api_release(g, v);
     api_push(g, api_make_sv(orl, i32));
     kit_cg_push_int(g, 0, i32);
     api_cg_cmp(g, CMP_NE);
     api_cg_convert_kind(g, dty, CV_TRUNC);
     return 1;
   }
   {
     Operand addr = api_wide8_addr(g, v, sty);
     Operand lolane = api_wide8_load_lane(g, addr, lo);
     api_release(g, v);
     api_push(g, api_make_sv(lolane, i32));
     if (dty != i32) api_cg_convert_kind(g, dty, CV_TRUNC);
     return 1;
   }
 }
 
 static void api_i128_unop(KitCg* g, UnOp iop) {
   KitCgTypeId i128 = builtin_id(KIT_CG_BUILTIN_I128);
   const char* name = NULL;
   ApiSValue args[1];
   KitCgTypeId ps[1];
   if (iop == UO_NEG)
     name = "__negti2";
   else if (iop == UO_BNOT)
     name = "__kit_notti3";
   else {
     compiler_panic(g->c, g->cur_loc, "KitCg: unsupported i128 unop");
     return;
   }
   args[0] = api_pop(g);
   ps[0] = i128;
   api_runtime_call_values(g, name, i128, ps, 1, args);
 }
 
 /* Map a relational op to the form used to compare a __kit_*cmpti2
  * result (-1/0/1, a signed i32) against zero. */
 static CmpOp api_i128_cmp_vs_zero(CmpOp cop) {
   switch (cop) {
     case CMP_EQ:
       return CMP_EQ;
     case CMP_NE:
       return CMP_NE;
     case CMP_LT_S:
     case CMP_LT_U:
       return CMP_LT_S;
     case CMP_LE_S:
     case CMP_LE_U:
       return CMP_LE_S;
     case CMP_GT_S:
     case CMP_GT_U:
       return CMP_GT_S;
     case CMP_GE_S:
     case CMP_GE_U:
       return CMP_GE_S;
     default:
       return CMP_NE;
   }
 }
 
 static void api_i128_cmp(KitCg* g, CmpOp cop) {
   KitCgTypeId i128 = builtin_id(KIT_CG_BUILTIN_I128);
   KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32);
   const char* name =
       api_i128_cmp_is_unsigned(cop) ? "__kit_ucmpti2" : "__kit_cmpti2";
   KitCgTypeId ps[2] = {i128, i128};
   ApiSValue args[2];
   args[1] = api_pop(g);
   args[0] = api_pop(g);
   api_runtime_call_values(g, name, i32, ps, 2, args);
   kit_cg_push_int(g, 0, i32);
   api_cg_cmp(g, api_i128_cmp_vs_zero(cop));
 }
 
 /* int<->i128 conversions. Returns 1 if it handled the conversion and
  * consumed *v, 0 to fall through to the generic path. */
 int api_try_i128_convert(KitCg* g, ConvKind ck, KitCgTypeId sty,
                          KitCgTypeId dty, ApiSValue* v) {
   KitCgTypeId i128 = builtin_id(KIT_CG_BUILTIN_I128);
   KitCgTypeId i64 = builtin_id(KIT_CG_BUILTIN_I64);
   int s_is_128 = api_is_i128_type(g->c, sty);
   int d_is_128 = api_is_i128_type(g->c, dty);
   if (!s_is_128 && !d_is_128) return 0;
   if (s_is_128 && d_is_128) {
     /* signed<->unsigned i128 reinterpret: identical layout. */
     v->type = dty;
     v->op.type = dty;
     api_push(g, *v);
     return 1;
   }
   if (d_is_128) {
     u32 sw = kit_cg_type_int_width((KitCompiler*)g->c, sty);
     const char* name = (ck == CV_SEXT) ? "__kit_sext64ti" : "__kit_zext64ti";
     ApiSValue arg;
     KitCgTypeId ps[1];
     if (sw == 0) return 0; /* float->i128 unsupported here */
     if (sw >= 64) {
       arg = *v;
       arg.type = i64;
       arg.op.type = i64;
     } else {
       api_push(g, *v);
       api_cg_convert_kind(g, i64, ck);
       arg = api_pop(g);
     }
     ps[0] = i64;
     api_runtime_call_values(g, name, i128, ps, 1, &arg);
     return 1;
   }
   /* s_is_128, dty is _Bool: "value != 0" over the full 128 bits, not a
    * low-lane truncation (a value whose only set bits are above bit 63 must
    * still become 1). Reuse the runtime i128 compare. */
   if (api_is_bool_type(g->c, dty)) {
     api_push(g, *v);
     kit_cg_push_int(g, 0, i128);
     api_i128_cmp(g, CMP_NE); /* leaves i32 0/1 */
     api_cg_convert_kind(g, dty, CV_TRUNC);
     return 1;
   }
   /* s_is_128, dty is a narrower integer: take the low 64 bits, then
    * truncate further if needed. */
   {
     u32 dw = kit_cg_type_int_width((KitCompiler*)g->c, dty);
     i32 lo_off = g->c->target.big_endian ? 8 : 0;
     Operand addr;
     Operand lo;
     if (dw == 0) return 0; /* i128->float unsupported here */
     addr = api_i128_addr(g, v);
     lo = api_i128_load_lane(g, addr, lo_off);
     api_release(g, v);
     if (dw >= 64) {
       api_push(g, api_make_sv(lo, dty));
     } else {
       api_push(g, api_make_sv(lo, i64));
       api_cg_convert_kind(g, dty, CV_TRUNC);
     }
     return 1;
   }
 }
 
 void kit_cg_int_binop(KitCg* g, KitCgIntBinOp op, uint32_t flags) {
   BinOp iop = api_map_int_binop(op);
   ApiConstValue result_const;
   KitCgTypeId ty = KIT_CG_TYPE_NONE;
   if (g && g->sp >= 2u) {
     ApiConstValue cb = api_const_at(g, 0);
     ApiConstValue ca = api_const_at(g, 1);
     ty = api_sv_type(&g->stack[g->sp - 2u]);
     api_const_fold_binop(g, iop, ty, ca, cb, flags, &result_const);
   } else {
     result_const = api_const_unknown(KIT_CG_TYPE_NONE);
   }
   if (api_unevaluated(g)) {
     api_cg_binop(g, iop, flags);
     return;
   }
   if (g && (api_i128_stack_top(g, 0) || api_i128_stack_top(g, 1))) {
     api_i128_binop(g, iop);
     api_const_set_top(g, result_const);
     return;
   }
   /* 64-bit int split into 32-bit lanes: mul/div/rem/shift become __*di3
    * runtime calls; add/sub/and/or/xor are emitted inline as 2-word lane ops
    * (no compiler-rt helper exists for them). Both keep the value
    * memory-resident so the allocator never tries to put 8 bytes in one 4-byte
    * value slot. */
   if (g && (api_wide64_stack_top(g, 0) || api_wide64_stack_top(g, 1))) {
     if (api_wideint64_binop_helper(iop))
       api_wideint64_binop(g, iop);
     else
       api_wide64_binop_inline(g, iop);
     api_const_set_top(g, result_const);
     return;
   }
   api_cg_binop(g, iop, flags);
 }
 
 void kit_cg_int_unop(KitCg* g, KitCgIntUnOp op, uint32_t flags) {
   UnOp iop = api_map_int_unop(op);
   ApiConstValue result_const;
   KitCgTypeId ty = KIT_CG_TYPE_NONE;
   if (g && g->sp >= 1u) {
     ApiConstValue ca = api_const_at(g, 0);
     ty = api_sv_type(&g->stack[g->sp - 1u]);
     api_const_fold_unop(g, iop, ty, ca, flags, &result_const);
   } else {
     result_const = api_const_unknown(KIT_CG_TYPE_NONE);
   }
   if (api_unevaluated(g)) {
     api_cg_unop(g, iop, flags);
     return;
   }
   if (g && api_i128_stack_top(g, 0) && (iop == UO_NEG || iop == UO_BNOT)) {
     api_i128_unop(g, iop);
     api_const_set_top(g, result_const);
     return;
   }
   /* Split 64-bit int: neg/bnot are inline 2-word lane ops; logical-not (!x) is
    * the full-value truthiness test (lo|hi)==0. */
   if (g && api_wide64_stack_top(g, 0)) {
     if (iop == UO_NEG || iop == UO_BNOT) {
       api_wide64_unop_inline(g, iop);
       api_const_set_top(g, result_const);
       return;
     }
     if (iop == UO_NOT) {
       KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32);
       ApiSValue v = api_pop(g);
       KitCgTypeId ty = v.type ? v.type : v.op.type;
       Operand orl = api_wide8_or_lanes(g, &v, ty);
       api_release(g, &v);
       api_push(g, api_make_sv(orl, i32));
       kit_cg_push_int(g, 0, i32);
       api_cg_cmp(g, CMP_EQ);
       api_cg_convert_kind(g, ty, CV_ZEXT);
       api_const_set_top(g, result_const);
       return;
     }
   }
   api_cg_unop(g, iop, flags);
 }
 
 void kit_cg_int_cmp(KitCg* g, KitCgIntCmpOp op) {
   CmpOp cop = api_map_int_cmp(op);
   ApiConstValue result_const;
   if (g && g->sp >= 2u) {
     ApiConstValue cb = api_const_at(g, 0);
     ApiConstValue ca = api_const_at(g, 1);
     api_const_fold_cmp(g, cop, ca, cb, &result_const);
   } else {
     result_const = api_const_unknown(builtin_id(KIT_CG_BUILTIN_I32));
   }
   if (api_unevaluated(g)) {
     api_cg_cmp(g, cop);
     return;
   }
   if (g && (api_i128_stack_top(g, 0) || api_i128_stack_top(g, 1))) {
     api_i128_cmp(g, cop);
     api_const_set_top(g, result_const);
     return;
   }
   if (g && (api_wide64_stack_top(g, 0) || api_wide64_stack_top(g, 1))) {
     api_wide64_cmp_inline(g, cop);
     api_const_set_top(g, result_const);
     return;
   }
   api_cg_cmp(g, cop);
 }
 
 const char* api_i128_binop_helper(BinOp op) {
   switch (op) {
     case BO_IADD:
       return "__kit_addti3";
     case BO_ISUB:
       return "__kit_subti3";
     case BO_IMUL:
       return "__multi3";
     case BO_SDIV:
       return "__divti3";
     case BO_UDIV:
       return "__udivti3";
     case BO_SREM:
       return "__modti3";
     case BO_UREM:
       return "__umodti3";
     case BO_AND:
       return "__kit_andti3";
     case BO_OR:
       return "__kit_orti3";
     case BO_XOR:
       return "__kit_xorti3";
     case BO_SHL:
       return "__ashlti3";
     case BO_SHR_U:
       return "__lshrti3";
     case BO_SHR_S:
       return "__ashrti3";
     case BO_FADD:
     case BO_FSUB:
     case BO_FMUL:
     case BO_FDIV:
     default:
       return NULL;
   }
 }
 
 int api_i128_cmp_is_unsigned(CmpOp op) {
   return op == CMP_LT_U || op == CMP_LE_U || op == CMP_GT_U || op == CMP_GE_U;
 }
 
 const char* api_f128_binop_helper(KitCgFpBinOp op) {
   switch (op) {
     case KIT_CG_FP_ADD:
       return "__addtf3";
     case KIT_CG_FP_SUB:
       return "__subtf3";
     case KIT_CG_FP_MUL:
       return "__multf3";
     case KIT_CG_FP_DIV:
       return "__divtf3";
   }
   return NULL;
 }
 
 /* Runtime helper name for double (f64) arithmetic on a target that lacks a
  * hardware double unit. Mirrors api_f128_binop_helper with the __*df3 names. */
 static const char* api_softdf_binop_helper(KitCgFpBinOp op) {
   switch (op) {
     case KIT_CG_FP_ADD:
       return "__adddf3";
     case KIT_CG_FP_SUB:
       return "__subdf3";
     case KIT_CG_FP_MUL:
       return "__muldf3";
     case KIT_CG_FP_DIV:
       return "__divdf3";
   }
   return NULL;
 }
 
 int api_f128_stack_top(KitCg* g, u32 depth) {
   return api_stack_wide_kind(g, depth) == WK_F128;
 }
 
 /* True when the target has no hardware double: float_abi is SOFT (ilp32/lp64,
  * no FP regs) or SINGLE (ilp32f/lp64f, only float in FP regs — double is always
  * soft). DOUBLE (rv64 lp64d) and DEFAULT (x64/aa64 hardware-double targets that
  * never set float_abi) keep the inline hardware path, so existing rv64/x64/aa64
  * codegen is unchanged. */
 static int api_target_double_is_soft(KitCg* g) {
   if (!g) return 0;
   return g->c->target.float_abi == KIT_FLOAT_ABI_SOFT ||
          g->c->target.float_abi == KIT_FLOAT_ABI_SINGLE;
 }
 
 /* True when ty is a 64-bit float (double) AND the target lacks hardware double.
  * f128 is handled by the separate api_f128_* path, so width must be exactly 64.
  */
 static int api_type_is_soft_double(KitCg* g, KitCgTypeId ty) {
   if (!api_target_double_is_soft(g)) return 0;
   return kit_cg_type_float_width((KitCompiler*)g->c, ty) == 64;
 }
 
 static int api_soft_double_stack_top(KitCg* g, u32 depth) {
   return api_stack_wide_kind(g, depth) == WK_SOFT_DOUBLE;
 }
 
 /* f32 under pure-soft ilp32/lp64 (float_abi SOFT, no FP unit): single-precision
  * arithmetic/compare/convert is also a libcall. Under SINGLE (ilp32f) float is
  * hardware (fadd.s etc.) so this is false; DOUBLE/DEFAULT keep hardware too. */
 static int api_type_is_soft_single(KitCg* g, KitCgTypeId ty) {
   if (!g || g->c->target.float_abi != KIT_FLOAT_ABI_SOFT) return 0;
   return kit_cg_type_float_width((KitCompiler*)g->c, ty) == 32;
 }
 
 static int api_soft_single_stack_top(KitCg* g, u32 depth) {
   return api_stack_wide_kind(g, depth) == WK_SOFT_SINGLE;
 }
 
 /* Runtime helper for f32 arithmetic on a soft-float target (mirrors
  * api_softdf_binop_helper with the __*sf3 names). */
 static const char* api_softsf_binop_helper(KitCgFpBinOp op) {
   switch (op) {
     case KIT_CG_FP_ADD:
       return "__addsf3";
     case KIT_CG_FP_SUB:
       return "__subsf3";
     case KIT_CG_FP_MUL:
       return "__mulsf3";
     case KIT_CG_FP_DIV:
       return "__divsf3";
   }
   return NULL;
 }
 
 /* Soft-float binary arithmetic via a single libcall `name(a,b)`, both operands
  * (and the result) of type `opty`. Consumes the two operands on the stack and
  * pushes the result. Shared by the f128 (tf), soft-double (df) and soft-single
  * (sf) paths — only the helper name and operand type differ; the runtime ABI
  * (two same-type args, same-type result) is width-neutral. `what` names the
  * width in the panic raised when `op` has no helper. Mirrors api_softfp_cmp. */
 static void api_softfp_binop(KitCg* g, const char* name, KitCgTypeId opty,
                              const char* what) {
   KitCgTypeId ps[2];
   ApiSValue args[2];
   if (!name)
     compiler_panic(g->c, g->cur_loc, "KitCg: unsupported %s binop", what);
   args[1] = api_pop(g);
   args[0] = api_pop(g);
   ps[0] = opty;
   ps[1] = opty;
   api_runtime_call_values(g, name, opty, ps, 2, args);
 }
 
 void api_f128_call_unary(KitCg* g, const char* name, KitCgTypeId ret,
                          KitCgTypeId param) {
   ApiSValue args[1];
   KitCgTypeId ps[1];
   args[0] = api_pop(g);
   ps[0] = param;
   api_runtime_call_values(g, name, ret, ps, 1, args);
 }
 
 void kit_cg_fp_binop(KitCg* g, KitCgFpBinOp op, uint32_t flags) {
   (void)flags;
   if (api_unevaluated(g)) {
     api_cg_binop(g, api_map_fp_binop(op), 0);
     return;
   }
   if (api_f128_stack_top(g, 0) || api_f128_stack_top(g, 1)) {
     api_softfp_binop(g, api_f128_binop_helper(op),
                      builtin_id(KIT_CG_BUILTIN_F128), "f128");
     return;
   }
   if (api_soft_double_stack_top(g, 0) || api_soft_double_stack_top(g, 1)) {
     api_softfp_binop(g, api_softdf_binop_helper(op),
                      builtin_id(KIT_CG_BUILTIN_F64), "soft double");
     return;
   }
   if (api_soft_single_stack_top(g, 0) || api_soft_single_stack_top(g, 1)) {
     api_softfp_binop(g, api_softsf_binop_helper(op),
                      builtin_id(KIT_CG_BUILTIN_F32), "soft single");
     return;
   }
   api_cg_binop(g, api_map_fp_binop(op), 0);
 }
 
 void kit_cg_fp_unop(KitCg* g, KitCgFpUnOp op, uint32_t flags) {
   (void)flags;
   if (!g) return;
   CG_REQUIRE(g, op == KIT_CG_FP_NEG, "KitCg: FP unary op unsupported");
   if (api_unevaluated(g)) {
     api_cg_unop(g, UO_FNEG, 0);
     return;
   }
   if (api_f128_stack_top(g, 0)) {
     KitCgTypeId f128 = builtin_id(KIT_CG_BUILTIN_F128);
     api_f128_call_unary(g, "__negtf2", f128, f128);
     return;
   }
   /* Soft float has no FP unit, so negation is a libcall too (the inline FNEG
    * path emits fsgnj on an FP register, which does not exist here). */
   if (api_soft_double_stack_top(g, 0)) {
     KitCgTypeId f64 = builtin_id(KIT_CG_BUILTIN_F64);
     api_f128_call_unary(g, "__negdf2", f64, f64);
     return;
   }
   if (api_soft_single_stack_top(g, 0)) {
     KitCgTypeId f32 = builtin_id(KIT_CG_BUILTIN_F32);
     api_f128_call_unary(g, "__negsf2", f32, f32);
     return;
   }
   api_cg_unop(g, UO_FNEG, 0);
 }
 
 /* Soft-float single-libcall comparison: call `name(a,b)` (both operands of type
  * `opty`) and test its i32 three-way result against 0 with `icmp`. Consumes the
  * two operands on the stack and pushes the i32 boolean. Shared by the f128 (tf)
  * and soft-double (df) paths — only the helper name and operand type differ;
  * the compiler-rt NaN-sign convention is identical for both. */
 static void api_softfp_cmp_call(KitCg* g, const char* name, KitCgTypeId opty,
                                 CmpOp icmp) {
   KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32);
   KitCgTypeId ps[2];
   ApiSValue args[2];
   ps[0] = opty;
   ps[1] = opty;
   args[1] = api_pop(g);
   args[0] = api_pop(g);
   api_runtime_call_values(g, name, i32, ps, 2, args);
   kit_cg_push_int(g, 0, i32);
   api_cg_cmp(g, icmp);
 }
 
 /* UEQ and ONE are the only soft-float predicates that cannot be a single
  * libcall: "equal" and "unordered" both yield a nonzero magnitude from
  * __eq*2/__ne*2, so they need a separate __unord*2 to split them.
  *   UEQ = (__eq*2(a,b) == 0) || (__unord*2(a,b) != 0)
  *   ONE = (__ne*2(a,b) != 0) && (__unord*2(a,b) == 0)
  * `suffix` is "tf" (f128) or "df" (double); `opty` the matching operand type.
  * The operands are dup'd (kit_cg_dup copies into a fresh owned local) so each
  * libcall consumes its own copy. */
 static void api_softfp_cmp_with_unord(KitCg* g, KitCgFpCmpOp op,
                                       const char* suffix, KitCgTypeId opty) {
   char relname[16];
   char unordname[16];
   CmpOp relcmp = (op == KIT_CG_FP_UEQ) ? CMP_EQ : CMP_NE;
   const char* rel = (op == KIT_CG_FP_UEQ) ? "eq" : "ne";
   snprintf(relname, sizeof relname, "__%s%s2", rel, suffix);
   snprintf(unordname, sizeof unordname, "__unord%s2", suffix);
   /* [a, b] -> [a, b, a, b] */
   kit_cg_dup2(g);
   /* relation on the top (dup'd) copy: [a, b, R] */
   api_softfp_cmp_call(g, relname, opty, relcmp);
   /* bring the original a, b back to TOS with R underneath: [R, a, b] */
   kit_cg_rot3(g);
   kit_cg_rot3(g);
   if (op == KIT_CG_FP_UEQ) {
     api_softfp_cmp_call(g, unordname, opty, CMP_NE); /* [R, unordered?] */
     api_cg_binop(g, BO_OR, 0);                       /* R || unordered */
   } else {
     api_softfp_cmp_call(g, unordname, opty, CMP_EQ); /* [R, ordered?] */
     api_cg_binop(g, BO_AND, 0);                      /* R && ordered */
   }
 }
 
 /* Emit a soft-float comparison for either f128 (suffix "tf", opty f128) or
  * soft double (suffix "df", opty f64). The predicate->helper mapping and the
  * compiler-rt NaN-sign convention are XLEN/width-neutral, so a single body
  * serves both — only the suffix and operand type vary. */
 static void api_softfp_cmp(KitCg* g, KitCgFpCmpOp op, const char* suffix,
                            KitCgTypeId opty) {
   char name[16];
   switch (op) {
     case KIT_CG_FP_OEQ:
       snprintf(name, sizeof name, "__eq%s2", suffix);
       api_softfp_cmp_call(g, name, opty, CMP_EQ);
       return;
     case KIT_CG_FP_UNE:
       snprintf(name, sizeof name, "__ne%s2", suffix);
       api_softfp_cmp_call(g, name, opty, CMP_NE);
       return;
     case KIT_CG_FP_OLT:
       snprintf(name, sizeof name, "__lt%s2", suffix);
       api_softfp_cmp_call(g, name, opty, CMP_LT_S);
       return;
     case KIT_CG_FP_OLE:
       snprintf(name, sizeof name, "__le%s2", suffix);
       api_softfp_cmp_call(g, name, opty, CMP_LE_S);
       return;
     case KIT_CG_FP_OGT:
       snprintf(name, sizeof name, "__gt%s2", suffix);
       api_softfp_cmp_call(g, name, opty, CMP_GT_S);
       return;
     case KIT_CG_FP_OGE:
       snprintf(name, sizeof name, "__ge%s2", suffix);
       api_softfp_cmp_call(g, name, opty, CMP_GE_S);
       return;
     /* unordered duals via the opposite-sign helper (NaN flips the test): */
     case KIT_CG_FP_UGE:
       snprintf(name, sizeof name, "__lt%s2", suffix);
       api_softfp_cmp_call(g, name, opty, CMP_GE_S);
       return;
     case KIT_CG_FP_UGT:
       snprintf(name, sizeof name, "__le%s2", suffix);
       api_softfp_cmp_call(g, name, opty, CMP_GT_S);
       return;
     case KIT_CG_FP_ULT:
       snprintf(name, sizeof name, "__ge%s2", suffix);
       api_softfp_cmp_call(g, name, opty, CMP_LT_S);
       return;
     case KIT_CG_FP_ULE:
       snprintf(name, sizeof name, "__gt%s2", suffix);
       api_softfp_cmp_call(g, name, opty, CMP_LE_S);
       return;
     case KIT_CG_FP_UEQ:
     case KIT_CG_FP_ONE:
       api_softfp_cmp_with_unord(g, op, suffix, opty);
       return;
   }
 }
 
 void kit_cg_fp_cmp(KitCg* g, KitCgFpCmpOp op) {
   /* f128/long double and soft double are both soft-float: the comparison is a
    * libcall returning a three-way i32 we test against 0. kit's runtime uses the
    * standard compiler-rt sign convention (rt/lib/impl/fp_compare_impl.inc):
    *   __le-family (__eq*2/__ne*2/__lt*2/__le*2): NaN -> +1
    *   __ge-family (__ge*2/__gt*2):               NaN -> -1
    * so each ordered predicate AND its unordered dual maps to one libcall,
    * choosing the helper whose NaN sign makes the integer test fall the right
    * way (ordered: NaN must fail; unordered: NaN must pass). Only UEQ/ONE, which
    * must split "equal" from "unordered", need a second (__unord*2) call. The
    * convention is width-neutral, so the same logic drives the tf and df
    * suffixes via api_softfp_cmp. */
   if (api_unevaluated(g)) {
     api_cg_cmp(g, api_map_fp_cmp(op));
     return;
   }
   if (api_f128_stack_top(g, 0) || api_f128_stack_top(g, 1)) {
     api_softfp_cmp(g, op, "tf", builtin_id(KIT_CG_BUILTIN_F128));
     return;
   }
   if (api_soft_double_stack_top(g, 0) || api_soft_double_stack_top(g, 1)) {
     api_softfp_cmp(g, op, "df", builtin_id(KIT_CG_BUILTIN_F64));
     return;
   }
   if (api_soft_single_stack_top(g, 0) || api_soft_single_stack_top(g, 1)) {
     api_softfp_cmp(g, op, "sf", builtin_id(KIT_CG_BUILTIN_F32));
     return;
   }
   api_cg_cmp(g, api_map_fp_cmp(op));
 }
 
 void kit_cg_sext(KitCg* g, KitCgTypeId dst) {
   api_cg_convert_kind(g, dst, CV_SEXT);
 }
 
 void kit_cg_zext(KitCg* g, KitCgTypeId dst) {
   api_cg_convert_kind(g, dst, CV_ZEXT);
 }
 
 void kit_cg_trunc(KitCg* g, KitCgTypeId dst) {
   api_cg_convert_kind(g, dst, CV_TRUNC);
 }
 
 void kit_cg_ptr_to_int(KitCg* g, KitCgTypeId dst) {
   api_cg_convert_kind(g, dst, CV_BITCAST);
 }
 
 void kit_cg_int_to_ptr(KitCg* g, KitCgTypeId dst) {
   api_cg_convert_kind(g, dst, CV_BITCAST);
 }
 
 void kit_cg_bitcast(KitCg* g, KitCgTypeId dst) {
   api_cg_convert_kind(g, dst, CV_BITCAST);
 }
 
 void kit_cg_fpext(KitCg* g, KitCgTypeId dst) {
   /* The f128 / soft-double predicates below validate `dst` internally (and the
    * fall-through reuses `dst` directly), so the standalone resolve_type is
    * redundant: for a valid id resolve_type(dst)==dst, for an invalid id every
    * predicate is false and the fall-through validates. */
   KitCgTypeId dty = dst;
   if (!g) return;
   if (api_unevaluated(g)) {
     api_cg_convert_kind(g, dst, CV_FEXT);
     return;
   }
   if (api_is_f128_type(g->c, dty)) {
     ApiSValue v = api_pop(g);
     KitCgTypeId sty = api_sv_type(&v);
     const char* name = sty == builtin_id(KIT_CG_BUILTIN_F32) ? "__extendsftf2"
                                                              : "__extenddftf2";
     api_push(g, v);
     api_f128_call_unary(g, name, dty, sty);
     return;
   }
   /* float -> soft double: runtime widen via __extendsfdf2. */
   if (api_type_is_soft_double(g, dty)) {
     ApiSValue v = api_pop(g);
     KitCgTypeId sty = api_sv_type(&v);
     api_push(g, v);
     api_f128_call_unary(g, "__extendsfdf2", dty, sty);
     return;
   }
   api_cg_convert_kind(g, dst, CV_FEXT);
 }
 
 void kit_cg_fptrunc(KitCg* g, KitCgTypeId dst) {
   /* resolve_type(dst)==dst for any valid id; the predicates / fall-through
    * validate, so drop the redundant standalone validation (see kit_cg_fpext).
    */
   KitCgTypeId dty = dst;
   if (!g) return;
   if (api_unevaluated(g)) {
     api_cg_convert_kind(g, dst, CV_FTRUNC);
     return;
   }
   if (api_f128_stack_top(g, 0)) {
     ApiSValue v = api_pop(g);
     KitCgTypeId f128 = builtin_id(KIT_CG_BUILTIN_F128);
     const char* name =
         dty == builtin_id(KIT_CG_BUILTIN_F32) ? "__trunctfsf2" : "__trunctfdf2";
     api_push(g, v);
     api_f128_call_unary(g, name, dty, f128);
     return;
   }
   /* soft double -> float: runtime narrow via __truncdfsf2. */
   if (api_soft_double_stack_top(g, 0)) {
     ApiSValue v = api_pop(g);
     KitCgTypeId sty = api_sv_type(&v);
     api_push(g, v);
     api_f128_call_unary(g, "__truncdfsf2", dty, sty);
     return;
   }
   api_cg_convert_kind(g, dst, CV_FTRUNC);
 }
 
 /* The integer builtin a soft-float int<->float conversion uses for an operand
  * whose ABI size is `sz` bytes: i128 (>8), i64 (>4), else i32. Shared by all
  * eight conversion blocks (the same width ladder appeared in each). */
 static KitCgTypeId api_int_builtin_for_size(u32 sz) {
   return sz > 8 ? builtin_id(KIT_CG_BUILTIN_I128)
                 : (sz > 4 ? builtin_id(KIT_CG_BUILTIN_I64)
                           : builtin_id(KIT_CG_BUILTIN_I32));
 }
 
 /* The compiler-rt integer-width suffix matching api_int_builtin_for_size:
  * "ti" (i128), "di" (i64), "si" (i32). */
 static const char* api_int_suffix_for_size(u32 sz) {
   return sz > 8 ? "ti" : (sz > 4 ? "di" : "si");
 }
 
 /* The four width-laddered soft-float int<->float conversion families. The
  * compiler-rt name is a fixed prefix, the width-derived integer suffix
  * (api_int_suffix_for_size) and a fixed float suffix ("tf" for f128, "df" for
  * soft double); int->float places the integer suffix before the float suffix,
  * float->int after it. */
 typedef enum ApiFpConvOp {
   API_FPCONV_SINT_TO_FLOAT, /* __float<int><flt>   */
   API_FPCONV_UINT_TO_FLOAT, /* __floatun<int><flt> */
   API_FPCONV_FLOAT_TO_SINT, /* __fix<flt><int>     */
   API_FPCONV_FLOAT_TO_UINT, /* __fixuns<flt><int>  */
 } ApiFpConvOp;
 
 typedef struct FpConvDesc {
   const char* prefix;
   int int_first; /* 1: prefix+int+flt (int->float); 0: prefix+flt+int */
 } FpConvDesc;
 
 static const FpConvDesc kFpConvTable[] = {
     [API_FPCONV_SINT_TO_FLOAT] = {"__float", 1},
     [API_FPCONV_UINT_TO_FLOAT] = {"__floatun", 1},
     [API_FPCONV_FLOAT_TO_SINT] = {"__fix", 0},
     [API_FPCONV_FLOAT_TO_UINT] = {"__fixuns", 0},
 };
 
 /* Build the compiler-rt conversion libcall name for `op` with the given integer
  * width (`sz` bytes) and float suffix `flt` ("tf"/"df") into `buf`. The result
  * is byte-identical to the names the eight blocks previously open-coded. */
 static void api_fp_conv_name(char* buf, size_t cap, ApiFpConvOp op,
                              const char* flt, u32 sz) {
   const FpConvDesc* d = &kFpConvTable[op];
   const char* is = api_int_suffix_for_size(sz);
   if (d->int_first)
     snprintf(buf, cap, "%s%s%s", d->prefix, is, flt);
   else
     snprintf(buf, cap, "%s%s%s", d->prefix, flt, is);
 }
 
 void kit_cg_sint_to_float(KitCg* g, KitCgTypeId dst, KitCgRounding rounding) {
   (void)rounding;
   if (api_unevaluated(g)) {
     api_cg_convert_kind(g, dst, CV_ITOF_S);
     return;
   }
   if (api_is_f128_type(g->c, dst)) {
     ApiSValue v = api_pop(g);
     KitCgTypeId sty = api_sv_type(&v);
     u32 sz = (u32)abi_cg_sizeof(g->c->abi, sty);
     char name[16];
     api_fp_conv_name(name, sizeof name, API_FPCONV_SINT_TO_FLOAT, "tf", sz);
     api_push(g, v);
     api_f128_call_unary(g, name, dst, api_int_builtin_for_size(sz));
     return;
   }
   /* signed int -> soft double: __floatsidf (i32) / __floatdidf (i64). */
   if (api_type_is_soft_double(g, dst)) {
     ApiSValue v = api_pop(g);
     KitCgTypeId sty = api_sv_type(&v);
     u32 sz = (u32)abi_cg_sizeof(g->c->abi, sty);
     char name[16];
     api_fp_conv_name(name, sizeof name, API_FPCONV_SINT_TO_FLOAT, "df", sz);
     api_push(g, v);
     api_f128_call_unary(g, name, dst, api_int_builtin_for_size(sz));
     return;
   }
   /* signed split-i64 -> hardware single float: use __floatdisf. */
   if (api_wide64_stack_top(g, 0)) {
     api_f128_call_unary(g, "__floatdisf", dst, builtin_id(KIT_CG_BUILTIN_I64));
     return;
   }
   /* i32 -> soft single float (ilp32, no FPU): __floatsisf. */
   if (api_type_is_soft_single(g, dst)) {
     api_f128_call_unary(g, "__floatsisf", dst, builtin_id(KIT_CG_BUILTIN_I32));
     return;
   }
   api_cg_convert_kind(g, dst, CV_ITOF_S);
 }
 
 void kit_cg_uint_to_float(KitCg* g, KitCgTypeId dst, KitCgRounding rounding) {
   (void)rounding;
   if (api_unevaluated(g)) {
     api_cg_convert_kind(g, dst, CV_ITOF_U);
     return;
   }
   if (api_is_f128_type(g->c, dst)) {
     ApiSValue v = api_pop(g);
     KitCgTypeId sty = api_sv_type(&v);
     u32 sz = (u32)abi_cg_sizeof(g->c->abi, sty);
     char name[16];
     api_fp_conv_name(name, sizeof name, API_FPCONV_UINT_TO_FLOAT, "tf", sz);
     api_push(g, v);
     api_f128_call_unary(g, name, dst, api_int_builtin_for_size(sz));
     return;
   }
   /* unsigned int -> soft double: __floatunsidf (i32) / __floatundidf (i64). */
   if (api_type_is_soft_double(g, dst)) {
     ApiSValue v = api_pop(g);
     KitCgTypeId sty = api_sv_type(&v);
     u32 sz = (u32)abi_cg_sizeof(g->c->abi, sty);
     char name[16];
     api_fp_conv_name(name, sizeof name, API_FPCONV_UINT_TO_FLOAT, "df", sz);
     api_push(g, v);
     api_f128_call_unary(g, name, dst, api_int_builtin_for_size(sz));
     return;
   }
   /* unsigned i64 -> hardware single float: __floatundisf. */
   if (api_wide64_stack_top(g, 0)) {
     api_f128_call_unary(g, "__floatundisf", dst,
                         builtin_id(KIT_CG_BUILTIN_I64));
     return;
   }
   /* u32 -> soft single float (ilp32, no FPU): __floatunsisf. */
   if (api_type_is_soft_single(g, dst)) {
     api_f128_call_unary(g, "__floatunsisf", dst,
                         builtin_id(KIT_CG_BUILTIN_I32));
     return;
   }
   api_cg_convert_kind(g, dst, CV_ITOF_U);
 }
 
 void kit_cg_float_to_sint(KitCg* g, KitCgTypeId dst, KitCgRounding rounding) {
   (void)rounding;
   if (api_unevaluated(g)) {
     api_cg_convert_kind(g, dst, CV_FTOI_S);
     return;
   }
   if (api_f128_stack_top(g, 0)) {
     KitCgTypeId dty = dst;
     u32 sz = (u32)abi_cg_sizeof(g->c->abi, dty);
     KitCgTypeId rty = api_int_builtin_for_size(sz);
     char name[16];
     api_fp_conv_name(name, sizeof name, API_FPCONV_FLOAT_TO_SINT, "tf", sz);
     api_f128_call_unary(g, name, rty, builtin_id(KIT_CG_BUILTIN_F128));
     if (rty != dty) api_cg_convert_kind(g, dty, CV_TRUNC);
     return;
   }
   /* soft double -> signed int: __fixdfsi (i32) / __fixdfdi (i64). */
   if (api_soft_double_stack_top(g, 0)) {
     KitCgTypeId dty = dst;
     KitCgTypeId f64 = builtin_id(KIT_CG_BUILTIN_F64);
     u32 sz = (u32)abi_cg_sizeof(g->c->abi, dty);
     KitCgTypeId rty = api_int_builtin_for_size(sz);
     char name[16];
     api_fp_conv_name(name, sizeof name, API_FPCONV_FLOAT_TO_SINT, "df", sz);
     api_f128_call_unary(g, name, rty, f64);
     if (rty != dty) api_cg_convert_kind(g, dty, CV_TRUNC);
     return;
   }
   /* hardware single float -> split-i64: use __fixsfdi. */
   if (api_is_wide8_scalar_type(g->c, dst)) {
     api_f128_call_unary(g, "__fixsfdi", dst, builtin_id(KIT_CG_BUILTIN_F32));
     return;
   }
   /* soft single float -> signed int <=32 (ilp32, no FPU): __fixsfsi. */
   if (api_soft_single_stack_top(g, 0)) {
     KitCgTypeId dty = dst;
     KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32);
     api_f128_call_unary(g, "__fixsfsi", i32, builtin_id(KIT_CG_BUILTIN_F32));
     if (i32 != dty) api_cg_convert_kind(g, dty, CV_TRUNC);
     return;
   }
   api_cg_convert_kind(g, dst, CV_FTOI_S);
 }
 
 void kit_cg_float_to_uint(KitCg* g, KitCgTypeId dst, KitCgRounding rounding) {
   (void)rounding;
   if (api_unevaluated(g)) {
     api_cg_convert_kind(g, dst, CV_FTOI_U);
     return;
   }
   if (api_f128_stack_top(g, 0)) {
     KitCgTypeId dty = dst;
     u32 sz = (u32)abi_cg_sizeof(g->c->abi, dty);
     KitCgTypeId rty = api_int_builtin_for_size(sz);
     char name[16];
     api_fp_conv_name(name, sizeof name, API_FPCONV_FLOAT_TO_UINT, "tf", sz);
     api_f128_call_unary(g, name, rty, builtin_id(KIT_CG_BUILTIN_F128));
     if (rty != dty) api_cg_convert_kind(g, dty, CV_TRUNC);
     return;
   }
   /* soft double -> unsigned int: __fixunsdfsi (i32) / __fixunsdfdi (i64). */
   if (api_soft_double_stack_top(g, 0)) {
     KitCgTypeId dty = dst;
     KitCgTypeId f64 = builtin_id(KIT_CG_BUILTIN_F64);
     u32 sz = (u32)abi_cg_sizeof(g->c->abi, dty);
     KitCgTypeId rty = api_int_builtin_for_size(sz);
     char name[16];
     api_fp_conv_name(name, sizeof name, API_FPCONV_FLOAT_TO_UINT, "df", sz);
     api_f128_call_unary(g, name, rty, f64);
     if (rty != dty) api_cg_convert_kind(g, dty, CV_TRUNC);
     return;
   }
   /* hardware single float -> split-u64: use __fixunssfdi. */
   if (api_is_wide8_scalar_type(g->c, dst)) {
     api_f128_call_unary(g, "__fixunssfdi", dst, builtin_id(KIT_CG_BUILTIN_F32));
     return;
   }
   /* soft single float -> unsigned int <=32 (ilp32, no FPU): __fixunssfsi. */
   if (api_soft_single_stack_top(g, 0)) {
     KitCgTypeId dty = dst;
     KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32);
     api_f128_call_unary(g, "__fixunssfsi", i32, builtin_id(KIT_CG_BUILTIN_F32));
     if (i32 != dty) api_cg_convert_kind(g, dty, CV_TRUNC);
     return;
   }
   api_cg_convert_kind(g, dst, CV_FTOI_U);
 }
 
 /* ============================================================
  * Intrinsics (stub)
  * ============================================================ */
 
 /* One descriptor per KitCgIntrinsic, indexed by the enum value. The four
  * accessors below are field reads off this single source of truth; unmapped
  * intrinsics (FMA/cache/coro) use an INTRIN_NONE row. The table is laid
  * out in enum order; the _Static_assert guards its length so a new enumerator
  * is a compile error rather than a silently truncated index. */
 typedef struct IntrinDesc {
   IntrinKind kind;
   const char* name;
   bool is_void;
   bool is_overflow;
 } IntrinDesc;
 
 static const IntrinDesc kIntrinTable[] = {
     [KIT_CG_INTRIN_TRAP] = {INTRIN_TRAP, "trap", true, false},
     [KIT_CG_INTRIN_CLZ] = {INTRIN_CLZ, "clz", false, false},
     [KIT_CG_INTRIN_CTZ] = {INTRIN_CTZ, "ctz", false, false},
     [KIT_CG_INTRIN_POPCOUNT] = {INTRIN_POPCOUNT, "popcount", false, false},
     [KIT_CG_INTRIN_BSWAP] = {INTRIN_BSWAP, "bswap", false, false},
     [KIT_CG_INTRIN_SETJMP] = {INTRIN_SETJMP, "setjmp", false, false},
     [KIT_CG_INTRIN_LONGJMP] = {INTRIN_LONGJMP, "longjmp", true, false},
     [KIT_CG_INTRIN_SADD_OVERFLOW] = {INTRIN_SADD_OVERFLOW, "sadd_overflow",
                                      false, true},
     [KIT_CG_INTRIN_UADD_OVERFLOW] = {INTRIN_UADD_OVERFLOW, "uadd_overflow",
                                      false, true},
     [KIT_CG_INTRIN_SSUB_OVERFLOW] = {INTRIN_SSUB_OVERFLOW, "ssub_overflow",
                                      false, true},
     [KIT_CG_INTRIN_USUB_OVERFLOW] = {INTRIN_USUB_OVERFLOW, "usub_overflow",
                                      false, true},
     [KIT_CG_INTRIN_SMUL_OVERFLOW] = {INTRIN_SMUL_OVERFLOW, "smul_overflow",
                                      false, true},
     [KIT_CG_INTRIN_UMUL_OVERFLOW] = {INTRIN_UMUL_OVERFLOW, "umul_overflow",
                                      false, true},
     [KIT_CG_INTRIN_FMA] = {INTRIN_NONE, "fma", false, false},
     [KIT_CG_INTRIN_PREFETCH] = {INTRIN_PREFETCH, "prefetch", true, false},
     [KIT_CG_INTRIN_EXPECT] = {INTRIN_EXPECT, "expect", false, false},
     [KIT_CG_INTRIN_ASSUME_ALIGNED] = {INTRIN_ASSUME_ALIGNED, "assume_aligned",
                                       false, false},
     [KIT_CG_INTRIN_SYSCALL] = {INTRIN_SYSCALL, "syscall", false, false},
     [KIT_CG_INTRIN_IRQ_SAVE] = {INTRIN_IRQ_SAVE, "irq_save", false, false},
     [KIT_CG_INTRIN_IRQ_RESTORE] = {INTRIN_IRQ_RESTORE, "irq_restore", true,
                                    false},
     [KIT_CG_INTRIN_IRQ_DISABLE] = {INTRIN_IRQ_DISABLE, "irq_disable", true,
                                    false},
     [KIT_CG_INTRIN_IRQ_ENABLE] = {INTRIN_IRQ_ENABLE, "irq_enable", true, false},
     [KIT_CG_INTRIN_DMB] = {INTRIN_DMB, "dmb", true, false},
     [KIT_CG_INTRIN_DSB] = {INTRIN_DSB, "dsb", true, false},
     [KIT_CG_INTRIN_ISB] = {INTRIN_ISB, "isb", true, false},
     [KIT_CG_INTRIN_DCACHE_CLEAN] = {INTRIN_NONE, "dcache_clean", false, false},
     [KIT_CG_INTRIN_DCACHE_INVALIDATE] = {INTRIN_NONE, "dcache_invalidate",
                                          false, false},
     [KIT_CG_INTRIN_DCACHE_CLEAN_INVALIDATE] = {INTRIN_NONE,
                                                "dcache_clean_invalidate", false,
                                                false},
     [KIT_CG_INTRIN_ICACHE_INVALIDATE] = {INTRIN_NONE, "icache_invalidate",
                                          false, false},
     [KIT_CG_INTRIN_CPU_NOP] = {INTRIN_CPU_NOP, "cpu_nop", true, false},
     [KIT_CG_INTRIN_CPU_YIELD] = {INTRIN_CPU_YIELD, "cpu_yield", true, false},
     [KIT_CG_INTRIN_WFI] = {INTRIN_WFI, "wfi", true, false},
     [KIT_CG_INTRIN_WFE] = {INTRIN_WFE, "wfe", true, false},
     [KIT_CG_INTRIN_SEV] = {INTRIN_SEV, "sev", true, false},
     [KIT_CG_INTRIN_CORO_SWITCH] = {INTRIN_NONE, "coro_switch", false, false},
     [KIT_CG_INTRIN_FRAME_ADDRESS] = {INTRIN_FRAME_ADDRESS, "frame_address",
                                      false, false},
     [KIT_CG_INTRIN_RETURN_ADDRESS] = {INTRIN_RETURN_ADDRESS, "return_address",
                                       false, false},
     [KIT_CG_INTRIN_READCYCLECOUNTER] = {INTRIN_READCYCLECOUNTER,
                                         "readcyclecounter", false, false},
     [KIT_CG_INTRIN_SMUL_HIGH] = {INTRIN_SMUL_HIGH, "smul_high", false, false},
     [KIT_CG_INTRIN_UMUL_HIGH] = {INTRIN_UMUL_HIGH, "umul_high", false, false},
 };
 
 _Static_assert(sizeof(kIntrinTable) / sizeof(kIntrinTable[0]) ==
                    KIT_CG_INTRIN_UMUL_HIGH + 1,
                "kIntrinTable must have exactly one row per KitCgIntrinsic");
 
 /* Bounds-guarded row lookup: an out-of-range intrinsic falls back to the NONE
  * row, preserving the defensive `default:` behavior the four switches carried
  * before they collapsed into kIntrinTable. */
 static const IntrinDesc* intrin_desc(KitCgIntrinsic intrin) {
   static const IntrinDesc none = {INTRIN_NONE, NULL, false, false};
   unsigned i = (unsigned)intrin;
   return i < sizeof(kIntrinTable) / sizeof(kIntrinTable[0]) ? &kIntrinTable[i]
                                                             : &none;
 }
 
 IntrinKind api_map_intrinsic(KitCg* g, KitCgIntrinsic intrin,
                              KitCgTypeId result_type) {
   /* Width-by-type: backends derive operand width from the result type, so the
    * mapping no longer needs the size here. */
   (void)g;
   (void)result_type;
   return intrin_desc(intrin)->kind;
 }
 
 int api_intrinsic_is_void(KitCgIntrinsic intrin) {
   return intrin_desc(intrin)->is_void;
 }
 
 int api_intrinsic_is_overflow(KitCgIntrinsic intrin) {
   return intrin_desc(intrin)->is_overflow;
 }
 
 const char* api_intrinsic_name(KitCgIntrinsic intrin) {
   const char* name = intrin_desc(intrin)->name;
   return name ? name : "intrinsic";
 }
 
 void kit_cg_intrinsic(KitCg* g, KitCgIntrinsic intrin, uint32_t nargs,
                       KitCgTypeId result_type) {
   CgTarget* T;
   KitCgTypeId rty;
   KitCgTypeId int_ty;
   IntrinKind kind;
   ApiSValue* svs;
   Operand* args;
   Operand dsts[2];
   u32 ndst = 0;
   Heap* h;
   if (!g) return;
   if (api_unevaluated(g)) {
     KitCgTypeId rty_u = resolve_type(g->c, result_type);
     for (u32 i = 0; i < nargs; ++i) {
       ApiSValue sv = api_pop(g);
       api_release(g, &sv);
     }
     if (api_intrinsic_is_overflow(intrin)) {
       KitCgTypeId bool_ty = builtin_id(KIT_CG_BUILTIN_BOOL);
       if (!rty_u) rty_u = builtin_id(KIT_CG_BUILTIN_I32);
       api_push(g, api_uneval_value(g, rty_u));
       api_const_set_top(g, api_const_unknown(rty_u));
       api_push(g, api_uneval_value(g, bool_ty));
       api_const_set_top(g, api_const_unknown(bool_ty));
     } else if (!api_intrinsic_is_void(intrin) &&
                (rty_u && !cg_type_is_void(g->c, rty_u))) {
       api_push(g, api_uneval_value(g, rty_u));
       api_const_set_top(g, api_const_unknown(rty_u));
     }
     return;
   }
   /* readcyclecounter returns a 64-bit value. On rv32 that is a register pair
    * the single-register native intrinsic path can't carry, and the read spans
    * the cycle/cycleh CSRs with a re-read loop, so route to the libkit_rt helper
    * (mirrors the wide-64 clz/ctz routing below). rv64 reads it inline. */
   if (intrin == KIT_CG_INTRIN_READCYCLECOUNTER &&
       g->c->target.arch == KIT_ARCH_RV32) {
     api_runtime_call_values(g, "__kit_readcyclecounter",
                             builtin_id(KIT_CG_BUILTIN_I64), NULL, 0, NULL);
     return;
   }
   /* A 64-bit value is a two-register pair on the 32-bit native targets. The
    * ordinary intrinsic ABI carries one register per operand/result, so use the
    * rollover-free compiler runtime's limb implementation for high multiply. */
   if (nargs == 2 &&
       (intrin == KIT_CG_INTRIN_SMUL_HIGH ||
        intrin == KIT_CG_INTRIN_UMUL_HIGH) &&
       api_wide64_stack_top(g, 0) && api_wide64_stack_top(g, 1)) {
     ApiSValue args_h[2];
     KitCgTypeId i64 = builtin_id(KIT_CG_BUILTIN_I64);
     KitCgTypeId ps[2] = {i64, i64};
     args_h[1] = api_pop(g);
     args_h[0] = api_pop(g);
     api_runtime_call_values(g,
                             intrin == KIT_CG_INTRIN_SMUL_HIGH
                                 ? "__kit_mulhdi3"
                                 : "__kit_umulhdi3",
                             i64, ps, 2, args_h);
     return;
   }
   /* clz/ctz/popcount/bswap on a split 64-bit value cannot use the backend's
    * single-register software sequence. Route them to the compiler-rt __*di2
    * helpers, which decompose into 32-bit operations. (32-bit forms still lower
    * inline.) */
   if (nargs == 1 && api_wide64_stack_top(g, 0)) {
     const char* name = NULL;
     KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32);
     KitCgTypeId i64 = builtin_id(KIT_CG_BUILTIN_I64);
     KitCgTypeId ret = i32;
     switch (intrin) {
       case KIT_CG_INTRIN_CLZ:
         name = "__clzdi2";
         break;
       case KIT_CG_INTRIN_CTZ:
         name = "__ctzdi2";
         break;
       case KIT_CG_INTRIN_POPCOUNT:
         name = "__popcountdi2";
         break;
       case KIT_CG_INTRIN_BSWAP:
         name = "__bswapdi2";
         ret = i64;
         break;
       default:
         break;
     }
     if (name) {
       ApiSValue arg = api_pop(g);
       KitCgTypeId ps[1] = {i64};
       api_runtime_call_values(g, name, ret, ps, 1, &arg);
       if (ret == i32 && result_type != i32)
         api_cg_convert_kind(g, result_type, CV_ZEXT);
       return;
     }
   }
   /* __builtin_*_overflow on a split 64-bit operand pair traps in the native
    * backend (it only models single-register overflow). Legalize all 6 forms
    * inline as 2-lane / 4-lane ops, pushing [value, ok] like the native path.
    * Gated on both operands being wide64 so other targets are unchanged. */
   if (nargs == 2 && api_intrinsic_is_overflow(intrin) &&
       api_wide64_stack_top(g, 0) && api_wide64_stack_top(g, 1)) {
     api_wide64_overflow_inline(g, intrin);
     return;
   }
   if (nargs == 2 && intrin == KIT_CG_INTRIN_EXPECT &&
       api_wide64_stack_top(g, 1)) {
     ApiSValue expected = api_pop(g);
     ApiSValue val = api_pop(g);
     api_release(g, &expected);
     api_push(g, val);
     return;
   }
   T = g->target;
   h = g->c->ctx->heap;
   rty = resolve_type(g->c, result_type);
   int_ty = builtin_id(KIT_CG_BUILTIN_I32);
   kind = api_map_intrinsic(g, intrin, result_type);
   if (!kit_cg_target_supports_intrinsic(g->c, intrin) || kind == INTRIN_NONE) {
     compiler_panic(
         g->c, g->cur_loc, "KitCg: target '%s' does not support intrinsic '%s'",
         arch_kind_name(g->c->target.arch), api_intrinsic_name(intrin));
     return;
   }
 
   svs = NULL;
   args = NULL;
   if (nargs) {
     svs = (ApiSValue*)h->alloc(h, sizeof(*svs) * nargs, _Alignof(ApiSValue));
     args = (Operand*)h->alloc(h, sizeof(*args) * nargs, _Alignof(Operand));
     memset(args, 0, sizeof(*args) * nargs);
     for (u32 i = 0; i < nargs; ++i) {
       u32 idx = nargs - 1u - i;
       KitCgTypeId aty;
       svs[idx] = api_pop(g);
       aty = api_sv_type(&svs[idx]);
       if (api_sv_op_is(&svs[idx], OPK_IMM) &&
           (intrin == KIT_CG_INTRIN_EXPECT ||
            intrin == KIT_CG_INTRIN_ASSUME_ALIGNED ||
            intrin == KIT_CG_INTRIN_PREFETCH || intrin == KIT_CG_INTRIN_DMB ||
            intrin == KIT_CG_INTRIN_DSB ||
            intrin == KIT_CG_INTRIN_FRAME_ADDRESS ||
            intrin == KIT_CG_INTRIN_RETURN_ADDRESS)) {
         args[idx] = svs[idx].op;
       } else {
         args[idx] = api_force_local(g, &svs[idx], aty);
       }
     }
   }
 
   if (api_intrinsic_is_overflow(intrin)) {
     KitCgTypeId vty = rty ? rty : (nargs ? api_sv_type(&svs[0]) : int_ty);
     KitCgTypeId bool_ty = builtin_id(KIT_CG_BUILTIN_BOOL);
     CGLocal rr = api_alloc_temp_local(g, vty);
     CGLocal ok = api_alloc_temp_local(g, bool_ty);
     dsts[0] = api_op_local(rr, vty);
     dsts[1] = api_op_local(ok, bool_ty);
     ndst = 2;
   } else if (!api_intrinsic_is_void(intrin) && !cg_type_is_void(g->c, rty)) {
     CGLocal rr = api_alloc_temp_local(g, rty);
     dsts[0] = api_op_local(rr, rty);
     ndst = 1;
   }
 
   T->intrinsic(T, kind, ndst ? dsts : NULL, ndst, args, nargs);
 
   for (u32 i = 0; i < nargs; ++i) api_release(g, &svs[i]);
   if (svs) h->free(h, svs, sizeof(*svs) * nargs);
   if (args) h->free(h, args, sizeof(*args) * nargs);
 
   if (api_intrinsic_is_overflow(intrin)) {
     api_push(g, api_make_sv(dsts[0], dsts[0].type));
     api_push(g, api_make_sv(dsts[1], dsts[1].type));
   } else if (ndst == 1) {
     api_push(g, api_make_sv(dsts[0], rty));
   }
 }
 
 /* ============================================================
  * Atomics (stub)
  * ============================================================ */
 
 KitCgTypeId api_atomic_pointee(KitCg* g, KitCgTypeId pty, const char* who) {
   KitCgTypeId pointee = cg_type_pointee(g->c, pty);
   CG_REQUIRE(g, pointee, "%.*s: operand is not a pointer",
              SLICE_ARG(slice_from_cstr(who)));
   return pointee;
 }