kit

control.c (56488B)

---

 #include "cg/internal.h"
 #include "core/metrics.h"
 
 KitCgLabel kit_cg_label_new(KitCg* g) {
   if (!g) return KIT_CG_LABEL_NONE;
   if (api_unevaluated(g) && g->fn_ret_type == KIT_CG_TYPE_NONE)
     return KIT_CG_LABEL_NONE;
   return (KitCgLabel)g->target->label_new(g->target);
 }
 
 void kit_cg_label_place(KitCg* g, KitCgLabel label) {
   if (!g) return;
   if (api_unevaluated(g)) return;
   api_local_const_control_boundary(g);
   g->target->label_place(g->target, (Label)label);
 }
 
 void kit_cg_jump(KitCg* g, KitCgLabel label) {
   if (!g) return;
   if (api_unevaluated(g)) return;
   api_local_const_control_boundary(g);
   g->target->jump(g->target, (Label)label);
 }
 
 void api_branch_if(KitCg* g, ApiSValue* v, int branch_when_true, Label label) {
   CgTarget* T;
   KitCgTypeId ty;
   if (!g) return;
   if (api_unevaluated(g)) {
     api_release(g, v);
     return;
   }
   api_local_const_control_boundary(g);
   T = g->target;
   ty = v->type ? v->type : builtin_id(KIT_CG_BUILTIN_I32);
   if (v->op.kind == OPK_IMM && api_sv_kind(v) == SV_OPERAND) {
     if ((v->op.v.imm != 0) == !!branch_when_true) T->jump(T, label);
     api_release(g, v);
     return;
   }
   if (api_sv_kind(v) == SV_CMP) {
     CmpOp op = branch_when_true ? v->delayed->cmp.op
                                 : api_invert_cmp(v->delayed->cmp.op);
     /* Flag dead-transient operands so the -O0 backend drops them at the branch
      * instead of spilling them (eager dead-operand drop). The compare has no
      * local destination, so pass an immediate as the dst sentinel. */
     Operand none = api_op_imm(0, builtin_id(KIT_CG_BUILTIN_I32));
     Operand ca = api_op_kill_if_dead(g, v->delayed->cmp.a, none);
     Operand cb = api_op_kill_if_dead(g, v->delayed->cmp.b, none);
     T->cmp_branch(T, op, ca, cb, label);
     api_release(g, v);
     return;
   }
   if (api_is_i128_type(g->c, ty)) {
     KitCgTypeId i128 = builtin_id(KIT_CG_BUILTIN_I128);
     KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32);
     KitCgTypeId ps[2] = {i128, i128};
     ApiSValue args[2];
     ApiSValue cmp;
     args[0] = *v;
     args[1] = api_make_sv(api_op_imm(0, i128), i128);
     api_runtime_call_values(g, "__kit_ucmpti2", i32, ps, 2, args);
     cmp = api_pop(g);
     api_branch_if(g, &cmp, branch_when_true, label);
     return;
   }
   /* Split-lane 8-byte truthiness: branch on (lo | hi) != 0. The value is
    * memory-resident, so a single-slot CMP_NE-vs-zero would only see the low
    * word; OR the two lanes into an i32 first. */
   if (api_is_wide8_scalar_type(g->c, ty)) {
     KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32);
     Operand orl = api_wide8_or_lanes(g, v, ty);
     Operand zero = api_op_imm(0, i32);
     T->cmp_branch(T, branch_when_true ? CMP_NE : CMP_EQ, orl, zero, label);
     api_release(g, v);
     return;
   }
   {
     Operand a = api_force_local(g, v, ty);
     Operand zero = api_op_imm(0, ty);
     /* The truthiness test is `v`'s last use; drop it at the branch if it is a
      * dead transient (zero is the non-local dst sentinel). */
     a = api_op_kill_if_dead(g, a, zero);
     T->cmp_branch(T, branch_when_true ? CMP_NE : CMP_EQ, a, zero, label);
     api_release(g, v);
   }
 }
 
 void kit_cg_branch_true(KitCg* g, KitCgLabel label) {
   ApiSValue v;
   if (!g) return;
   v = api_pop(g);
   api_branch_if(g, &v, 1, (Label)label);
 }
 
 void kit_cg_branch_false(KitCg* g, KitCgLabel label) {
   ApiSValue v;
   if (!g) return;
   v = api_pop(g);
   api_branch_if(g, &v, 0, (Label)label);
 }
 
 void cg_lower_switch_default(CgTarget* t, const CGSwitchDesc* d) {
   /* Cmp-and-branch chain: one cmp_branch per case, then jump to
    * default (or fall through if LABEL_NONE). The fallback shape; the
    * frontend-facing kit_cg_switch picks chain vs. jump-table up
    * front (see cg_plan_switch) and routes here only for the chain
    * case. Backend overrides (the C target's switch_) and opt's IR
    * replay both reach this from outside the cg API, so the lowering
    * stays target-only and uses just cmp_branch + jump. */
   for (u32 i = 0; i < d->ncases; ++i) {
     Operand imm = api_op_imm((i64)d->cases[i].value, d->selector_type);
     t->cmp_branch(t, CMP_EQ, d->selector, imm, d->cases[i].label);
   }
   if (d->default_label != LABEL_NONE) {
     t->jump(t, d->default_label);
   }
 }
 
 /* Density / sizing thresholds for the O1 jump-table heuristic. At O0
  * we only honor an explicit KIT_CG_SWITCH_JUMP_TABLE hint — the policy
  * is single-pass and tied to frontend intent rather than analysis. O1
  * runs a linear scan over the case list and picks a single global table
  * when the cases are dense enough; clustering / multi-table layouts are
  * out of scope here. */
 #define CG_SWITCH_TABLE_MIN_CASES_O1 4u
 #define CG_SWITCH_TABLE_MAX_SPAN_O1 4096u
 #define CG_SWITCH_TABLE_DENSITY_RECIP_O1 4u /* ncases * recip >= span */
 
 typedef enum CGSwitchPlanKind {
   CG_SWITCH_PLAN_CHAIN,
   CG_SWITCH_PLAN_TABLE,
 } CGSwitchPlanKind;
 
 typedef struct CGSwitchPlan {
   CGSwitchPlanKind kind;
   i64 vmin;
   u64 span;
 } CGSwitchPlan;
 
 /* Single pass over the cases array, deriving (vmin, span) for the tightest
  * dispatch window. Dispatch is `idx = (sel - vmin) mod 2^width` with an
  * unsigned bounds check, so the cases live on a circle of size 2^width and
  * the minimal enclosing window is the complement of the largest gap between
  * them. We don't sort to find that gap; instead we evaluate the two windows
  * whose seam sits at a fixed point and take the smaller:
  *
  *   - the *signed* window [smin, smax], seam at the signed midpoint — tight
  *     for values near 0 or near 2^width (e.g. a (T)-1 case folded next to
  *     small positives);
  *   - the *unsigned* window [umin, umax], seam at 0 / 2^width — tight for
  *     values straddling the signed midpoint.
  *
  * A table-eligible cluster (span <= a few thousand) is far smaller than
  * 2^(width-1), so it can cross at most one of those two seams; the smaller
  * window is therefore the true minimum. Selector signedness isn't even
  * visible here (CG integer builtins carry only a width), and both windows
  * dispatch correctly under the modular index, so taking the min is a pure
  * code-size win. Returns 0 if the selector type is unusable (>64 bits) or
  * both windows degenerate to the full range. */
 static int cg_switch_extents(Compiler* c, const CGSwitchDesc* d, i64* out_vmin,
                              u64* out_span) {
   u32 width;
   u64 mask;
   i64 smin;
   i64 smax;
   u64 umin;
   u64 umax;
   u32 i;
   u64 sspan;
   u64 uspan;
   width = kit_cg_type_int_width((KitCompiler*)c, d->selector_type);
   if (!width || width > 64u) return 0;
   if (d->ncases == 0) return 0;
   mask = (width >= 64u) ? UINT64_MAX : (((u64)1u << width) - 1u);
   smin = INT64_MAX;
   smax = INT64_MIN;
   umin = UINT64_MAX;
   umax = 0;
   for (i = 0; i < d->ncases; ++i) {
     u64 raw = d->cases[i].value & mask;
     i64 sv = (width == 64u) ? (i64)d->cases[i].value
                             : api_sign_extend_width(d->cases[i].value, width);
     if (sv < smin) smin = sv;
     if (sv > smax) smax = sv;
     if (raw < umin) umin = raw;
     if (raw > umax) umax = raw;
   }
   /* span = delta + 1; a delta of UINT64_MAX is the degenerate full-range
    * case the table can't represent. */
   {
     u64 sdelta = (u64)smax - (u64)smin;
     sspan = sdelta == UINT64_MAX ? 0u : sdelta + 1u;
   }
   {
     u64 udelta = umax - umin;
     uspan = udelta == UINT64_MAX ? 0u : udelta + 1u;
   }
   if (sspan == 0u && uspan == 0u) return 0;
   if (uspan != 0u && (sspan == 0u || uspan < sspan)) {
     *out_span = uspan;
     *out_vmin = (i64)umin; /* a bit pattern; emission indexes modularly */
   } else {
     *out_span = sspan;
     *out_vmin = smin;
   }
   return 1;
 }
 
 static CGSwitchPlan cg_plan_switch(KitCg* g, const CGSwitchDesc* d) {
   CGSwitchPlan plan;
   plan.kind = CG_SWITCH_PLAN_CHAIN;
   plan.vmin = 0;
   plan.span = 0;
   if (d->ncases == 0) return plan;
   if (d->default_label == LABEL_NONE) return plan;
   if (d->hint == KIT_CG_SWITCH_BRANCH_CHAIN) return plan;
   if (!cg_switch_extents(g->c, d, &plan.vmin, &plan.span)) return plan;
   if (d->hint == KIT_CG_SWITCH_JUMP_TABLE) {
     /* Frontend explicitly opted in. Honor unless the span is wildly
      * out of bounds — a forced hint shouldn't blow up code size on a
      * misshapen switch. */
     if (plan.span > CG_SWITCH_TABLE_MAX_SPAN_O1) return plan;
     plan.kind = CG_SWITCH_PLAN_TABLE;
     return plan;
   }
   /* TARGET_DEFAULT: O0 keeps the chain; O1+ runs the density check. */
   if (d->opt_level == 0) return plan;
   if (d->ncases < CG_SWITCH_TABLE_MIN_CASES_O1) return plan;
   if (plan.span > CG_SWITCH_TABLE_MAX_SPAN_O1) return plan;
   if (plan.span > (u64)d->ncases * CG_SWITCH_TABLE_DENSITY_RECIP_O1)
     return plan;
   plan.kind = CG_SWITCH_PLAN_TABLE;
   return plan;
 }
 
 /* Emit a dense jump-table dispatch using cg-API ops. The selector value
  * is still on the value stack on entry. Routing through the cg API
  * means the same primitives work for direct CG (lowered to machine
  * ops) and for the opt wrapper (recorded as IR_BINOP / IR_CMP_BRANCH /
  * IR_LOAD / IR_INDIRECT_BRANCH, which pass_emit already lowers
  * natively without ever materializing IR_SWITCH). */
 static void cg_emit_switch_table(KitCg* g, const CGSwitchDesc* d,
                                  const CGSwitchPlan* plan) {
   Compiler* c;
   Heap* h;
   KitCgTypeId sel_ty;
   u32 sel_w;
   KitCgTypeId i64_ty;
   KitCgTypeId void_ptr_ty;
   KitCgTypeId arr_ty;
   Label* labels;
   KitCgLabel* targets;
   ObjSymId table_sym;
   KitCgDecl decl;
   KitCgMemAccess acc;
   u64 i;
   u32 width;
   c = g->c;
   h = (Heap*)c->ctx->heap;
   sel_ty = d->selector_type;
   sel_w = kit_cg_type_int_width((KitCompiler*)c, sel_ty);
   i64_ty = builtin_id(KIT_CG_BUILTIN_I64);
   void_ptr_ty = cg_type_ptr_to(c, builtin_id(KIT_CG_BUILTIN_VOID));
 
   /* 1. Keep the original selector, then compute idx = sel - vmin
    *    (selector_type wraparound is what we want; the unsigned bounds
    *    check below catches both negative-underflow and positive-overflow
    *    cases). The selector copy lets us recompute idx after the bounds
    *    branch instead of carrying a duplicated arithmetic value across
    *    control flow. */
   kit_cg_dup(g); /* [sel, sel] */
   kit_cg_push_int(g, (uint64_t)plan->vmin, sel_ty);
   kit_cg_int_binop(g, KIT_CG_INT_SUB, 0); /* [sel, idx] */
 
   /* 2. Bounds check: branch to default when idx u> span-1. */
   kit_cg_dup(g); /* [sel, idx, idx] */
   kit_cg_push_int(g, plan->span - 1u, sel_ty);
   kit_cg_int_cmp(g, KIT_CG_INT_GT_U);                  /* [sel, idx, cond] */
   kit_cg_branch_true(g, (KitCgLabel)d->default_label); /* [sel, idx] */
 
   /* 3. Recompute idx from the preserved selector for table addressing. */
   kit_cg_drop(g); /* [sel] */
   kit_cg_push_int(g, (uint64_t)plan->vmin, sel_ty);
   kit_cg_int_binop(g, KIT_CG_INT_SUB, 0); /* [idx] */
 
   /* 4. Widen idx to i64 so the subsequent index multiply runs at
    *    pointer width regardless of selector signedness. The bounds
    *    check above already established 0 <= idx < span, so zext is
    *    value-preserving. */
   if (sel_w < 64u) {
     kit_cg_zext(g, i64_ty);
   }
 
   /* 5. Build the dense label[] table and emit the rodata table. */
   labels =
       (Label*)h->alloc(h, (size_t)plan->span * sizeof *labels, _Alignof(Label));
   if (!labels) compiler_panic(c, g->cur_loc, "kit_cg_switch: oom");
   for (i = 0; i < plan->span; ++i) labels[i] = d->default_label;
   width = sel_w;
   {
     /* Index modularly: vmin may be the unsigned-window origin (a value with
      * the sign bit set), so a signed `vi - vmin` could overflow. This mirrors
      * the runtime `idx = sel - vmin` computed in sel_ty, and reproduces the
      * old non-wrapping subtraction exactly for the signed-window case. */
     u64 mask = (width >= 64u) ? UINT64_MAX : (((u64)1u << width) - 1u);
     for (i = 0; i < d->ncases; ++i) {
       u64 table_index = ((u64)d->cases[i].value - (u64)plan->vmin) & mask;
       labels[table_index] = d->cases[i].label;
     }
   }
   table_sym = api_emit_label_table(g, labels, (u32)plan->span);
   h->free(h, labels, (size_t)plan->span * sizeof *labels);
   if (table_sym == OBJ_SYM_NONE) {
     /* api_emit_label_table panics on real failure; this only fires if
      * a future caller asks for a 0-entry table (which cg_plan_switch
      * already rules out). */
     compiler_panic(c, g->cur_loc, "kit_cg_switch: table emission failed");
     return;
   }
   arr_ty = kit_cg_type_array((KitCompiler*)c, void_ptr_ty, plan->span);
   memset(&decl, 0, sizeof decl);
   decl.kind = KIT_CG_DECL_OBJECT;
   decl.sym.bind = KIT_SB_LOCAL;
   decl.sym.visibility = KIT_CG_VIS_DEFAULT;
   decl.as.object.flags = KIT_CG_OBJ_READONLY;
   api_remember_sym(g, table_sym, arr_ty, decl);
 
   /* 6. Load table[idx]: bitcast the table pointer to a pointer-to-element so
    *    the element place carries the pointer-size scale, put the index on top,
    *    project the element place, then load it. */
   kit_cg_push_symbol_addr(g, (KitCgSym)table_sym, 0); /* [idx, &table] */
   kit_cg_bitcast(g, cg_type_ptr_to(c, void_ptr_ty));  /* &table : void** */
   kit_cg_swap(g);                                     /* [&table, idx] */
   kit_cg_elem(g, 0);                                  /* [&table[idx]] */
   memset(&acc, 0, sizeof acc);
   acc.type = void_ptr_ty;
   acc.align = (uint32_t)c->target.ptr_align;
   kit_cg_load(g, acc); /* [label_addr] */
 
   /* 7. Indirect branch with the full closed target set (every case +
    *    default), so backends doing branch-target hardening (BTI/IBT/CFG)
    *    can stamp landing pads on every reachable label. */
   targets = (KitCgLabel*)h->alloc(h, (d->ncases + 1u) * sizeof *targets,
                                   _Alignof(KitCgLabel));
   if (!targets) compiler_panic(c, g->cur_loc, "kit_cg_switch: oom");
   for (i = 0; i < d->ncases; ++i) {
     targets[i] = (KitCgLabel)d->cases[i].label;
   }
   targets[d->ncases] = (KitCgLabel)d->default_label;
   kit_cg_computed_goto(g, targets, d->ncases + 1u);
   h->free(h, targets, (d->ncases + 1u) * sizeof *targets);
 }
 
 void kit_cg_switch(KitCg* g, KitCgSwitch sw) {
   ApiSValue selector;
   CGSwitchDesc desc;
   Heap* h;
   CGSwitchCase* cases = NULL;
   CGSwitchPlan plan;
   int native_switch_override;
   if (!g) return;
   if (g->sp == 0) return;
   if (api_unevaluated(g)) {
     selector = api_pop(g);
     api_release(g, &selector);
     return;
   }
   api_local_const_control_boundary(g);
   memset(&desc, 0, sizeof desc);
   desc.selector_type = resolve_type(g->c, sw.selector_type);
   if (!desc.selector_type) {
     ApiSValue tmp = g->stack[g->sp - 1u];
     desc.selector_type = api_sv_type(&tmp);
   }
   desc.default_label = (Label)sw.default_label;
   desc.ncases = sw.ncases;
   desc.hint = (u8)sw.hint;
   desc.opt_level = (u8)g->opt_level;
   if (sw.ncases) {
     h = g->c->ctx->heap;
     cases = (CGSwitchCase*)h->alloc(h, sw.ncases * sizeof(CGSwitchCase),
                                     _Alignof(CGSwitchCase));
     if (!cases)
       compiler_panic(g->c, g->cur_loc, "kit_cg_switch: out of memory");
     for (u32 i = 0; i < sw.ncases; ++i) {
       cases[i].value = sw.cases[i].value;
       cases[i].label = (Label)sw.cases[i].label;
     }
     desc.cases = cases;
   }
 
   /* Direct O0 targets may override switch_ for a single-pass branch-chain
    * lowering. Still honor an explicit jump-table hint so tests and frontends
    * can exercise the semantic label-table path without enabling O1. */
   native_switch_override = (g->target->switch_ && g->opt_level == 0 &&
                             desc.hint != KIT_CG_SWITCH_JUMP_TABLE);
   plan = native_switch_override ? (CGSwitchPlan){CG_SWITCH_PLAN_CHAIN, 0, 0}
                                 : cg_plan_switch(g, &desc);
 
   /* The label-table lowering materializes a rodata table of code-label
    * addresses and an indirect branch. Targets that can't express that (Wasm)
    * realize dense dispatch through their switch_ hook (br_table) instead, so
    * hand the plan—hint and all—to switch_ rather than the table path. */
   if (plan.kind == CG_SWITCH_PLAN_TABLE && g->target->switch_ &&
       g->target->supports_label_table &&
       !g->target->supports_label_table(g->target)) {
     plan.kind = CG_SWITCH_PLAN_CHAIN;
   }
 
   if (plan.kind == CG_SWITCH_PLAN_TABLE) {
     /* Selector stays on the value stack; cg_emit_switch_table consumes
      * it via cg-API ops so the path also records cleanly under opt. */
     metrics_count(g->c, "cg.switch.table", 1);
     cg_emit_switch_table(g, &desc, &plan);
   } else {
     metrics_count(g->c, "cg.switch.chain", 1);
     selector = api_pop(g);
     desc.selector =
         api_force_local_unless_imm(g, &selector, desc.selector_type);
     if (g->target->switch_) {
       g->target->switch_(g->target, &desc);
     } else {
       cg_lower_switch_default(g->target, &desc);
     }
     api_release(g, &selector);
   }
   if (cases) {
     h = g->c->ctx->heap;
     h->free(h, cases, sw.ncases * sizeof(CGSwitchCase));
   }
 }
 
 void kit_cg_push_label_addr(KitCg* g, KitCgLabel label, KitCgTypeId ptr_type) {
   KitCgTypeId ty;
   CGLocal r;
   Operand dst;
   if (!g) return;
   ty = resolve_type(g->c, ptr_type);
   if (!ty) ty = cg_type_ptr_to(g->c, builtin_id(KIT_CG_BUILTIN_VOID));
   if (api_unevaluated(g)) {
     api_push(g, api_uneval_value(g, ty));
     api_const_set_top(g, api_const_unknown(ty));
     return;
   }
   r = api_alloc_temp_local(g, ty);
   dst = api_op_local(r, ty);
   g->target->load_label_addr(g->target, dst, (Label)label);
   api_push(g, api_make_sv(dst, ty));
 }
 
 void kit_cg_computed_goto(KitCg* g, const KitCgLabel* valid_targets,
                           uint32_t ntargets) {
   ApiSValue target;
   KitCgTypeId target_ty;
   Operand target_op;
   if (!g) return;
   if (api_unevaluated(g)) {
     target = api_pop(g);
     api_release(g, &target);
     return;
   }
   CG_REQUIRE(g, valid_targets && ntargets != 0,
              "kit_cg_computed_goto: valid_targets must be non-empty");
   api_local_const_control_boundary(g);
   target = api_pop(g);
   target_ty = api_sv_type(&target);
   target_op = api_force_local(g, &target, target_ty);
   g->target->indirect_branch(g->target, target_op, (const Label*)valid_targets,
                              ntargets);
   api_release(g, &target);
 }
 
 void kit_cg_unreachable(KitCg* g) {
   if (!g) return;
   if (api_unevaluated(g)) return;
   api_local_const_control_boundary(g);
   g->target->unreachable(g->target);
 }
 
 /* ============================================================
  * Scopes / structured control flow
  * ============================================================ */
 
 /* Grow g->scopes to hold at least `want` entries (mirrors api_grow_locals).
  * New slots are zeroed so their `active`/`generation` start clean. */
 int api_grow_scopes(KitCg* g, u32 want) {
   Heap* h = g->c->ctx->heap;
   ApiCgScope* nb;
   u32 cap;
   if (g->scopes_cap >= want) return 1;
   cap = g->scopes_cap ? g->scopes_cap : 16u;
   while (cap < want) cap *= 2u;
   nb = (ApiCgScope*)h->alloc(h, sizeof(*nb) * cap, _Alignof(ApiCgScope));
   if (!nb) return 0;
   memset(nb, 0, sizeof(*nb) * cap);
   if (g->scopes) {
     memcpy(nb, g->scopes, sizeof(*nb) * g->nscopes);
     h->free(h, g->scopes, sizeof(*g->scopes) * g->scopes_cap);
   }
   g->scopes = nb;
   g->scopes_cap = cap;
   return 1;
 }
 
 /* The handle packs the 1-based scope index into the low 16 bits and the
  * generation into the upper 16 — see api_scope_from_handle. */
 KitCgScope api_scope_handle(u32 idx, u32 generation) {
   return (KitCgScope)((generation << 16) | ((idx + 1u) & 0xffffu));
 }
 
 ApiCgScope* api_scope_from_handle(KitCg* g, KitCgScope scope, int require_top,
                                   const char* who) {
   u32 scope_index;
   u32 generation;
   ApiCgScope* s;
   if (!g || scope == 0) return NULL;
   scope_index = ((u32)scope & 0xffffu);
   generation = ((u32)scope >> 16);
   CG_REQUIRE(g, scope_index != 0 && scope_index <= API_CG_MAX_SCOPES,
              "%.*s: invalid scope handle", SLICE_ARG(slice_from_cstr(who)));
   scope_index--;
   CG_REQUIRE(g, scope_index < g->nscopes, "%.*s: stale scope handle",
              SLICE_ARG(slice_from_cstr(who)));
   CG_REQUIRE(g, !require_top || scope_index + 1u == g->nscopes,
              "%.*s: non-LIFO scope end", SLICE_ARG(slice_from_cstr(who)));
   s = &g->scopes[scope_index];
   CG_REQUIRE(g, s->active && s->generation == generation,
              "%.*s: stale scope handle", SLICE_ARG(slice_from_cstr(who)));
   return s;
 }
 
 static int api_require_loop_scope(KitCg* g, const ApiCgScope* s,
                                   const char* op);
 
 int api_scope_has_result(const ApiCgScope* s) { return s->nresults != 0; }
 
 /* Store one already-popped value into a scope carry local. Shared by the result
  * and loop-param paths; `local`/`type` select which carry slot. */
 static void api_scope_store_one(KitCg* g, CGLocal local, KitCgTypeId type,
                                 ApiSValue* v) {
   Operand dst = api_op_local(local, type);
   Operand src =
       api_sv_op_is_local_or_imm(v) ? v->op : api_force_local(g, v, type);
   g->target->store(g->target, dst, src, api_mem_for_lvalue(g, &dst, type));
   api_release(g, v);
 }
 
 /* Reload one carry local onto the value stack (fresh temp, load, push). */
 static void api_scope_reload_one(KitCg* g, CGLocal local, KitCgTypeId type,
                                  const void* lang_type, u16 lang_flags) {
   CGLocal r = api_alloc_temp_local(g, type);
   Operand dst = api_op_local(r, type);
   Operand src = api_op_local(local, type);
   ApiSValue sv;
   g->target->load(g->target, dst, src, api_mem_for_lvalue(g, &src, type));
   sv = api_make_sv(dst, type);
   sv.lang_type = lang_type;
   sv.lang_flags = lang_flags;
   api_push(g, sv);
 }
 
 /* Pop the scope's N results into a caller array (out[i] = result i, value-stack
  * bottom->top). The top of stack — the last result — is popped first. */
 static void api_scope_pop_results(KitCg* g, ApiCgScope* s, ApiSValue* out) {
   u32 k;
   for (k = 0; k < s->nresults; ++k) out[s->nresults - 1u - k] = api_pop(g);
 }
 
 /* Pop N results off the value stack and store each into its carry local. */
 void api_scope_store_results(KitCg* g, ApiCgScope* s) {
   u32 k;
   for (k = 0; k < s->nresults; ++k) {
     u32 idx = s->nresults - 1u - k; /* TOS popped first -> highest index */
     ApiSValue v = api_pop(g);
     s->result_lang_types[idx] = v.lang_type;
     s->result_lang_flags[idx] = v.lang_flags;
     api_scope_store_one(g, s->result_locals[idx], s->result_types[idx], &v);
   }
 }
 
 /* Reload all N result carry locals onto the value stack (inverse of store). */
 void api_scope_push_results(KitCg* g, ApiCgScope* s) {
   u32 k;
   for (k = 0; k < s->nresults; ++k)
     api_scope_reload_one(g, s->result_locals[k], s->result_types[k],
                          s->result_lang_types[k], s->result_lang_flags[k]);
 }
 
 /* Pop N params off the value stack into the loop-param carry locals. */
 static void api_scope_store_params(KitCg* g, ApiCgScope* s) {
   u32 k;
   for (k = 0; k < s->nparams; ++k) {
     u32 idx = s->nparams - 1u - k;
     ApiSValue v = api_pop(g);
     s->param_lang_types[idx] = v.lang_type;
     s->param_lang_flags[idx] = v.lang_flags;
     api_scope_store_one(g, s->param_locals[idx], s->param_types[idx], &v);
   }
 }
 
 /* Reload all N loop-param carry locals onto the value stack. */
 static void api_scope_reload_params(KitCg* g, ApiCgScope* s) {
   u32 k;
   for (k = 0; k < s->nparams; ++k)
     api_scope_reload_one(g, s->param_locals[k], s->param_types[k],
                          s->param_lang_types[k], s->param_lang_flags[k]);
 }
 
 /* Allocate a memory-resident carry local of the given type. `resolved` is the
  * canonicalized type id (for the local's type); `raw` is the as-supplied id
  * used for the ABI size/align query, matching the historical single-result
  * path. */
 static CGLocal api_scope_alloc_carry_local(KitCg* g, KitCgTypeId resolved,
                                            KitCgTypeId raw) {
   CGLocalDesc ld;
   memset(&ld, 0, sizeof ld);
   ld.type = resolved;
   ld.size = abi_cg_sizeof(g->c->abi, raw);
   ld.align = abi_cg_alignof(g->c->abi, raw);
   ld.flags = CG_LOCAL_MEMORY_REQUIRED;
   return g->target->local(g->target, &ld);
 }
 
 /* Point a scope's carry-local vectors at storage: the inline buffers for the
  * common small arity (no allocation), or one heap block when either count
  * exceeds API_CG_SCOPE_SIG_INLINE. Returns 0 only on heap OOM. */
 static int api_scope_setup_sig(KitCg* g, ApiCgScope* s,
                                const KitCgScopeSig* sig) {
   u32 nr = sig ? sig->nresults : 0u;
   u32 np = sig ? sig->nparams : 0u;
   s->nresults = nr;
   s->nparams = np;
   s->heap_block = NULL;
   if (nr <= API_CG_SCOPE_SIG_INLINE && np <= API_CG_SCOPE_SIG_INLINE) {
     s->result_locals = s->result_locals_inl;
     s->result_types = s->result_types_inl;
     s->result_lang_types = s->result_lang_types_inl;
     s->result_lang_flags = s->result_lang_flags_inl;
     s->param_locals = s->param_locals_inl;
     s->param_types = s->param_types_inl;
     s->param_lang_types = s->param_lang_types_inl;
     s->param_lang_flags = s->param_lang_flags_inl;
     return 1;
   }
   {
     /* CGLocal and KitCgTypeId are both u32; pack the four vectors into one
      * block as [result_locals][result_types][param_locals][param_types]. */
     Heap* h = g->c->ctx->heap;
     u32 total = 2u * nr + 2u * np;
     u32* blk = (u32*)h->alloc(h, sizeof(u32) * total, _Alignof(u32));
     if (!blk) return 0;
     s->heap_block = blk;
     s->result_locals = blk;
     s->result_types = blk + nr;
     s->param_locals = blk + 2u * nr;
     s->param_types = blk + 2u * nr + np;
     s->heap_lang_types_block =
         (const void**)h->alloc(h, sizeof(void*) * (nr + np), _Alignof(void*));
     s->heap_lang_flags_block =
         (u16*)h->alloc(h, sizeof(u16) * (nr + np), _Alignof(u16));
     if (!s->heap_lang_types_block || !s->heap_lang_flags_block) {
       if (s->heap_lang_types_block)
         h->free(h, s->heap_lang_types_block, sizeof(void*) * (nr + np));
       if (s->heap_lang_flags_block)
         h->free(h, s->heap_lang_flags_block, sizeof(u16) * (nr + np));
       h->free(h, blk, sizeof(u32) * total);
       s->heap_block = NULL;
       s->heap_lang_types_block = NULL;
       s->heap_lang_flags_block = NULL;
       return 0;
     }
     s->result_lang_types = s->heap_lang_types_block;
     s->param_lang_types = s->heap_lang_types_block + nr;
     s->result_lang_flags = s->heap_lang_flags_block;
     s->param_lang_flags = s->heap_lang_flags_block + nr;
     return 1;
   }
 }
 
 static void api_scope_free_sig(KitCg* g, ApiCgScope* s) {
   Heap* h = g->c->ctx->heap;
   if (s->heap_block) {
     u32 total = 2u * s->nresults + 2u * s->nparams;
     h->free(h, s->heap_block, sizeof(u32) * total);
     s->heap_block = NULL;
   }
   if (s->heap_lang_types_block) {
     h->free(h, s->heap_lang_types_block,
             sizeof(void*) * (s->nresults + s->nparams));
     s->heap_lang_types_block = NULL;
   }
   if (s->heap_lang_flags_block) {
     h->free(h, s->heap_lang_flags_block,
             sizeof(u16) * (s->nresults + s->nparams));
     s->heap_lang_flags_block = NULL;
   }
 }
 
 static KitCgScope api_scope_begin_sig_kind(KitCg* g, u8 kind,
                                            const KitCgScopeSig* sig) {
   Label break_lbl, cont_lbl;
   CGScopeDesc d;
   ApiCgScope* s;
   CGScope target_scope;
   u32 idx, k;
   if (!g) return 0;
   break_lbl = g->target->label_new(g->target);
   cont_lbl =
       (kind == SCOPE_LOOP) ? g->target->label_new(g->target) : LABEL_NONE;
 
   /* Depth past what the 16-bit handle index can address (see api_scope_handle)
    * — unreachable for any realistic source. */
   CG_REQUIRE(g, g->nscopes < API_CG_MAX_SCOPES,
              "KitCg: too many nested scopes");
   if (!api_grow_scopes(g, g->nscopes + 1u)) {
     compiler_panic(g->c, g->cur_loc, "KitCg: out of memory growing scopes");
     return 0;
   }
   idx = g->nscopes;
   s = &g->scopes[idx];
   s->break_lbl = break_lbl;
   s->continue_lbl = cont_lbl;
   s->generation = ++g->scope_generation;
   if (s->generation == 0) s->generation = ++g->scope_generation;
   s->active = 1;
   if (!api_scope_setup_sig(g, s, sig)) {
     compiler_panic(g->c, g->cur_loc,
                    "KitCg: out of memory for scope signature");
     return 0;
   }
   for (k = 0; k < s->nresults; ++k)
     s->result_types[k] = resolve_type(g->c, sig->results[k]);
   for (k = 0; k < s->nparams; ++k)
     s->param_types[k] = resolve_type(g->c, sig->params[k]);
 
   /* Loop preheader: snapshot the params off the value stack into their carry
    * locals BEFORE the loop header, so the pop runs once in the preheader and
    * every back edge (store_params + jump cont) re-stores into the same locals.
    * Allocating a local emits nothing; the param stores belong before cont_lbl.
    */
   if (kind == SCOPE_LOOP && s->nparams) {
     for (k = 0; k < s->nparams; ++k)
       s->param_locals[k] =
           api_scope_alloc_carry_local(g, s->param_types[k], sig->params[k]);
     api_scope_store_params(g, s);
   }
 
   api_local_const_control_boundary(g);
   if (cont_lbl != LABEL_NONE) g->target->label_place(g->target, cont_lbl);
   g->nscopes++;
 
   memset(&d, 0, sizeof d);
   d.kind = kind;
   d.break_label = break_lbl;
   d.continue_label = cont_lbl;
   /* CGScopeDesc.result_type stays a single informational field (backends ignore
    * it): the single-result type, or NONE for void / multi-value. Multi-value is
    * realized purely through the CG carry locals above. */
   d.result_type = (s->nresults == 1u) ? s->result_types[0] : KIT_CG_TYPE_NONE;
   target_scope = g->target->scope_begin(g->target, &d);
   s->target_scope = target_scope;
 
   /* Allocate the result carry locals after scope_begin, matching the historical
    * single-result ordering. */
   for (k = 0; k < s->nresults; ++k)
     s->result_locals[k] =
         api_scope_alloc_carry_local(g, s->result_types[k], sig->results[k]);
 
   /* The loop body observes the params on the value stack: reload them after the
    * header so a back edge re-establishes exactly this shape. */
   if (kind == SCOPE_LOOP && s->nparams) api_scope_reload_params(g, s);
 
   return api_scope_handle(idx, s->generation);
 }
 
 KitCgScope kit_cg_scope_begin(KitCg* g) {
   KitCgScopeSig sig;
   memset(&sig, 0, sizeof sig);
   return api_scope_begin_sig_kind(g, (u8)SCOPE_LOOP, &sig);
 }
 
 KitCgScope kit_cg_scope_begin_value(KitCg* g, KitCgTypeId result_type) {
   KitCgScopeSig sig;
   KitCgTypeId resolved;
   memset(&sig, 0, sizeof sig);
   if (!g) return 0;
   resolved = resolve_type(g->c, result_type);
   CG_REQUIRE(g,
              resolved != KIT_CG_TYPE_NONE && !cg_type_is_void(g->c, resolved),
              "KitCg: value scope requires non-void result type");
   sig.results = &result_type;
   sig.nresults = 1u;
   return api_scope_begin_sig_kind(g, (u8)SCOPE_LOOP, &sig);
 }
 
 KitCgScope kit_cg_block_begin(KitCg* g) {
   KitCgScopeSig sig;
   memset(&sig, 0, sizeof sig);
   return api_scope_begin_sig_kind(g, (u8)SCOPE_BLOCK, &sig);
 }
 
 KitCgScope kit_cg_block_begin_value(KitCg* g, KitCgTypeId result_type) {
   KitCgScopeSig sig;
   KitCgTypeId resolved;
   memset(&sig, 0, sizeof sig);
   if (!g) return 0;
   resolved = resolve_type(g->c, result_type);
   CG_REQUIRE(g,
              resolved != KIT_CG_TYPE_NONE && !cg_type_is_void(g->c, resolved),
              "KitCg: value block requires non-void result type");
   sig.results = &result_type;
   sig.nresults = 1u;
   return api_scope_begin_sig_kind(g, (u8)SCOPE_BLOCK, &sig);
 }
 
 KitCgScope kit_cg_scope_begin_sig(KitCg* g, const KitCgScopeSig* sig) {
   return api_scope_begin_sig_kind(g, (u8)SCOPE_LOOP, sig);
 }
 
 KitCgScope kit_cg_block_begin_sig(KitCg* g, const KitCgScopeSig* sig) {
   return api_scope_begin_sig_kind(g, (u8)SCOPE_BLOCK, sig);
 }
 
 void kit_cg_scope_store_params(KitCg* g, KitCgScope scope) {
   ApiCgScope* s =
       api_scope_from_handle(g, scope, 0, "KitCg: scope_store_params");
   if (!s) return;
   if (!api_require_loop_scope(g, s, "scope_store_params")) return;
   api_scope_store_params(g, s);
 }
 
 KitCgLabel kit_cg_scope_break_label(KitCg* g, KitCgScope scope) {
   ApiCgScope* s =
       api_scope_from_handle(g, scope, 0, "KitCg: scope_break_label");
   return s ? (KitCgLabel)s->break_lbl : (KitCgLabel)0;
 }
 
 KitCgLabel kit_cg_scope_continue_label(KitCg* g, KitCgScope scope) {
   ApiCgScope* s =
       api_scope_from_handle(g, scope, 0, "KitCg: scope_continue_label");
   return s ? (KitCgLabel)s->continue_lbl : (KitCgLabel)0;
 }
 
 void kit_cg_scope_end(KitCg* g, KitCgScope scope) {
   ApiCgScope* s = api_scope_from_handle(g, scope, 1, "KitCg: scope_end");
   if (!s) return;
   /* The fall-through results are on TOS; store them so they join any
    * branch-supplied copies in the same carry locals, then reload after the
    * break label as the scope's outputs. */
   api_scope_store_results(g, s);
   api_local_const_control_boundary(g);
   g->target->label_place(g->target, s->break_lbl);
   g->target->scope_end(g->target, s->target_scope);
   api_scope_push_results(g, s);
   api_scope_free_sig(g, s);
   s->active = 0;
   g->nscopes--;
 }
 
 void kit_cg_break(KitCg* g, KitCgScope scope) {
   ApiCgScope* s = api_scope_from_handle(g, scope, 0, "KitCg: break");
   if (!s) return;
   api_scope_store_results(g, s);
   api_local_const_control_boundary(g);
   g->target->jump(g->target, s->break_lbl);
 }
 
 void kit_cg_scope_end_unreachable(KitCg* g, KitCgScope scope) {
   /* Close a scope whose fall-through is unreachable (its body terminated via a
    * branch/return). There are no fall-through results to pop; the carry locals
    * already hold whatever the branch that exited stored. Mirrors scope_end but
    * skips the leading store_results. The caller must have the value stack at
    * the scope's base depth. */
   ApiCgScope* s =
       api_scope_from_handle(g, scope, 1, "KitCg: scope_end_unreachable");
   if (!s) return;
   api_local_const_control_boundary(g);
   g->target->label_place(g->target, s->break_lbl);
   g->target->scope_end(g->target, s->target_scope);
   api_scope_push_results(g, s);
   api_scope_free_sig(g, s);
   s->active = 0;
   g->nscopes--;
 }
 
 /* Store the pre-popped result array into the scope's carry locals (forward). */
 static void api_scope_store_results_from(KitCg* g, ApiCgScope* s,
                                          ApiSValue* rs) {
   u32 k;
   for (k = 0; k < s->nresults; ++k) {
     s->result_lang_types[k] = rs[k].lang_type;
     s->result_lang_flags[k] = rs[k].lang_flags;
     api_scope_store_one(g, s->result_locals[k], s->result_types[k], &rs[k]);
   }
 }
 
 /* Shared body of break_true / break_false: pop the condition, then — when the
  * scope carries results — pop them all (so they are consumed on both the taken
  * and not-taken paths, per the API contract), and store-then-jump only when the
  * branch is taken. `break_when` selects the sense. The single-result path is
  * byte-identical to the historical code. */
 static void api_break_cond(KitCg* g, KitCgScope scope, int break_when,
                            const char* who) {
   ApiCgScope* s;
   ApiSValue cond;
   if (!g || scope == 0) return;
   s = api_scope_from_handle(g, scope, 0, who);
   if (!s) return;
   cond = api_pop(g);
 
   if (!api_scope_has_result(s)) {
     api_branch_if(g, &cond, break_when, s->break_lbl);
     return;
   }
   {
     Heap* h = g->c->ctx->heap;
     ApiSValue* rs = (ApiSValue*)h->alloc(h, sizeof(ApiSValue) * s->nresults,
                                          _Alignof(ApiSValue));
     if (!rs) compiler_panic(g->c, g->cur_loc, "KitCg: out of memory");
     api_scope_pop_results(g, s, rs);
     if (api_sv_kind(&cond) == SV_OPERAND && cond.op.kind == OPK_IMM) {
       if ((cond.op.v.imm != 0) == !!break_when) {
         api_scope_store_results_from(g, s, rs);
         api_local_const_control_boundary(g);
         g->target->jump(g->target, s->break_lbl);
       } else {
         u32 k;
         for (k = 0; k < s->nresults; ++k) api_release(g, &rs[k]);
       }
       api_release(g, &cond);
     } else {
       Label skip = g->target->label_new(g->target);
       api_branch_if(g, &cond, !break_when, skip);
       api_scope_store_results_from(g, s, rs);
       api_local_const_control_boundary(g);
       g->target->jump(g->target, s->break_lbl);
       api_local_const_control_boundary(g);
       g->target->label_place(g->target, skip);
     }
     h->free(h, rs, sizeof(ApiSValue) * s->nresults);
   }
 }
 
 void kit_cg_break_true(KitCg* g, KitCgScope scope) {
   api_break_cond(g, scope, 1, "KitCg: break_true");
 }
 
 void kit_cg_break_false(KitCg* g, KitCgScope scope) {
   api_break_cond(g, scope, 0, "KitCg: break_false");
 }
 
 /* continue jumps to the loop header, which a forward-only block scope does not
  * have (its continue_lbl is LABEL_NONE). Reject it with a clean diagnostic
  * rather than emitting a jump to a nonexistent label. */
 static int api_require_loop_scope(KitCg* g, const ApiCgScope* s,
                                   const char* op) {
   CG_REQUIRE(g, s->continue_lbl != LABEL_NONE,
              "KitCg: %s is not valid on a forward-only block scope", op);
   return 1;
 }
 
 void kit_cg_continue(KitCg* g, KitCgScope scope) {
   ApiCgScope* s = api_scope_from_handle(g, scope, 0, "KitCg: continue");
   if (!s) return;
   if (!api_require_loop_scope(g, s, "continue")) return;
   api_local_const_control_boundary(g);
   g->target->jump(g->target, s->continue_lbl);
 }
 
 void kit_cg_continue_true(KitCg* g, KitCgScope scope) {
   ApiCgScope* s;
   ApiSValue v;
   if (!g || scope == 0) return;
   s = api_scope_from_handle(g, scope, 0, "KitCg: continue_true");
   if (!s) return;
   if (!api_require_loop_scope(g, s, "continue_true")) return;
   v = api_pop(g);
   api_branch_if(g, &v, 1, s->continue_lbl);
 }
 
 void kit_cg_continue_false(KitCg* g, KitCgScope scope) {
   ApiCgScope* s;
   ApiSValue v;
   if (!g || scope == 0) return;
   s = api_scope_from_handle(g, scope, 0, "KitCg: continue_false");
   if (!s) return;
   if (!api_require_loop_scope(g, s, "continue_false")) return;
   v = api_pop(g);
   api_branch_if(g, &v, 0, s->continue_lbl);
 }
 
 /* ============================================================
  * Dynamic stack allocation / variadics (stubs)
  * ============================================================ */
 
 void kit_cg_alloca(KitCg* g, uint32_t align, KitCgTypeId result_ptr_type) {
   ApiSValue sz;
   CgTarget* T;
   KitCgTypeId pty;
   Operand sz_op;
   CGLocal rr;
   Operand dst;
   if (!g) return;
   T = g->target;
   sz = api_pop(g);
   pty = resolve_type(g->c, result_ptr_type);
   if (!pty) pty = cg_type_ptr_to(g->c, builtin_id(KIT_CG_BUILTIN_VOID));
   if (api_unevaluated(g)) {
     api_release(g, &sz);
     api_push(g, api_uneval_value(g, pty));
     api_const_set_top(g, api_const_unknown(pty));
     return;
   }
   sz_op = api_sv_op_is(&sz, OPK_IMM)
               ? sz.op
               : api_force_local(g, &sz, api_sv_type(&sz));
   rr = api_alloc_temp_local(g, pty);
   dst = api_op_local(rr, pty);
   T->alloca_(T, dst, sz_op, align ? align : 16);
   api_release(g, &sz);
   api_push(g, api_make_sv(dst, pty));
 }
 
 void kit_cg_vararg_start(KitCg* g) {
   ApiSValue ap;
   CgTarget* T;
   Operand ap_op;
   if (!g) return;
   T = g->target;
   ap = api_pop(g);
   if (api_unevaluated(g)) {
     api_release(g, &ap);
     return;
   }
   ap_op = api_force_local(g, &ap, api_sv_type(&ap));
   T->va_start_(T, ap_op);
   api_release(g, &ap);
 }
 
 void kit_cg_vararg_next(KitCg* g, KitCgTypeId type) {
   ApiSValue ap;
   CgTarget* T;
   KitCgTypeId ty;
   Operand ap_op;
   CGLocal rr;
   Operand dst;
   if (!g) return;
   T = g->target;
   ty = resolve_type(g->c, type);
   if (!ty) return;
   ap = api_pop(g);
   if (api_unevaluated(g)) {
     api_release(g, &ap);
     api_push(g, api_uneval_value(g, ty));
     api_const_set_top(g, api_const_unknown(ty));
     return;
   }
   ap_op = api_force_local(g, &ap, api_sv_type(&ap));
   rr = api_alloc_temp_local(g, ty);
   dst = api_op_local(rr, ty);
   T->va_arg_(T, dst, ap_op, ty);
   api_release(g, &ap);
   api_push(g, api_make_sv(dst, ty));
 }
 
 void kit_cg_vararg_end(KitCg* g) {
   ApiSValue ap;
   CgTarget* T;
   Operand ap_op;
   if (!g) return;
   T = g->target;
   ap = api_pop(g);
   if (api_unevaluated(g)) {
     api_release(g, &ap);
     return;
   }
   ap_op = api_force_local(g, &ap, api_sv_type(&ap));
   T->va_end_(T, ap_op);
   api_release(g, &ap);
 }
 
 void kit_cg_vararg_copy(KitCg* g) {
   ApiSValue src, dst;
   CgTarget* T;
   Operand src_op, dst_op;
   if (!g) return;
   T = g->target;
   src = api_pop(g);
   dst = api_pop(g);
   if (api_unevaluated(g)) {
     api_release(g, &src);
     api_release(g, &dst);
     return;
   }
   src_op = api_force_local(g, &src, api_sv_type(&src));
   dst_op = api_force_local(g, &dst, api_sv_type(&dst));
   T->va_copy_(T, dst_op, src_op);
   api_release(g, &src);
   api_release(g, &dst);
 }
 
 /* ============================================================
  * Memory operations (stubs)
  * ============================================================ */
 
 void kit_cg_memcpy(KitCg* g, uint64_t size, KitCgMemAccess dst_access,
                    KitCgMemAccess src_access) {
   ApiSValue src, dst;
   CgTarget* T;
   AggregateAccess agg;
   Operand dst_op, src_op;
   if (!g) return;
   api_local_const_memory_boundary(g);
   (void)src_access;
   if (size > UINT32_MAX) {
     compiler_panic(g->c, g->cur_loc, "KitCg: memcpy size exceeds CgTarget");
     return;
   }
   T = g->target;
   src = api_pop(g);
   dst = api_pop(g);
   if (api_unevaluated(g)) {
     api_release(g, &src);
     api_release(g, &dst);
     return;
   }
   api_require_pointer_value(g, "memcpy destination", api_sv_type(&dst));
   api_require_pointer_value(g, "memcpy source", api_sv_type(&src));
   dst_op = api_force_local(g, &dst, api_sv_type(&dst));
   src_op = api_force_local(g, &src, api_sv_type(&src));
   memset(&agg, 0, sizeof agg);
   agg.size = (u32)size;
   agg.align = dst_access.align ? dst_access.align : (u32)size;
   T->copy_bytes(T, dst_op, src_op, agg);
   api_release(g, &dst);
   api_release(g, &src);
 }
 
 void kit_cg_memmove(KitCg* g, uint64_t size, KitCgMemAccess dst_access,
                     KitCgMemAccess src_access) {
   ApiSValue src, dst;
   Operand args[3];
   if (!g) return;
   api_local_const_memory_boundary(g);
   (void)dst_access;
   (void)src_access;
   if (size > INT64_MAX) {
     compiler_panic(g->c, g->cur_loc, "KitCg: memmove size exceeds CgTarget");
     return;
   }
   src = api_pop(g);
   dst = api_pop(g);
   if (api_unevaluated(g)) {
     api_release(g, &src);
     api_release(g, &dst);
     return;
   }
   api_require_pointer_value(g, "memmove destination", api_sv_type(&dst));
   api_require_pointer_value(g, "memmove source", api_sv_type(&src));
   args[0] = api_force_local(g, &dst, api_sv_type(&dst));
   args[1] = api_force_local(g, &src, api_sv_type(&src));
   args[2] = api_op_imm((i64)size, builtin_id(KIT_CG_BUILTIN_I64));
   g->target->intrinsic(g->target, INTRIN_MEMMOVE, NULL, 0, args, 3);
   api_release(g, &dst);
   api_release(g, &src);
 }
 
 void kit_cg_memset(KitCg* g, uint8_t val, uint64_t size,
                    KitCgMemAccess dst_access) {
   ApiSValue dst;
   CgTarget* T;
   AggregateAccess agg;
   Operand dst_op, byte_val;
   if (!g) return;
   api_local_const_memory_boundary(g);
   if (size > UINT32_MAX) {
     compiler_panic(g->c, g->cur_loc, "KitCg: memset size exceeds CgTarget");
     return;
   }
   T = g->target;
   dst = api_pop(g);
   if (api_unevaluated(g)) {
     api_release(g, &dst);
     return;
   }
   api_require_pointer_value(g, "memset destination", api_sv_type(&dst));
   dst_op = api_force_local(g, &dst, api_sv_type(&dst));
   byte_val = api_op_imm((i64)val, KIT_CG_TYPE_NONE);
   memset(&agg, 0, sizeof agg);
   agg.size = (u32)size;
   agg.align = dst_access.align ? dst_access.align : (u32)size;
   T->set_bytes(T, dst_op, byte_val, agg);
   api_release(g, &dst);
 }
 
 /* log2 of a {1,2,4,8} scale, else -1. */
 static int cg_scale_to_log2(u32 scale) {
   switch (scale) {
     case 1:
       return 0;
     case 2:
       return 1;
     case 4:
       return 2;
     case 8:
       return 3;
     default:
       return -1;
   }
 }
 
 static int cg_checked_scaled_offset(i64 index, u32 scale, i64 offset,
                                     i64* out) {
   i64 product;
   if (__builtin_mul_overflow(index, (i64)scale, &product)) return 0;
   if (__builtin_add_overflow(product, offset, out)) return 0;
   return 1;
 }
 
 /* Shared core for kit_cg_elem / kit_cg_elem_scaled. `elem_size` is the index
  * stride in bytes; 0 means "derive from the base pointee size" (the kit_cg_elem
  * behavior). The place's access type is always the base pointee. */
 static void api_cg_elem(KitCg* g, u32 elem_size, int64_t offset) {
   ApiSValue idx, base;
   CgTarget* T;
   KitCgTypeId base_ty, base_ptr_ty, elem_ty, idx_ty;
   const CgType* base_info;
   u32 elemsz;
   Operand base_op, idx_op;
   CGLocal base_local;
   if (!g) return;
   T = g->target;
   idx = api_pop(g);
   base = api_pop(g);
   base_ty = api_sv_type(&base);
   base_info = cg_type_get(g->c, base_ty);
   CG_REQUIRE(g,
              !api_is_lvalue_sv(&base) && base_info &&
                  base_info->kind == KIT_CG_TYPE_PTR,
              "KitCg: elem requires a pointer value base (decay an "
              "array to a pointer first)");
   elem_ty = base_info->ptr.pointee;
   base_ptr_ty = base_ty;
   elemsz = elem_size ? elem_size : (u32)abi_cg_sizeof(g->c->abi, elem_ty);
   idx_ty = idx.type ? idx.type : idx.op.type;
   if (!idx_ty) idx_ty = builtin_id(KIT_CG_BUILTIN_I64);
   if (api_unevaluated(g)) {
     api_release(g, &base);
     api_release(g, &idx);
     api_push(g, api_uneval_place(g, elem_ty));
     api_const_set_top(g, api_const_unknown(elem_ty));
     return;
   }
   idx_op = api_force_local_unless_imm(g, &idx, idx_ty);
 
   /* Constant index folds entirely into the displacement — no instructions, just
    * a larger offset on the place. A symbol-address base can remain a GLOBAL
    * place, folding the symbol addend with index*scale+offset instead of first
    * materializing &sym into a temporary. */
   if (idx_op.kind == OPK_IMM) {
     i64 ofs;
     Operand place;
     if (cg_checked_scaled_offset(idx_op.v.imm, elemsz, offset, &ofs) &&
         api_sv_kind(&base) == SV_OPERAND && base.op.kind == OPK_GLOBAL &&
         !__builtin_add_overflow(base.op.v.global.addend, ofs, &ofs)) {
       place = api_op_global(base.op.v.global.sym, ofs, elem_ty);
     } else {
       base_op = api_force_local(g, &base, base_ptr_ty);
       base_local = base_op.v.local;
       if (cg_checked_scaled_offset(idx_op.v.imm, elemsz, offset, &ofs) &&
           ofs >= INT32_MIN && ofs <= INT32_MAX) {
         place = api_op_indirect(base_local, (i32)ofs, elem_ty);
       } else if (cg_checked_scaled_offset(idx_op.v.imm, elemsz, offset, &ofs)) {
         CGLocal r = api_alloc_temp_local(g, base_ptr_ty);
         Operand ro = api_op_local(r, base_ptr_ty);
         T->binop(T, BO_IADD, ro, api_op_local(base_local, base_ptr_ty),
                  api_op_imm(ofs, base_ptr_ty));
         place = api_op_indirect(r, 0, elem_ty);
       } else {
         CGLocal ir = api_alloc_temp_local(g, idx_ty);
         Operand idx_local = api_op_local(ir, idx_ty);
         int lg2 = cg_scale_to_log2(elemsz);
         u8 log2_scale = 0;
         i32 place_offset;
         T->load_imm(T, idx_local, idx_op.v.imm);
         if (lg2 >= 0) {
           log2_scale = (u8)lg2;
         } else {
           CGLocal sr = api_alloc_temp_local(g, idx_ty);
           T->binop(T, BO_IMUL, api_op_local(sr, idx_ty), idx_local,
                    api_op_imm((i64)elemsz, idx_ty));
           ir = sr;
         }
         if (offset >= INT32_MIN && offset <= INT32_MAX) {
           place_offset = (i32)offset;
         } else {
           CGLocal r = api_alloc_temp_local(g, base_ptr_ty);
           Operand ro = api_op_local(r, base_ptr_ty);
           T->binop(T, BO_IADD, ro, api_op_local(base_local, base_ptr_ty),
                    api_op_imm(offset, base_ptr_ty));
           base_local = r;
           place_offset = 0;
         }
         place = api_op_indirect_indexed(base_local, ir, log2_scale,
                                         place_offset, elem_ty);
       }
     }
     api_release(g, &base);
     api_release(g, &idx);
     api_push(g, api_make_lv(place, elem_ty));
     return;
   }
 
   /* Dynamic index: build a scaled-index place so the backend emits one
    * [base + index*scale] addressing mode. A power-of-two element size rides in
    * log2_scale; any other size pre-multiplies the index once (scale 1). The
    * index is copied into a fresh local for unambiguous ownership. */
   {
     int lg2 = cg_scale_to_log2(elemsz);
     CGLocal ir;
     u8 log2_scale;
     i32 place_offset;
     base_op = api_force_local(g, &base, base_ptr_ty);
     base_local = base_op.v.local;
     if (offset >= INT32_MIN && offset <= INT32_MAX) {
       place_offset = (i32)offset;
     } else {
       CGLocal r = api_alloc_temp_local(g, base_ptr_ty);
       Operand ro = api_op_local(r, base_ptr_ty);
       T->binop(T, BO_IADD, ro, api_op_local(base_local, base_ptr_ty),
                api_op_imm(offset, base_ptr_ty));
       base_local = r;
       place_offset = 0;
     }
     /* Force the index to a local first so we can tell whether it is a dead
      * transient (provably not referenced elsewhere) that we can take over as
      * the index slot directly, skipping the ownership copy. */
     idx_op = api_force_local_unless_imm(g, &idx, idx_ty);
     if (idx.op.kind == OPK_LOCAL) idx_op = idx.op;
     if (lg2 >= 0 && idx_op.kind == OPK_LOCAL && idx_op.v.local != base_local &&
         api_coalesce_on(g) && api_temp_dead(g, idx_op.v.local)) {
       ir = idx_op.v.local; /* reuse the dead transient; no copy */
       log2_scale = (u8)lg2;
     } else {
       ir = api_alloc_temp_local(g, idx_ty);
       if (lg2 >= 0) {
         T->copy(T, api_op_local(ir, idx_ty), idx_op);
         log2_scale = (u8)lg2;
       } else {
         T->binop(T, BO_IMUL, api_op_local(ir, idx_ty), idx_op,
                  api_op_imm((i64)elemsz, idx_ty));
         log2_scale = 0;
       }
     }
     api_release(g, &base);
     api_release(g, &idx);
     api_push(g, api_make_lv(api_op_indirect_indexed(base_local, ir, log2_scale,
                                                     place_offset, elem_ty),
                             elem_ty));
   }
 }
 
 void kit_cg_elem(KitCg* g, int64_t offset) { api_cg_elem(g, 0u, offset); }
 
 void kit_cg_elem_scaled(KitCg* g, uint32_t elem_size, int64_t offset) {
   api_cg_elem(g, elem_size, offset);
 }
 
 static void api_cg_field_at_place(KitCg* g, ApiSValue base,
                                   int64_t field_offset, KitCgTypeId field_ty) {
   CgTarget* T;
   KitCgTypeId base_ty;
   KitCgTypeId base_ptr_ty;
   Operand result;
   if (!g) return;
   T = g->target;
   api_ensure_local(g, &base);
   CG_REQUIRE(g, api_is_lvalue_sv(&base),
              "KitCg: field_at requires a place; deref a pointer first");
   field_ty = resolve_type(g->c, field_ty);
   CG_REQUIRE(g, field_ty, "KitCg: field_at has invalid field type");
   if (api_unevaluated(g)) {
     api_release(g, &base);
     api_push(g, api_uneval_place(g, field_ty));
     api_const_set_top(g, api_const_unknown(field_ty));
     return;
   }
   base_ty = api_sv_type(&base);
   if (!base_ty) base_ty = builtin_id(KIT_CG_BUILTIN_VOID);
   base_ptr_ty = cg_type_ptr_to(g->c, base_ty);
   if (base.op.kind == OPK_GLOBAL) {
     i64 addend;
     if (!__builtin_add_overflow(base.op.v.global.addend, field_offset,
                                 &addend)) {
       result = api_op_global(base.op.v.global.sym, addend, field_ty);
       api_release(g, &base);
       api_push(g, api_make_lv(result, field_ty));
       return;
     }
   } else if (base.op.kind == OPK_INDIRECT && field_offset >= INT32_MIN &&
              field_offset <= INT32_MAX) {
     i32 ofs;
     if (!__builtin_add_overflow(base.op.v.ind.ofs, (i32)field_offset, &ofs)) {
       /* Fold the field offset into the displacement, preserving any index/scale
        * a preceding `elem` left so `p[i].f` stays one
        * [base+index*scale+off]. */
       result = api_op_indirect_indexed(base.op.v.ind.base, base.op.v.ind.index,
                                        base.op.v.ind.log2_scale, ofs, field_ty);
       api_release(g, &base);
       api_push(g, api_make_lv(result, field_ty));
       return;
     }
   }
   {
     Operand base_addr = api_lvalue_addr(g, &base, base_ptr_ty);
     if (field_offset == 0) {
       result = base_addr;
     } else {
       CGLocal fr = api_alloc_temp_local(g, base_ptr_ty);
       result = api_op_local(fr, base_ptr_ty);
       T->binop(T, BO_IADD, result, base_addr,
                api_op_imm(field_offset, base_ptr_ty));
     }
     api_release(g, &base);
     api_push(
         g, api_make_lv(api_op_indirect(result.v.local, 0, field_ty), field_ty));
   }
 }
 
 void kit_cg_field_at(KitCg* g, int64_t byte_offset, KitCgTypeId field_type) {
   ApiSValue base;
   if (!g) return;
   base = api_pop(g);
   api_cg_field_at_place(g, base, byte_offset, field_type);
 }
 
 void kit_cg_field(KitCg* g, uint32_t field_index) {
   ApiSValue base;
   KitCgTypeId rec_ty;
   KitCgTypeId base_ty;
   KitCgTypeId field_ty;
   KitCgTypeId rec_ptr_ty;
   const CgType* rec_info;
   const ABIRecordLayout* layout;
   u32 field_offset;
   if (!g) return;
   base = api_pop(g);
   api_ensure_local(g, &base);
   base_ty = api_sv_type(&base);
   CG_REQUIRE(g, api_is_lvalue_sv(&base),
              "KitCg: field requires a record place; deref a pointer first");
   rec_ty = base_ty;
   rec_ptr_ty = cg_type_ptr_to(g->c, rec_ty);
   layout = abi_cg_record_layout(g->c->abi, rec_ty);
   CG_REQUIRE(g, layout && field_index < layout->nfields,
              "KitCg: invalid field index");
   rec_info = cg_type_get(g->c, rec_ty);
   CG_REQUIRE(g,
              rec_info && rec_info->kind == KIT_CG_TYPE_RECORD &&
                  field_index < rec_info->record.nfields,
              "KitCg: invalid record base");
   field_ty = rec_info->record.fields[field_index].type;
   field_offset = layout->fields[field_index].offset;
   if (api_unevaluated(g)) {
     ApiSValue sv = api_uneval_place(g, field_ty);
     if (layout->fields[field_index].bit_width != 0 ||
         (rec_info->record.fields[field_index].flags & KIT_CG_FIELD_BITFIELD) !=
             0) {
       sv.bitfield.bit_offset = layout->fields[field_index].bit_offset;
       sv.bitfield.bit_width = layout->fields[field_index].bit_width;
       sv.bitfield.bit_storage_size = layout->fields[field_index].storage_size;
       sv.bitfield.bit_signed =
           rec_info->record.fields[field_index].bit_signed ? 1u : 0u;
     }
     api_release(g, &base);
     api_push(g, sv);
     api_const_set_top(g, api_const_unknown(field_ty));
     return;
   }
   if (layout->fields[field_index].bit_width != 0 ||
       (rec_info->record.fields[field_index].flags & KIT_CG_FIELD_BITFIELD) !=
           0) {
     Operand base_addr;
     ApiSValue sv;
     CG_REQUIRE(g, layout->fields[field_index].bit_width != 0,
                "KitCg: zero-width bit-field access");
     /* Project to a bit-field PLACE: the place addresses the enclosing storage
      * unit and carries the bit-field geometry from the record layout. A plain
      * load/store on this place performs the extract/insert; there is no
      * separate bit-field memop and no bit-field rider on KitCgMemAccess. */
     base_addr = api_lvalue_addr(g, &base, rec_ptr_ty);
     sv = api_make_lv(base_addr, field_ty);
     sv.bitfield.bit_offset = layout->fields[field_index].bit_offset;
     sv.bitfield.bit_width = layout->fields[field_index].bit_width;
     sv.bitfield.bit_storage_size = layout->fields[field_index].storage_size;
     sv.bitfield.bit_signed =
         rec_info->record.fields[field_index].bit_signed ? 1u : 0u;
     api_release(g, &base);
     api_push(g, sv);
     return;
   }
   api_cg_field_at_place(g, base, (int64_t)field_offset, field_ty);
 }
 
 void kit_cg_field_bits(KitCg* g, uint16_t bit_offset, uint16_t bit_width,
                        uint32_t bit_storage_size, int bit_signed) {
   ApiSValue* top;
   if (!g || g->sp == 0) return;
   top = &g->stack[g->sp - 1u];
   CG_REQUIRE(g, api_is_lvalue_sv(top),
              "KitCg: field_bits requires a place destination");
   top->bitfield.bit_offset = bit_offset;
   top->bitfield.bit_width = bit_width;
   top->bitfield.bit_storage_size = bit_storage_size;
   top->bitfield.bit_signed = bit_signed ? 1u : 0u;
 }
 
 /* ============================================================
  * Calls / return
  * ============================================================ */
 
 /* Shared scaffolding for kit_cg_call / kit_cg_call_symbol. The two
  * public entry points differ only in how the callee is obtained and in
  * their pre-call stack-depth check; everything else (arg packaging, return
  * storage allocation, post-call release, result push) is identical. These
  * helpers carry the common shape and are the natural targets for any future
  * change that wants to vary call-shape policy (e.g. an ABI-driven storage
  * decision). */