kit

control.c (37245B)

---

 #include "cg/internal.h"
 #include "core/metrics.h"
 
 KitCgLabel kit_cg_label_new(KitCg* g) {
   if (!g) return KIT_CG_LABEL_NONE;
   return (KitCgLabel)g->target->label_new(g->target);
 }
 
 void kit_cg_label_place(KitCg* g, KitCgLabel label) {
   if (!g) return;
   api_local_const_control_boundary(g);
   g->target->label_place(g->target, (Label)label);
 }
 
 void kit_cg_jump(KitCg* g, KitCgLabel label) {
   if (!g) return;
   api_local_const_control_boundary(g);
   g->target->jump(g->target, (Label)label);
 }
 
 void api_branch_if(KitCg* g, ApiSValue* v, int branch_when_true, Label label) {
   CgTarget* T;
   KitCgTypeId ty;
   if (!g) return;
   api_local_const_control_boundary(g);
   T = g->target;
   ty = v->type ? v->type : builtin_id(KIT_CG_BUILTIN_I32);
   if (v->op.kind == OPK_IMM && v->kind == SV_OPERAND) {
     if ((v->op.v.imm != 0) == !!branch_when_true) T->jump(T, label);
     api_release(g, v);
     return;
   }
   if (v->kind == SV_CMP) {
     CmpOp op = branch_when_true ? v->delayed.cmp.op
                                 : api_invert_cmp(v->delayed.cmp.op);
     T->cmp_branch(T, op, v->delayed.cmp.a, v->delayed.cmp.b, label);
     api_release(g, v);
     return;
   }
   if (api_is_i128_type(g->c, ty)) {
     KitCgTypeId i128 = builtin_id(KIT_CG_BUILTIN_I128);
     KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32);
     KitCgTypeId ps[2] = {i128, i128};
     ApiSValue args[2];
     ApiSValue cmp;
     args[0] = *v;
     args[1] = api_make_sv(api_op_imm(0, i128), i128);
     api_runtime_call_values(g, "__kit_ucmpti2", i32, ps, 2, args);
     cmp = api_pop(g);
     api_branch_if(g, &cmp, branch_when_true, label);
     return;
   }
   /* Split-lane 8-byte truthiness: branch on (lo | hi) != 0. The value is
    * memory-resident, so a single-slot CMP_NE-vs-zero would only see the low
    * word; OR the two lanes into an i32 first. */
   if (api_is_wide8_scalar_type(g->c, ty)) {
     KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32);
     Operand orl = api_wide8_or_lanes(g, v, ty);
     Operand zero = api_op_imm(0, i32);
     T->cmp_branch(T, branch_when_true ? CMP_NE : CMP_EQ, orl, zero, label);
     api_release(g, v);
     return;
   }
   {
     Operand a = api_force_local(g, v, ty);
     Operand zero = api_op_imm(0, ty);
     T->cmp_branch(T, branch_when_true ? CMP_NE : CMP_EQ, a, zero, label);
     api_release(g, v);
   }
 }
 
 void kit_cg_branch_true(KitCg* g, KitCgLabel label) {
   ApiSValue v;
   if (!g) return;
   v = api_pop(g);
   api_branch_if(g, &v, 1, (Label)label);
 }
 
 void kit_cg_branch_false(KitCg* g, KitCgLabel label) {
   ApiSValue v;
   if (!g) return;
   v = api_pop(g);
   api_branch_if(g, &v, 0, (Label)label);
 }
 
 void cg_lower_switch_default(CgTarget* t, const CGSwitchDesc* d) {
   /* Cmp-and-branch chain: one cmp_branch per case, then jump to
    * default (or fall through if LABEL_NONE). The fallback shape; the
    * frontend-facing kit_cg_switch picks chain vs. jump-table up
    * front (see cg_plan_switch) and routes here only for the chain
    * case. Backend overrides (the C target's switch_) and opt's IR
    * replay both reach this from outside the cg API, so the lowering
    * stays target-only and uses just cmp_branch + jump. */
   for (u32 i = 0; i < d->ncases; ++i) {
     Operand imm = api_op_imm((i64)d->cases[i].value, d->selector_type);
     t->cmp_branch(t, CMP_EQ, d->selector, imm, d->cases[i].label);
   }
   if (d->default_label != LABEL_NONE) {
     t->jump(t, d->default_label);
   }
 }
 
 /* Density / sizing thresholds for the O1 jump-table heuristic. At O0
  * we only honor an explicit KIT_CG_SWITCH_JUMP_TABLE hint — the policy
  * is single-pass and tied to frontend intent rather than analysis. O1
  * runs a linear scan over the case list and picks a single global table
  * when the cases are dense enough; clustering / multi-table layouts are
  * out of scope here. */
 #define CG_SWITCH_TABLE_MIN_CASES_O1 4u
 #define CG_SWITCH_TABLE_MAX_SPAN_O1 4096u
 #define CG_SWITCH_TABLE_DENSITY_RECIP_O1 4u /* ncases * recip >= span */
 
 typedef enum CGSwitchPlanKind {
   CG_SWITCH_PLAN_CHAIN,
   CG_SWITCH_PLAN_TABLE,
 } CGSwitchPlanKind;
 
 typedef struct CGSwitchPlan {
   CGSwitchPlanKind kind;
   i64 vmin;
   u64 span;
 } CGSwitchPlan;
 
 /* Single pass over the cases array: derive (vmin, vmax, span) in the
  * selector type's signed interpretation. Frontends store case values as
  * the u64 bit pattern of an i64 sign-extended from the selector's
  * width, so api_sign_extend_width recovers the source ordering for
  * both signed and unsigned selectors that fit in i64. Returns 0 if the
  * selector type is unusable for our policy (>64 bits) or if span
  * overflows u64. */
 static int cg_switch_extents(Compiler* c, const CGSwitchDesc* d, i64* out_vmin,
                              u64* out_span) {
   u32 width;
   i64 vmin;
   i64 vmax;
   u32 i;
   width = kit_cg_type_int_width((KitCompiler*)c, d->selector_type);
   if (!width || width > 64u) return 0;
   vmin = INT64_MAX;
   vmax = INT64_MIN;
   for (i = 0; i < d->ncases; ++i) {
     i64 vi = (width == 64u) ? (i64)d->cases[i].value
                             : api_sign_extend_width(d->cases[i].value, width);
     if (vi < vmin) vmin = vi;
     if (vi > vmax) vmax = vi;
   }
   if (vmax < vmin) return 0;
   {
     u64 delta = (u64)vmax - (u64)vmin;
     if (delta == UINT64_MAX) return 0;
     *out_span = delta + 1u;
   }
   *out_vmin = vmin;
   return 1;
 }
 
 static CGSwitchPlan cg_plan_switch(KitCg* g, const CGSwitchDesc* d) {
   CGSwitchPlan plan;
   plan.kind = CG_SWITCH_PLAN_CHAIN;
   plan.vmin = 0;
   plan.span = 0;
   if (d->ncases == 0) return plan;
   if (d->default_label == LABEL_NONE) return plan;
   if (d->hint == KIT_CG_SWITCH_BRANCH_CHAIN) return plan;
   if (!cg_switch_extents(g->c, d, &plan.vmin, &plan.span)) return plan;
   if (d->hint == KIT_CG_SWITCH_JUMP_TABLE) {
     /* Frontend explicitly opted in. Honor unless the span is wildly
      * out of bounds — a forced hint shouldn't blow up code size on a
      * misshapen switch. */
     if (plan.span > CG_SWITCH_TABLE_MAX_SPAN_O1) return plan;
     plan.kind = CG_SWITCH_PLAN_TABLE;
     return plan;
   }
   /* TARGET_DEFAULT: O0 keeps the chain; O1+ runs the density check. */
   if (d->opt_level == 0) return plan;
   if (d->opt_level >= 2 && plan.span != d->ncases) return plan;
   if (d->ncases < CG_SWITCH_TABLE_MIN_CASES_O1) return plan;
   if (plan.span > CG_SWITCH_TABLE_MAX_SPAN_O1) return plan;
   if (plan.span > (u64)d->ncases * CG_SWITCH_TABLE_DENSITY_RECIP_O1)
     return plan;
   plan.kind = CG_SWITCH_PLAN_TABLE;
   return plan;
 }
 
 /* Emit a dense jump-table dispatch using cg-API ops. The selector value
  * is still on the value stack on entry. Routing through the cg API
  * means the same primitives work for direct CG (lowered to machine
  * ops) and for the opt wrapper (recorded as IR_BINOP / IR_CMP_BRANCH /
  * IR_LOAD / IR_INDIRECT_BRANCH, which pass_emit already lowers
  * natively without ever materializing IR_SWITCH). */
 static void cg_emit_switch_table(KitCg* g, const CGSwitchDesc* d,
                                  const CGSwitchPlan* plan) {
   Compiler* c;
   Heap* h;
   KitCgTypeId sel_ty;
   u32 sel_w;
   KitCgTypeId i64_ty;
   KitCgTypeId void_ptr_ty;
   KitCgTypeId arr_ty;
   Label* labels;
   KitCgLabel* targets;
   ObjSymId table_sym;
   KitCgDecl decl;
   KitCgMemAccess acc;
   u64 i;
   u32 width;
   c = g->c;
   h = (Heap*)c->ctx->heap;
   sel_ty = d->selector_type;
   sel_w = kit_cg_type_int_width((KitCompiler*)c, sel_ty);
   i64_ty = builtin_id(KIT_CG_BUILTIN_I64);
   void_ptr_ty = cg_type_ptr_to(c, builtin_id(KIT_CG_BUILTIN_VOID));
 
   /* 1. Keep the original selector, then compute idx = sel - vmin
    *    (selector_type wraparound is what we want; the unsigned bounds
    *    check below catches both negative-underflow and positive-overflow
    *    cases). The selector copy lets us recompute idx after the bounds
    *    branch instead of carrying a duplicated arithmetic value across
    *    control flow. */
   kit_cg_dup(g); /* [sel, sel] */
   kit_cg_push_int(g, (uint64_t)plan->vmin, sel_ty);
   kit_cg_int_binop(g, KIT_CG_INT_SUB, 0); /* [sel, idx] */
 
   /* 2. Bounds check: branch to default when idx u> span-1. */
   kit_cg_dup(g); /* [sel, idx, idx] */
   kit_cg_push_int(g, plan->span - 1u, sel_ty);
   kit_cg_int_cmp(g, KIT_CG_INT_GT_U);                  /* [sel, idx, cond] */
   kit_cg_branch_true(g, (KitCgLabel)d->default_label); /* [sel, idx] */
 
   /* 3. Recompute idx from the preserved selector for table addressing. */
   kit_cg_drop(g); /* [sel] */
   kit_cg_push_int(g, (uint64_t)plan->vmin, sel_ty);
   kit_cg_int_binop(g, KIT_CG_INT_SUB, 0); /* [idx] */
 
   /* 4. Widen idx to i64 so the subsequent index multiply runs at
    *    pointer width regardless of selector signedness. The bounds
    *    check above already established 0 <= idx < span, so zext is
    *    value-preserving. */
   if (sel_w < 64u) {
     kit_cg_zext(g, i64_ty);
   }
 
   /* 5. Build the dense label[] table and emit the rodata table. */
   labels =
       (Label*)h->alloc(h, (size_t)plan->span * sizeof *labels, _Alignof(Label));
   if (!labels) compiler_panic(c, g->cur_loc, "kit_cg_switch: oom");
   for (i = 0; i < plan->span; ++i) labels[i] = d->default_label;
   width = sel_w;
   for (i = 0; i < d->ncases; ++i) {
     i64 vi = (width == 64u) ? (i64)d->cases[i].value
                             : api_sign_extend_width(d->cases[i].value, width);
     u64 table_index = (u64)(vi - plan->vmin);
     labels[table_index] = d->cases[i].label;
   }
   table_sym = api_emit_label_table(g, labels, (u32)plan->span);
   h->free(h, labels, (size_t)plan->span * sizeof *labels);
   if (table_sym == OBJ_SYM_NONE) {
     /* api_emit_label_table panics on real failure; this only fires if
      * a future caller asks for a 0-entry table (which cg_plan_switch
      * already rules out). */
     compiler_panic(c, g->cur_loc, "kit_cg_switch: table emission failed");
     return;
   }
   arr_ty = kit_cg_type_array((KitCompiler*)c, void_ptr_ty, plan->span);
   memset(&decl, 0, sizeof decl);
   decl.kind = KIT_CG_DECL_OBJECT;
   decl.sym.bind = KIT_SB_LOCAL;
   decl.sym.visibility = KIT_CG_VIS_DEFAULT;
   decl.as.object.flags = KIT_CG_OBJ_READONLY;
   api_remember_sym(g, table_sym, arr_ty, decl);
 
   /* 6. Load table[idx]: bitcast the table pointer to a pointer-to-element so
    *    the element place carries the pointer-size scale, put the index on top,
    *    project the element place, then load it. */
   kit_cg_push_symbol_addr(g, (KitCgSym)table_sym, 0); /* [idx, &table] */
   kit_cg_bitcast(g, cg_type_ptr_to(c, void_ptr_ty));  /* &table : void** */
   kit_cg_swap(g);                                     /* [&table, idx] */
   kit_cg_elem(g, 0);                                  /* [&table[idx]] */
   memset(&acc, 0, sizeof acc);
   acc.type = void_ptr_ty;
   acc.align = (uint32_t)c->target.ptr_align;
   kit_cg_load(g, acc); /* [label_addr] */
 
   /* 7. Indirect branch with the full closed target set (every case +
    *    default), so backends doing branch-target hardening (BTI/IBT/CFG)
    *    can stamp landing pads on every reachable label. */
   targets = (KitCgLabel*)h->alloc(h, (d->ncases + 1u) * sizeof *targets,
                                   _Alignof(KitCgLabel));
   if (!targets) compiler_panic(c, g->cur_loc, "kit_cg_switch: oom");
   for (i = 0; i < d->ncases; ++i) {
     targets[i] = (KitCgLabel)d->cases[i].label;
   }
   targets[d->ncases] = (KitCgLabel)d->default_label;
   kit_cg_computed_goto(g, targets, d->ncases + 1u);
   h->free(h, targets, (d->ncases + 1u) * sizeof *targets);
 }
 
 void kit_cg_switch(KitCg* g, KitCgSwitch sw) {
   ApiSValue selector;
   CGSwitchDesc desc;
   Heap* h;
   CGSwitchCase* cases = NULL;
   CGSwitchPlan plan;
   int native_switch_override;
   if (!g) return;
   if (g->sp == 0) return;
   api_local_const_control_boundary(g);
   memset(&desc, 0, sizeof desc);
   desc.selector_type = resolve_type(g->c, sw.selector_type);
   if (!desc.selector_type) {
     ApiSValue tmp = g->stack[g->sp - 1u];
     desc.selector_type = api_sv_type(&tmp);
   }
   desc.default_label = (Label)sw.default_label;
   desc.ncases = sw.ncases;
   desc.hint = (u8)sw.hint;
   desc.opt_level = (u8)g->opt_level;
   if (sw.ncases) {
     h = g->c->ctx->heap;
     cases = (CGSwitchCase*)h->alloc(h, sw.ncases * sizeof(CGSwitchCase),
                                     _Alignof(CGSwitchCase));
     if (!cases)
       compiler_panic(g->c, g->cur_loc, "kit_cg_switch: out of memory");
     for (u32 i = 0; i < sw.ncases; ++i) {
       cases[i].value = sw.cases[i].value;
       cases[i].label = (Label)sw.cases[i].label;
     }
     desc.cases = cases;
   }
 
   /* Direct O0 targets may override switch_ for a single-pass branch-chain
    * lowering. Still honor an explicit jump-table hint so tests and frontends
    * can exercise the semantic label-table path without enabling O1. */
   native_switch_override = (g->target->switch_ && g->opt_level == 0 &&
                             desc.hint != KIT_CG_SWITCH_JUMP_TABLE);
   plan = native_switch_override ? (CGSwitchPlan){CG_SWITCH_PLAN_CHAIN, 0, 0}
                                 : cg_plan_switch(g, &desc);
 
   /* The label-table lowering materializes a rodata table of code-label
    * addresses and an indirect branch. Targets that can't express that (Wasm)
    * realize dense dispatch through their switch_ hook (br_table) instead, so
    * hand the plan—hint and all—to switch_ rather than the table path. */
   if (plan.kind == CG_SWITCH_PLAN_TABLE && g->target->switch_ &&
       g->target->supports_label_table &&
       !g->target->supports_label_table(g->target)) {
     plan.kind = CG_SWITCH_PLAN_CHAIN;
   }
 
   if (plan.kind == CG_SWITCH_PLAN_TABLE) {
     /* Selector stays on the value stack; cg_emit_switch_table consumes
      * it via cg-API ops so the path also records cleanly under opt. */
     metrics_count(g->c, "cg.switch.table", 1);
     cg_emit_switch_table(g, &desc, &plan);
   } else {
     metrics_count(g->c, "cg.switch.chain", 1);
     selector = api_pop(g);
     desc.selector =
         api_force_local_unless_imm(g, &selector, desc.selector_type);
     if (g->target->switch_) {
       g->target->switch_(g->target, &desc);
     } else {
       cg_lower_switch_default(g->target, &desc);
     }
     api_release(g, &selector);
   }
   if (cases) {
     h = g->c->ctx->heap;
     h->free(h, cases, sw.ncases * sizeof(CGSwitchCase));
   }
 }
 
 void kit_cg_push_label_addr(KitCg* g, KitCgLabel label, KitCgTypeId ptr_type) {
   KitCgTypeId ty;
   CGLocal r;
   Operand dst;
   if (!g) return;
   ty = resolve_type(g->c, ptr_type);
   if (!ty) ty = cg_type_ptr_to(g->c, builtin_id(KIT_CG_BUILTIN_VOID));
   r = api_alloc_temp_local(g, ty);
   dst = api_op_local(r, ty);
   g->target->load_label_addr(g->target, dst, (Label)label);
   api_push(g, api_make_sv(dst, ty));
 }
 
 void kit_cg_computed_goto(KitCg* g, const KitCgLabel* valid_targets,
                           uint32_t ntargets) {
   ApiSValue target;
   KitCgTypeId target_ty;
   Operand target_op;
   if (!g) return;
   if (!valid_targets || ntargets == 0) {
     compiler_panic(g->c, g->cur_loc,
                    "kit_cg_computed_goto: valid_targets must be non-empty");
     return;
   }
   api_local_const_control_boundary(g);
   target = api_pop(g);
   target_ty = api_sv_type(&target);
   target_op = api_force_local(g, &target, target_ty);
   g->target->indirect_branch(g->target, target_op, (const Label*)valid_targets,
                              ntargets);
   api_release(g, &target);
 }
 
 void kit_cg_unreachable(KitCg* g) {
   if (!g) return;
   api_local_const_control_boundary(g);
   g->target->unreachable(g->target);
 }
 
 /* ============================================================
  * Scopes / structured control flow
  * ============================================================ */
 
 KitCgScope api_scope_handle(u32 idx, u32 generation) {
   return (KitCgScope)((generation << 8) | ((idx + 1u) & 0xffu));
 }
 
 ApiCgScope* api_scope_from_handle(KitCg* g, KitCgScope scope, int require_top,
                                   const char* who) {
   u32 scope_index;
   u32 generation;
   ApiCgScope* s;
   if (!g || scope == 0) return NULL;
   scope_index = ((u32)scope & 0xffu);
   generation = ((u32)scope >> 8);
   if (scope_index == 0 || scope_index > API_CG_MAX_SCOPES) {
     compiler_panic(g->c, g->cur_loc, "%.*s: invalid scope handle",
                    SLICE_ARG(slice_from_cstr(who)));
     return NULL;
   }
   scope_index--;
   if (scope_index >= g->nscopes) {
     compiler_panic(g->c, g->cur_loc, "%.*s: stale scope handle",
                    SLICE_ARG(slice_from_cstr(who)));
     return NULL;
   }
   if (require_top && scope_index + 1u != g->nscopes) {
     compiler_panic(g->c, g->cur_loc, "%.*s: non-LIFO scope end",
                    SLICE_ARG(slice_from_cstr(who)));
     return NULL;
   }
   s = &g->scopes[scope_index];
   if (!s->active || s->generation != generation) {
     compiler_panic(g->c, g->cur_loc, "%.*s: stale scope handle",
                    SLICE_ARG(slice_from_cstr(who)));
     return NULL;
   }
   return s;
 }
 
 int api_scope_has_result(const ApiCgScope* s) {
   return s->result_type != KIT_CG_TYPE_NONE;
 }
 
 void api_scope_store_result(KitCg* g, ApiCgScope* s, ApiSValue* result) {
   Operand dst;
   Operand src;
   if (!api_scope_has_result(s)) return;
   dst = api_op_local(s->result_local, s->result_type);
   src = api_sv_op_is_local_or_imm(result)
             ? result->op
             : api_force_local(g, result, s->result_type);
   g->target->store(g->target, dst, src,
                    api_mem_for_lvalue(g, &dst, s->result_type));
   api_release(g, result);
 }
 
 void api_scope_push_result(KitCg* g, ApiCgScope* s) {
   Operand dst;
   Operand src;
   CGLocal r;
   if (!api_scope_has_result(s)) return;
   r = api_alloc_temp_local(g, s->result_type);
   dst = api_op_local(r, s->result_type);
   src = api_op_local(s->result_local, s->result_type);
   g->target->load(g->target, dst, src,
                   api_mem_for_lvalue(g, &src, s->result_type));
   api_push(g, api_make_sv(dst, s->result_type));
 }
 
 static KitCgScope api_scope_begin_kind(KitCg* g, u8 kind,
                                        KitCgTypeId result_type) {
   Label break_lbl, cont_lbl;
   CGScopeDesc d;
   ApiCgScope* s;
   CGScope target_scope;
   u32 idx;
   if (!g) return 0;
   break_lbl = g->target->label_new(g->target);
   cont_lbl =
       (kind == SCOPE_LOOP) ? g->target->label_new(g->target) : LABEL_NONE;
   api_local_const_control_boundary(g);
   if (cont_lbl != LABEL_NONE) g->target->label_place(g->target, cont_lbl);
 
   if (g->nscopes >= API_CG_MAX_SCOPES) {
     compiler_panic(g->c, g->cur_loc, "KitCg: too many nested scopes");
     return 0;
   }
   idx = g->nscopes;
   s = &g->scopes[idx];
   s->break_lbl = break_lbl;
   s->continue_lbl = cont_lbl;
   s->result_type = resolve_type(g->c, result_type);
   s->generation = ++g->scope_generation;
   if (s->generation == 0) s->generation = ++g->scope_generation;
   s->active = 1;
   g->nscopes++;
 
   memset(&d, 0, sizeof d);
   d.kind = kind;
   d.break_label = break_lbl;
   d.continue_label = cont_lbl;
   d.result_type = s->result_type;
   target_scope = g->target->scope_begin(g->target, &d);
   s->target_scope = target_scope;
   s->result_local = CG_LOCAL_NONE;
   if (api_scope_has_result(s)) {
     CGLocalDesc ld;
     memset(&ld, 0, sizeof ld);
     ld.type = s->result_type;
     ld.size = abi_cg_sizeof(g->c->abi, result_type);
     ld.align = abi_cg_alignof(g->c->abi, result_type);
     ld.flags = CG_LOCAL_MEMORY_REQUIRED;
     s->result_local = g->target->local(g->target, &ld);
   }
 
   return api_scope_handle(idx, s->generation);
 }
 
 KitCgScope kit_cg_scope_begin(KitCg* g, KitCgTypeId result_type) {
   return api_scope_begin_kind(g, (u8)SCOPE_LOOP, result_type);
 }
 
 KitCgScope kit_cg_block_begin(KitCg* g, KitCgTypeId result_type) {
   return api_scope_begin_kind(g, (u8)SCOPE_BLOCK, result_type);
 }
 
 KitCgLabel kit_cg_scope_break_label(KitCg* g, KitCgScope scope) {
   ApiCgScope* s =
       api_scope_from_handle(g, scope, 0, "KitCg: scope_break_label");
   return s ? (KitCgLabel)s->break_lbl : (KitCgLabel)0;
 }
 
 KitCgLabel kit_cg_scope_continue_label(KitCg* g, KitCgScope scope) {
   ApiCgScope* s =
       api_scope_from_handle(g, scope, 0, "KitCg: scope_continue_label");
   return s ? (KitCgLabel)s->continue_lbl : (KitCgLabel)0;
 }
 
 void kit_cg_scope_end(KitCg* g, KitCgScope scope) {
   ApiCgScope* s = api_scope_from_handle(g, scope, 1, "KitCg: scope_end");
   if (!s) return;
   if (api_scope_has_result(s)) {
     ApiSValue result = api_pop(g);
     api_scope_store_result(g, s, &result);
   }
   api_local_const_control_boundary(g);
   g->target->label_place(g->target, s->break_lbl);
   g->target->scope_end(g->target, s->target_scope);
   api_scope_push_result(g, s);
   s->active = 0;
   g->nscopes--;
 }
 
 void kit_cg_break(KitCg* g, KitCgScope scope) {
   ApiCgScope* s = api_scope_from_handle(g, scope, 0, "KitCg: break");
   if (!s) return;
   if (api_scope_has_result(s)) {
     ApiSValue result = api_pop(g);
     api_scope_store_result(g, s, &result);
   }
   api_local_const_control_boundary(g);
   g->target->jump(g->target, s->break_lbl);
 }
 
 void kit_cg_break_true(KitCg* g, KitCgScope scope) {
   ApiCgScope* s;
   ApiSValue cond;
   if (!g || scope == 0) return;
   s = api_scope_from_handle(g, scope, 0, "KitCg: break_true");
   if (!s) return;
   cond = api_pop(g);
 
   if (api_scope_has_result(s)) {
     ApiSValue result = api_pop(g);
     if (cond.kind == SV_OPERAND && cond.op.kind == OPK_IMM) {
       if (cond.op.v.imm != 0) {
         api_scope_store_result(g, s, &result);
         api_local_const_control_boundary(g);
         g->target->jump(g->target, s->break_lbl);
       } else {
         api_release(g, &result);
       }
       api_release(g, &cond);
     } else {
       Label skip = g->target->label_new(g->target);
       api_branch_if(g, &cond, 0, skip);
       api_scope_store_result(g, s, &result);
       api_local_const_control_boundary(g);
       g->target->jump(g->target, s->break_lbl);
       api_local_const_control_boundary(g);
       g->target->label_place(g->target, skip);
     }
   } else {
     api_branch_if(g, &cond, 1, s->break_lbl);
   }
 }
 
 void kit_cg_break_false(KitCg* g, KitCgScope scope) {
   ApiCgScope* s;
   ApiSValue cond;
   if (!g || scope == 0) return;
   s = api_scope_from_handle(g, scope, 0, "KitCg: break_false");
   if (!s) return;
   cond = api_pop(g);
 
   if (api_scope_has_result(s)) {
     ApiSValue result = api_pop(g);
     if (cond.kind == SV_OPERAND && cond.op.kind == OPK_IMM) {
       if (cond.op.v.imm == 0) {
         api_scope_store_result(g, s, &result);
         api_local_const_control_boundary(g);
         g->target->jump(g->target, s->break_lbl);
       } else {
         api_release(g, &result);
       }
       api_release(g, &cond);
     } else {
       Label skip = g->target->label_new(g->target);
       api_branch_if(g, &cond, 1, skip);
       api_scope_store_result(g, s, &result);
       api_local_const_control_boundary(g);
       g->target->jump(g->target, s->break_lbl);
       api_local_const_control_boundary(g);
       g->target->label_place(g->target, skip);
     }
   } else {
     api_branch_if(g, &cond, 0, s->break_lbl);
   }
 }
 
 /* continue jumps to the loop header, which a forward-only block scope does not
  * have (its continue_lbl is LABEL_NONE). Reject it with a clean diagnostic
  * rather than emitting a jump to a nonexistent label. */
 static int api_require_loop_scope(KitCg* g, const ApiCgScope* s,
                                   const char* op) {
   if (s->continue_lbl == LABEL_NONE) {
     compiler_panic(g->c, g->cur_loc,
                    "KitCg: %s is not valid on a forward-only block scope", op);
     return 0;
   }
   return 1;
 }
 
 void kit_cg_continue(KitCg* g, KitCgScope scope) {
   ApiCgScope* s = api_scope_from_handle(g, scope, 0, "KitCg: continue");
   if (!s) return;
   if (!api_require_loop_scope(g, s, "continue")) return;
   api_local_const_control_boundary(g);
   g->target->jump(g->target, s->continue_lbl);
 }
 
 void kit_cg_continue_true(KitCg* g, KitCgScope scope) {
   ApiCgScope* s;
   ApiSValue v;
   if (!g || scope == 0) return;
   s = api_scope_from_handle(g, scope, 0, "KitCg: continue_true");
   if (!s) return;
   if (!api_require_loop_scope(g, s, "continue_true")) return;
   v = api_pop(g);
   api_branch_if(g, &v, 1, s->continue_lbl);
 }
 
 void kit_cg_continue_false(KitCg* g, KitCgScope scope) {
   ApiCgScope* s;
   ApiSValue v;
   if (!g || scope == 0) return;
   s = api_scope_from_handle(g, scope, 0, "KitCg: continue_false");
   if (!s) return;
   if (!api_require_loop_scope(g, s, "continue_false")) return;
   v = api_pop(g);
   api_branch_if(g, &v, 0, s->continue_lbl);
 }
 
 /* ============================================================
  * Dynamic stack allocation / variadics (stubs)
  * ============================================================ */
 
 void kit_cg_alloca(KitCg* g, uint32_t align, KitCgTypeId result_ptr_type) {
   ApiSValue sz;
   CgTarget* T;
   KitCgTypeId pty;
   Operand sz_op;
   CGLocal rr;
   Operand dst;
   if (!g) return;
   T = g->target;
   sz = api_pop(g);
   pty = resolve_type(g->c, result_ptr_type);
   if (!pty) pty = cg_type_ptr_to(g->c, builtin_id(KIT_CG_BUILTIN_VOID));
   sz_op = api_sv_op_is(&sz, OPK_IMM)
               ? sz.op
               : api_force_local(g, &sz, api_sv_type(&sz));
   rr = api_alloc_temp_local(g, pty);
   dst = api_op_local(rr, pty);
   T->alloca_(T, dst, sz_op, align ? align : 16);
   api_release(g, &sz);
   api_push(g, api_make_sv(dst, pty));
 }
 
 void kit_cg_vararg_start(KitCg* g) {
   ApiSValue ap;
   CgTarget* T;
   Operand ap_op;
   if (!g) return;
   T = g->target;
   ap = api_pop(g);
   ap_op = api_force_local(g, &ap, api_sv_type(&ap));
   T->va_start_(T, ap_op);
   api_release(g, &ap);
 }
 
 void kit_cg_vararg_next(KitCg* g, KitCgTypeId type) {
   ApiSValue ap;
   CgTarget* T;
   KitCgTypeId ty;
   Operand ap_op;
   CGLocal rr;
   Operand dst;
   if (!g) return;
   T = g->target;
   ty = resolve_type(g->c, type);
   if (!ty) return;
   ap = api_pop(g);
   ap_op = api_force_local(g, &ap, api_sv_type(&ap));
   rr = api_alloc_temp_local(g, ty);
   dst = api_op_local(rr, ty);
   T->va_arg_(T, dst, ap_op, ty);
   api_release(g, &ap);
   api_push(g, api_make_sv(dst, ty));
 }
 
 void kit_cg_vararg_end(KitCg* g) {
   ApiSValue ap;
   CgTarget* T;
   Operand ap_op;
   if (!g) return;
   T = g->target;
   ap = api_pop(g);
   ap_op = api_force_local(g, &ap, api_sv_type(&ap));
   T->va_end_(T, ap_op);
   api_release(g, &ap);
 }
 
 void kit_cg_vararg_copy(KitCg* g) {
   ApiSValue src, dst;
   CgTarget* T;
   Operand src_op, dst_op;
   if (!g) return;
   T = g->target;
   src = api_pop(g);
   dst = api_pop(g);
   src_op = api_force_local(g, &src, api_sv_type(&src));
   dst_op = api_force_local(g, &dst, api_sv_type(&dst));
   T->va_copy_(T, dst_op, src_op);
   api_release(g, &src);
   api_release(g, &dst);
 }
 
 /* ============================================================
  * Memory operations (stubs)
  * ============================================================ */
 
 void kit_cg_memcpy(KitCg* g, uint64_t size, KitCgMemAccess dst_access,
                    KitCgMemAccess src_access) {
   ApiSValue src, dst;
   CgTarget* T;
   AggregateAccess agg;
   Operand dst_op, src_op;
   if (!g) return;
   api_local_const_memory_boundary(g);
   (void)src_access;
   if (size > UINT32_MAX) {
     compiler_panic(g->c, g->cur_loc, "KitCg: memcpy size exceeds CgTarget");
     return;
   }
   T = g->target;
   src = api_pop(g);
   dst = api_pop(g);
   api_require_pointer_value(g, "memcpy destination", api_sv_type(&dst));
   api_require_pointer_value(g, "memcpy source", api_sv_type(&src));
   dst_op = api_force_local(g, &dst, api_sv_type(&dst));
   src_op = api_force_local(g, &src, api_sv_type(&src));
   memset(&agg, 0, sizeof agg);
   agg.size = (u32)size;
   agg.align = dst_access.align ? dst_access.align : (u32)size;
   T->copy_bytes(T, dst_op, src_op, agg);
   api_release(g, &dst);
   api_release(g, &src);
 }
 
 void kit_cg_memmove(KitCg* g, uint64_t size, KitCgMemAccess dst_access,
                     KitCgMemAccess src_access) {
   ApiSValue src, dst;
   Operand args[3];
   if (!g) return;
   api_local_const_memory_boundary(g);
   (void)dst_access;
   (void)src_access;
   if (size > INT64_MAX) {
     compiler_panic(g->c, g->cur_loc, "KitCg: memmove size exceeds CgTarget");
     return;
   }
   src = api_pop(g);
   dst = api_pop(g);
   api_require_pointer_value(g, "memmove destination", api_sv_type(&dst));
   api_require_pointer_value(g, "memmove source", api_sv_type(&src));
   args[0] = api_force_local(g, &dst, api_sv_type(&dst));
   args[1] = api_force_local(g, &src, api_sv_type(&src));
   args[2] = api_op_imm((i64)size, builtin_id(KIT_CG_BUILTIN_I64));
   g->target->intrinsic(g->target, INTRIN_MEMMOVE, NULL, 0, args, 3);
   api_release(g, &dst);
   api_release(g, &src);
 }
 
 void kit_cg_memset(KitCg* g, uint8_t val, uint64_t size,
                    KitCgMemAccess dst_access) {
   ApiSValue dst;
   CgTarget* T;
   AggregateAccess agg;
   Operand dst_op, byte_val;
   if (!g) return;
   api_local_const_memory_boundary(g);
   if (size > UINT32_MAX) {
     compiler_panic(g->c, g->cur_loc, "KitCg: memset size exceeds CgTarget");
     return;
   }
   T = g->target;
   dst = api_pop(g);
   api_require_pointer_value(g, "memset destination", api_sv_type(&dst));
   dst_op = api_force_local(g, &dst, api_sv_type(&dst));
   byte_val = api_op_imm((i64)val, KIT_CG_TYPE_NONE);
   memset(&agg, 0, sizeof agg);
   agg.size = (u32)size;
   agg.align = dst_access.align ? dst_access.align : (u32)size;
   T->set_bytes(T, dst_op, byte_val, agg);
   api_release(g, &dst);
 }
 
 /* log2 of a {1,2,4,8} scale, else -1. */
 static int cg_scale_to_log2(u32 scale) {
   switch (scale) {
     case 1:
       return 0;
     case 2:
       return 1;
     case 4:
       return 2;
     case 8:
       return 3;
     default:
       return -1;
   }
 }
 
 void kit_cg_elem(KitCg* g, int64_t offset) {
   ApiSValue idx, base;
   CgTarget* T;
   KitCgTypeId base_ty, base_ptr_ty, elem_ty, idx_ty;
   const CgType* base_info;
   u32 elemsz;
   Operand base_op, idx_op;
   CGLocal base_local;
   if (!g) return;
   T = g->target;
   idx = api_pop(g);
   base = api_pop(g);
   base_ty = api_sv_type(&base);
   base_info = cg_type_get(g->c, base_ty);
   if (api_is_lvalue_sv(&base) || !base_info ||
       base_info->kind != KIT_CG_TYPE_PTR) {
     compiler_panic(g->c, g->cur_loc,
                    "KitCg: elem requires a pointer value base (decay an "
                    "array to a pointer first)");
     return;
   }
   elem_ty = base_info->ptr.pointee;
   base_ptr_ty = base_ty;
   elemsz = (u32)abi_cg_sizeof(g->c->abi, elem_ty);
   idx_ty = idx.type ? idx.type : idx.op.type;
   if (!idx_ty) idx_ty = builtin_id(KIT_CG_BUILTIN_I64);
   base_op = api_force_local(g, &base, base_ptr_ty);
   base_local = base_op.v.local;
   idx_op = api_force_local_unless_imm(g, &idx, idx_ty);
 
   /* Constant index folds entirely into the displacement — no instructions, just
    * a larger offset on the OPK_INDIRECT. */
   if (idx_op.kind == OPK_IMM) {
     i64 ofs = idx_op.v.imm * (i64)elemsz + offset;
     Operand place;
     if (ofs >= INT32_MIN && ofs <= INT32_MAX) {
       place = api_op_indirect(base_local, (i32)ofs, elem_ty);
     } else {
       CGLocal r = api_alloc_temp_local(g, base_ptr_ty);
       Operand ro = api_op_local(r, base_ptr_ty);
       T->binop(T, BO_IADD, ro, api_op_local(base_local, base_ptr_ty),
                api_op_imm(ofs, base_ptr_ty));
       place = api_op_indirect(r, 0, elem_ty);
     }
     api_release(g, &base);
     api_release(g, &idx);
     api_push(g, api_make_lv(place, elem_ty));
     return;
   }
 
   /* Dynamic index: build a scaled-index place so the backend emits one
    * [base + index*scale] addressing mode. A power-of-two element size rides in
    * log2_scale; any other size pre-multiplies the index once (scale 1). The
    * index is copied into a fresh local for unambiguous ownership. */
   {
     int lg2 = cg_scale_to_log2(elemsz);
     CGLocal ir = api_alloc_temp_local(g, idx_ty);
     Operand iro = api_op_local(ir, idx_ty);
     u8 log2_scale;
     /* Refresh idx_op: allocating `ir` may have materialized a delayed index. */
     idx_op = api_force_local_unless_imm(g, &idx, idx_ty);
     if (idx.op.kind == OPK_LOCAL) idx_op = idx.op;
     if (lg2 >= 0) {
       T->copy(T, iro, idx_op);
       log2_scale = (u8)lg2;
     } else {
       T->binop(T, BO_IMUL, iro, idx_op, api_op_imm((i64)elemsz, idx_ty));
       log2_scale = 0;
     }
     api_release(g, &base);
     api_release(g, &idx);
     api_push(g, api_make_lv(api_op_indirect_indexed(base_local, ir, log2_scale,
                                                     (i32)offset, elem_ty),
                             elem_ty));
   }
 }
 
 void kit_cg_field(KitCg* g, uint32_t field_index) {
   ApiSValue base;
   CgTarget* T;
   KitCgTypeId rec_ty;
   KitCgTypeId base_ty;
   KitCgTypeId field_ty;
   KitCgTypeId rec_ptr_ty;
   const CgType* rec_info;
   const ABIRecordLayout* layout;
   u32 field_offset;
   Operand result;
   CGLocal rr;
   if (!g) return;
   T = g->target;
   base = api_pop(g);
   api_ensure_local(g, &base);
   base_ty = api_sv_type(&base);
   if (!api_is_lvalue_sv(&base)) {
     compiler_panic(g->c, g->cur_loc,
                    "KitCg: field requires a record place; deref a pointer "
                    "first");
     api_release(g, &base);
     return;
   }
   rec_ty = base_ty;
   rec_ptr_ty = cg_type_ptr_to(g->c, rec_ty);
   layout = abi_cg_record_layout(g->c->abi, rec_ty);
   if (!layout || field_index >= layout->nfields) {
     compiler_panic(g->c, g->cur_loc, "KitCg: invalid field index");
     return;
   }
   rec_info = cg_type_get(g->c, rec_ty);
   if (!rec_info || rec_info->kind != KIT_CG_TYPE_RECORD ||
       field_index >= rec_info->record.nfields) {
     compiler_panic(g->c, g->cur_loc, "KitCg: invalid record base");
     return;
   }
   field_ty = rec_info->record.fields[field_index].type;
   field_offset = layout->fields[field_index].offset;
   if (layout->fields[field_index].bit_width != 0 ||
       (rec_info->record.fields[field_index].flags & KIT_CG_FIELD_BITFIELD) !=
           0) {
     Operand base_addr;
     ApiSValue sv;
     if (layout->fields[field_index].bit_width == 0) {
       compiler_panic(g->c, g->cur_loc, "KitCg: zero-width bit-field access");
       api_release(g, &base);
       return;
     }
     /* Project to a bit-field PLACE: the place addresses the enclosing storage
      * unit and carries the bit-field geometry from the record layout. A plain
      * load/store on this place performs the extract/insert; there is no
      * separate bit-field memop and no bit-field rider on KitCgMemAccess. */
     base_addr = api_lvalue_addr(g, &base, rec_ptr_ty);
     sv = api_make_lv(base_addr, field_ty);
     sv.bitfield.bit_offset = layout->fields[field_index].bit_offset;
     sv.bitfield.bit_width = layout->fields[field_index].bit_width;
     sv.bitfield.bit_storage_size = layout->fields[field_index].storage_size;
     sv.bitfield.bit_signed =
         rec_info->record.fields[field_index].bit_signed ? 1u : 0u;
     api_release(g, &base);
     api_push(g, sv);
     return;
   }
   if (base.op.kind == OPK_GLOBAL) {
     result =
         api_op_global(base.op.v.global.sym,
                       base.op.v.global.addend + (i64)field_offset, field_ty);
     api_push(g, api_make_lv(result, field_ty));
   } else if (base.op.kind == OPK_INDIRECT && field_offset <= (u32)INT32_MAX &&
              base.op.v.ind.ofs <= INT32_MAX - (i32)field_offset) {
     /* Fold the field offset into the displacement, preserving any index/scale
      * a preceding `elem` left so `p[i].f` stays one [base+index*scale+off]. */
     result = api_op_indirect_indexed(
         base.op.v.ind.base, base.op.v.ind.index, base.op.v.ind.log2_scale,
         base.op.v.ind.ofs + (i32)field_offset, field_ty);
     api_push(g, api_make_lv(result, field_ty));
   } else {
     Operand base_addr;
     rr = api_alloc_temp_local(g, rec_ptr_ty);
     base_addr = api_op_local(rr, rec_ptr_ty);
     T->addr_of(T, base_addr, base.op);
     api_release(g, &base);
     if (field_offset == 0) {
       result = base_addr;
     } else {
       CGLocal fr = api_alloc_temp_local(g, rec_ptr_ty);
       result = api_op_local(fr, rec_ptr_ty);
       T->binop(T, BO_IADD, result, base_addr,
                api_op_imm((i64)field_offset, rec_ptr_ty));
       api_release_temp_local(g, base_addr.v.local);
     }
     api_push(
         g, api_make_lv(api_op_indirect(result.v.local, 0, field_ty), field_ty));
   }
 }
 
 void kit_cg_field_bits(KitCg* g, uint16_t bit_offset, uint16_t bit_width,
                        uint32_t bit_storage_size, int bit_signed) {
   ApiSValue* top;
   if (!g || g->sp == 0) return;
   top = &g->stack[g->sp - 1u];
   if (!api_is_lvalue_sv(top)) {
     compiler_panic(g->c, g->cur_loc,
                    "KitCg: field_bits requires a place destination");
     return;
   }
   top->bitfield.bit_offset = bit_offset;
   top->bitfield.bit_width = bit_width;
   top->bitfield.bit_storage_size = bit_storage_size;
   top->bitfield.bit_signed = bit_signed ? 1u : 0u;
 }
 
 /* ============================================================
  * Calls / return
  * ============================================================ */
 
 /* Shared scaffolding for kit_cg_call / kit_cg_call_symbol. The two
  * public entry points differ only in how the callee is obtained and in
  * their pre-call stack-depth check; everything else (arg packaging, return
  * storage allocation, post-call release, result push) is identical. These
  * helpers carry the common shape and are the natural targets for any future
  * change that wants to vary call-shape policy (e.g. an ABI-driven storage
  * decision). */