kit

internal.h (34311B)

---

 #ifndef KIT_CG_INTERNAL_H
 #define KIT_CG_INTERNAL_H
 
 #include <kit/cg.h>
 #include <stdarg.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <string.h>
 
 #include "abi/abi.h"
 #include "asm/asm.h"
 #include "cg/cgtarget.h"
 #include "cg/type.h"
 #include "core/arena.h"
 #include "core/heap.h"
 #include "core/pool.h"
 #include "core/segvec.h"
 #include "core/slice.h"
 #include "core/strbuf.h"
 #include "debug/debug.h"
 #include "obj/obj.h"
 
 typedef uint32_t ObjSymId;
 
 typedef enum SResidency {
   RES_INHERENT,
   RES_LOCAL,
   RES_FIXED_LOCAL,
 } SResidency;
 
 typedef enum ApiSValueKind {
   SV_OPERAND,
   SV_CMP,
   SV_ARITH,
 } ApiSValueKind;
 
 typedef enum ApiDelayedArithKind {
   API_DELAYED_UNOP,
   API_DELAYED_BINOP,
 } ApiDelayedArithKind;
 
 typedef struct ApiDelayedCmp {
   Operand a;
   Operand b;
   CmpOp op;
   u8 a_owned;
   u8 b_owned;
   u8 pad[2];
 } ApiDelayedCmp;
 
 typedef struct ApiDelayedArith {
   Operand a;
   Operand b;
   BinOp bin_op;
   UnOp un_op;
   u8 kind;
   u8 a_owned;
   u8 b_owned;
   u8 pad;
 } ApiDelayedArith;
 
 /* The delayed cmp/arith payload of an SV_CMP / SV_ARITH value. It is held off
  * the hot value-stack node (ApiSValue carries only an ApiDelayed*, NULL for the
  * common SV_OPERAND case) so the node stays small and its per-push construction
  * touches no delayed state. Payloads are pooled per function (api_delayed_alloc
  * / _free, arena-backed with a freelist; see fold.c). `next_free` overlays the
  * payload while it sits on the freelist. */
 typedef union ApiDelayed {
   ApiDelayedCmp cmp;
   ApiDelayedArith arith;
   union ApiDelayed* next_free;
 } ApiDelayed;
 
 /* The scalar wide-class tag (ApiWideKind) and the packed per-type
  * classification it feeds (api_type_class) live in cg/type.h, with the rest of
  * the type-property queries. The value stack only stores the result (the
  * one-byte tag below) and reads it back. */
 
 /* Bit-field geometry carried by a bit-field PLACE. `kit_cg_field` fills this
  * from the record layout when it projects to a bit-field; a plain load/store on
  * the carrying place then performs the extract/insert. The storage MemAccess is
  * rebuilt from the place operand + field type at load/store time, so only the
  * bit geometry needs to ride on the place. width == 0 means "not a bit-field".
  *
  * `wide_kind` (ApiWideKind) rides in this struct's trailing slack: it is the
  * only one-byte hole in the value-stack node, and keeping it here leaves the
  * node size unchanged (static_assert in value.c). It is node state, not bit
  * geometry, so kit_cg_field_bits writes only the bit_* members and leaves it
  * intact, and node copies (dup/swap/rot) carry it along with the type. */
 typedef struct ApiBitField {
   u16 bit_offset;       /* target-endian bit offset within the storage unit */
   u16 bit_width;        /* 0 => the place is not a bit-field place */
   u32 bit_storage_size; /* storage-unit size in bytes */
   u8 bit_signed;        /* signed extraction on load */
   u8 wide_kind;         /* ApiWideKind, cached at api_push (node slack) */
   u8 pad[2];
 } ApiBitField;
 
 typedef struct ApiSValue {
   Operand op;
   /* Pooled delayed cmp/arith payload, or NULL for the common SV_OPERAND value.
    * A pointer (not the 64-byte union inline) keeps the node small; only the
    * SV_CMP / SV_ARITH producers in fold.c allocate one. */
   ApiDelayed* delayed;
   const void* lang_type; /* opaque frontend-owned slot fact */
   KitCgTypeId type;
   KitCgLocal source_local;
   ApiBitField bitfield; /* bit_width != 0 marks a bit-field PLACE subkind */
   u16 lang_flags;       /* frontend-defined flags; CG only copies/clears */
   u16 flags;            /* packed ApiSValueKind/SResidency/pinned/lvalue */
 } ApiSValue;
 
 typedef struct ApiConstValue {
   KitCgConstInt value;
   KitCgTypeId type;
 } ApiConstValue;
 
 #define API_CG_STACK_INITIAL 16u
 
 #define API_SV_KIND_SHIFT 0u
 #define API_SV_RES_SHIFT 2u
 #define API_SV_PINNED_SHIFT 4u
 #define API_SV_LVALUE_SHIFT 5u
 #define API_SV_KIND_MASK 0x3u
 #define API_SV_RES_MASK 0x3u
 #define API_SV_BOOL_MASK 0x1u
 #define API_SV_PACK(kind_, res_, pinned_, lvalue_)                      \
   ((u16)((((u16)(kind_) & API_SV_KIND_MASK) << API_SV_KIND_SHIFT) |     \
          (((u16)(res_) & API_SV_RES_MASK) << API_SV_RES_SHIFT) |        \
          (((u16)(pinned_) & API_SV_BOOL_MASK) << API_SV_PINNED_SHIFT) | \
          (((u16)(lvalue_) & API_SV_BOOL_MASK) << API_SV_LVALUE_SHIFT)))
 
 static inline ApiSValueKind api_sv_kind(const ApiSValue* sv) {
   return (ApiSValueKind)((sv->flags >> API_SV_KIND_SHIFT) & API_SV_KIND_MASK);
 }
 
 static inline void api_sv_set_kind(ApiSValue* sv, ApiSValueKind kind) {
   sv->flags = (u16)((sv->flags & ~(API_SV_KIND_MASK << API_SV_KIND_SHIFT)) |
                     (((u16)kind & API_SV_KIND_MASK) << API_SV_KIND_SHIFT));
 }
 
 static inline SResidency api_sv_res(const ApiSValue* sv) {
   return (SResidency)((sv->flags >> API_SV_RES_SHIFT) & API_SV_RES_MASK);
 }
 
 static inline void api_sv_set_res(ApiSValue* sv, SResidency res) {
   sv->flags = (u16)((sv->flags & ~(API_SV_RES_MASK << API_SV_RES_SHIFT)) |
                     (((u16)res & API_SV_RES_MASK) << API_SV_RES_SHIFT));
 }
 
 static inline int api_sv_pinned(const ApiSValue* sv) {
   return (int)((sv->flags >> API_SV_PINNED_SHIFT) & API_SV_BOOL_MASK);
 }
 
 static inline void api_sv_set_pinned(ApiSValue* sv, int pinned) {
   sv->flags = (u16)((sv->flags & ~(API_SV_BOOL_MASK << API_SV_PINNED_SHIFT)) |
                     (((u16)pinned & API_SV_BOOL_MASK) << API_SV_PINNED_SHIFT));
 }
 
 static inline int api_sv_lvalue_flag(const ApiSValue* sv) {
   return (int)((sv->flags >> API_SV_LVALUE_SHIFT) & API_SV_BOOL_MASK);
 }
 
 static inline void api_sv_set_lvalue(ApiSValue* sv, int lvalue) {
   sv->flags = (u16)((sv->flags & ~(API_SV_BOOL_MASK << API_SV_LVALUE_SHIFT)) |
                     (((u16)lvalue & API_SV_BOOL_MASK) << API_SV_LVALUE_SHIFT));
 }
 
 /* Largest scalar the codegen lowers as a native (lock-free) atomic. All
  * current targets — aa64, x64, rv64, wasm32 — provide 8-byte (i64-width)
  * atomics, so this is both the legality ceiling and the lock-free ceiling.
  * Note it is NOT the pointer width: wasm32 has 4-byte pointers but 8-byte
  * atomics. */
 #define CG_MAX_ATOMIC_SIZE 8u
 
 /* Arity that fits inline in an ApiCgScope without a heap allocation. The common
  * void/single-result scope (every C/toy statement) stays inline; a multi-value
  * block whose param or result count exceeds this spills its carry-local vectors
  * to one heap block (see api_scope_setup_sig), freed at scope_end. */
 #define API_CG_SCOPE_SIG_INLINE 4u
 
 typedef struct ApiCgScope {
   Label break_lbl;
   Label continue_lbl;
   CGScope target_scope;
   /* The carry locals (and their resolved types) that move the scope's results
    * across its exit edges and — for a loop — its params across back edges. A
    * value is stored into result_locals[i] before a break/end and reloaded after
    * the break label; loop params are snapshotted into param_locals[i] and
    * reloaded after the continue label. The pointers target the inline buffers
    * for small arity (no heap) or one heap block for larger; nresults<=1 and
    * nparams==0 reproduces the old single-result path byte-for-byte. */
   CGLocal* result_locals;
   KitCgTypeId* result_types;
   const void** result_lang_types;
   u16* result_lang_flags;
   CGLocal* param_locals;
   KitCgTypeId* param_types;
   const void** param_lang_types;
   u16* param_lang_flags;
   u32 nresults;
   u32 nparams;
   CGLocal result_locals_inl[API_CG_SCOPE_SIG_INLINE];
   KitCgTypeId result_types_inl[API_CG_SCOPE_SIG_INLINE];
   const void* result_lang_types_inl[API_CG_SCOPE_SIG_INLINE];
   u16 result_lang_flags_inl[API_CG_SCOPE_SIG_INLINE];
   CGLocal param_locals_inl[API_CG_SCOPE_SIG_INLINE];
   KitCgTypeId param_types_inl[API_CG_SCOPE_SIG_INLINE];
   const void* param_lang_types_inl[API_CG_SCOPE_SIG_INLINE];
   u16 param_lang_flags_inl[API_CG_SCOPE_SIG_INLINE];
   u32* heap_block; /* non-NULL when u32 arity vectors spilled to heap */
   const void** heap_lang_types_block;
   u16* heap_lang_flags_block;
   u32 generation;
   u8 active;
   u8 pad[3];
 } ApiCgScope;
 
 /* Upper bound on live (LIFO-nested) control scopes. This is the limit the
  * scope handle can encode, not a preallocated size: g->scopes grows on
  * demand (api_grow_scopes). The handle packs the 1-based scope index into
  * its low 16 bits (api_scope_handle), so a depth of 0xffff is the most that
  * remains addressable. Real translation units nest a few hundred deep at
  * most; this ceiling exists only to keep the handle representable. */
 #define API_CG_MAX_SCOPES 0xffffu
 
 typedef enum ApiSourceLocalKind {
   API_SOURCE_LOCAL_AUTO,
   API_SOURCE_LOCAL_PARAM,
 } ApiSourceLocalKind;
 
 typedef struct ApiSourceLocal {
   KitCgTypeId type;
   KitSym name;
   KitCgLocalAttrs attrs;
   SrcLoc loc;
   CGLocalDesc desc;
   CGLocal storage;
   i64 const_value;
   /* Intrusive link for the const-active list (KitCg.const_head; see fold.c).
    * KIT_CG_LOCAL_NONE terminates. const_listed marks membership so a re-store
    * does not double-link; it stays set (the entry lingers as a cleared no-op)
    * until the next boundary drains the list. */
   KitCgLocal const_next;
   u32 param_index;
   u8 kind;
   u8 const_valid;
   u8 const_listed;
   u8 pad[1];
 } ApiSourceLocal;
 
 struct KitCg {
   Compiler* c;
   ObjBuilder* obj;
   CgTarget* target;
   Debug* debug;
   KitCgUnitOptions cur_unit;
   u32 nsource_units;
   /* Monotonic, nonzero per source unit (set at kit_cg_begin_unit). Used to tell
    * a same-TU re-definition (legal tentative-definition coalescing) apart from
    * a genuine cross-TU contribution that must go through symresolve_merge. */
   u32 cur_unit_seq;
   u8 unit_active;
   u8 finished;
   u8 lifecycle_pad[2];
 
   ApiSValue* stack;
   ApiConstValue* const_stack;
   u32 sp;
   u32 cap;
   u32 unevaluated_depth;
 
   /* -O0 transient liveness. local_refs[h] = number of live value-stack entries
    * that reference CGLocal handle h (maintained at api_push/api_pop, plus a
    * pointer-range-guarded reseat at in-place operand changes). local_is_temp[h]
    * marks an api_alloc_temp_local transient — the only locals the single-pass
    * backend recycles (Fix A) or coalesces. The count is used only as a
    * fast-reject prefilter: a transient is treated as dead solely when the count
    * reads 0 AND a confirming stack scan (api_temp_dead) agrees, so any reseat
    * gap costs at most a missed optimization, never a miscompile. Sized on
    * demand by handle; cleared per function. NULL/0 until first use. */
   u32* local_refs;     /* live value-stack reference count per handle */
   u32* local_temp_gen; /* == func_gen iff the handle is this function's temp */
   u32 local_track_cap;
   u32 func_gen; /* bumped each function; stamps temps without O(cap) clears */
   u8 coalesce;  /* -O0 copy/dup coalescing + finer reclaim enabled */
   u8 coalesce_known; /* coalesce resolved from env (once) */
   u8 coalesce_pad[2];
 
   /* Off-node pool for SV_CMP / SV_ARITH delayed payloads. The arena is reset
    * per function (api_delayed_reset at func_begin, where the stack is also
    * dropped); delayed_free is an intrusive freelist that reuses payloads within
    * a function so the arena only grows to the live delayed-value working set.
    */
   Arena delayed_arena;
   ApiDelayed* delayed_free;
   u8 delayed_arena_init;
   u8 delayed_pad[3];
 
   ApiSourceLocal* locals;
   u32 nlocals;
   u32 locals_cap;
   /* Head of the const-active list threaded through ApiSourceLocal.const_next:
    * the locals that currently hold (or recently held, pending a boundary) a
    * forwardable constant. Lets the const-tracker clear in O(#tracked) rather
    * than scanning all nlocals on every store/call/branch/address-of. */
   KitCgLocal const_head;
 
   KitCgTypeId fn_ret_type; /* KIT_CG_TYPE_NONE/void == no result */
   SrcLoc cur_loc;
 
   CGFuncDesc fn_desc;
   CGParamDesc fn_params[64];
 
   KitCgTypeId* sym_types;
   KitCgDecl* sym_attrs;
   u32 sym_cap;
 
   DebugTypeId* debug_type_cache; /* indexed by KitCgTypeId */
   u8* debug_type_state;          /* 0 empty, 1 building, 2 complete */
   u32 debug_type_cap;
 
   /* Per-ObjSymId: the cur_unit_seq of the unit that last *defined* this symbol
    * (0 = not defined by any unit yet). Distinct from sym_attrs, which is reset
    * on every decl; this is written only when a definition is emitted. */
   u32* sym_def_seq;
   u32 sym_def_seq_cap;
 
   ApiCgScope* scopes; /* grows on demand; see api_grow_scopes */
   u32 nscopes;
   u32 scopes_cap;
   u32 scope_generation;
 
   u32 rodata_counter;
   int opt_level;
   u8 check_only;
   u8 function_sections;
   u8 data_sections;
   u8 stack_protector_mode; /* KitStackProtectorMode */
   u8 stack_protected;
   u8 stack_guard_tls;
   u8 stack_protector_pad[1];
   KitCgTypeId stack_guard_type;
   KitCgLocal stack_guard_local;
   KitCgSym stack_guard_sym;
   KitCgSym stack_fail_sym;
 
   ObjSecId data_sec;
   ObjSymId data_sym;
   u32 data_base;
   u64 data_size;
   u8 data_local_static_target;
   u8 data_atomize;
   u8 data_retain;
   u8 data_discard;
   u8 data_tls_collect;
   u8 data_tls_zero_fill;
   u8 data_tls_pad[2];
   u32 data_tls_align;
   Buf data_tls_bytes;
   ObjTlsReloc* data_tls_relocs;
   u32 data_tls_nrelocs;
   u32 data_tls_relocs_cap;
 };
 
 /* ------------------------------------------------------------------
  * API-misuse / invariant checks (debug-only)
  *
  * The cg.h contract treats a frontend that drives the value stack, scope
  * handles, or operand kinds incorrectly as a kit bug, not a recoverable
  * condition. CG_REQUIRE / CG_BUG state those invariants: in a debug build they
  * fire the same rich FATAL diagnostic + longjmp as a hand-written
  * compiler_panic, but under NDEBUG the whole check -- condition, message, and
  * argument evaluation -- is elided, so the release hot path pays nothing.
  *
  *   CG_REQUIRE(g, cond, fmt, ...)  assert the OK condition `cond` holds
  *   CG_BUG(g, fmt, ...)            mark an unreachable else/default arm
  *
  * Use ONLY for invariants a correct caller can never violate. Genuine resource
  * failures (OOM, object-emission failures), unsupported-operation guards whose
  * elision would silently miscompile, size/truncation limits, and configuration
  * errors stay as unconditional compiler_panic so they trap in every build.
  * ------------------------------------------------------------------ */
 #ifdef NDEBUG
 /* Elided: the sizeof keeps every operand in an unevaluated context so a value
  * or `who`-style diagnostic argument used only by the check does not become an
  * unused variable/parameter under -Werror, while emitting no runtime code. */
 #define CG_REQUIRE(g, cond, ...) ((void)sizeof((g), (cond), __VA_ARGS__, 0))
 #define CG_BUG(g, ...) ((void)sizeof((g), __VA_ARGS__, 0))
 #else
 #define CG_REQUIRE(g, cond, ...) \
   ((cond) ? (void)0 : compiler_panic((g)->c, (g)->cur_loc, __VA_ARGS__))
 #define CG_BUG(g, ...) compiler_panic((g)->c, (g)->cur_loc, __VA_ARGS__)
 #endif
 
 void cg_api_fini(Compiler*);
 
 void api_cg_binop(KitCg* g, BinOp iop, u32 flags);
 void api_cg_unop(KitCg* g, UnOp iop, u32 flags);
 void api_cg_cmp(KitCg* g, CmpOp cop);
 void api_cg_convert_kind(KitCg* g, KitCgTypeId dst_type, ConvKind ck);
 void kit_cg_int_binop(KitCg* g, KitCgIntBinOp op, uint32_t flags);
 void kit_cg_int_unop(KitCg* g, KitCgIntUnOp op, uint32_t flags);
 void kit_cg_int_cmp(KitCg* g, KitCgIntCmpOp op);
 const char* api_i128_binop_helper(BinOp op);
 int api_i128_cmp_is_unsigned(CmpOp op);
 const char* api_f128_binop_helper(KitCgFpBinOp op);
 int api_f128_stack_top(KitCg* g, u32 depth);
 void api_f128_call_unary(KitCg* g, const char* name, KitCgTypeId ret,
                          KitCgTypeId param);
 void kit_cg_fp_binop(KitCg* g, KitCgFpBinOp op, uint32_t flags);
 void kit_cg_fp_unop(KitCg* g, KitCgFpUnOp op, uint32_t flags);
 void kit_cg_fp_cmp(KitCg* g, KitCgFpCmpOp op);
 void kit_cg_sext(KitCg* g, KitCgTypeId dst);
 void kit_cg_zext(KitCg* g, KitCgTypeId dst);
 void kit_cg_trunc(KitCg* g, KitCgTypeId dst);
 void kit_cg_ptr_to_int(KitCg* g, KitCgTypeId dst);
 void kit_cg_int_to_ptr(KitCg* g, KitCgTypeId dst);
 void kit_cg_bitcast(KitCg* g, KitCgTypeId dst);
 void kit_cg_fpext(KitCg* g, KitCgTypeId dst);
 void kit_cg_fptrunc(KitCg* g, KitCgTypeId dst);
 void kit_cg_sint_to_float(KitCg* g, KitCgTypeId dst, KitCgRounding rounding);
 void kit_cg_uint_to_float(KitCg* g, KitCgTypeId dst, KitCgRounding rounding);
 void kit_cg_float_to_sint(KitCg* g, KitCgTypeId dst, KitCgRounding rounding);
 void kit_cg_float_to_uint(KitCg* g, KitCgTypeId dst, KitCgRounding rounding);
 IntrinKind api_map_intrinsic(KitCg* g, KitCgIntrinsic intrin,
                              KitCgTypeId result_type);
 int api_intrinsic_is_void(KitCgIntrinsic intrin);
 int api_intrinsic_is_overflow(KitCgIntrinsic intrin);
 const char* api_intrinsic_name(KitCgIntrinsic intrin);
 void kit_cg_intrinsic(KitCg* g, KitCgIntrinsic intrin, uint32_t nargs,
                       KitCgTypeId result_type);
 KitCgTypeId api_atomic_pointee(KitCg* g, KitCgTypeId pty, const char* who);
 const char* api_sym_cstr(KitCg* g, KitSym sym);
 int api_asm_parse_match_index(const char* s);
 const char* api_asm_constraint_body(const char* s);
 int api_asm_is_early_clobber(const char* s);
 int api_asm_is_reg_constraint(char c);
 void api_asm_memory_clobber_sv(KitCg* g, ApiSValue* sv, CGLocal local);
 void kit_cg_inline_asm(KitCg* g, KitCgInlineAsm asm_block);
 void kit_cg_file_scope_asm(KitCg* g, KitSlice asm_source);
 MemAccess api_mem_for_atomic(KitCg* g, KitCgTypeId val_ty);
 int kit_cg_atomic_is_legal(KitCompiler* c, KitCgMemAccess access,
                            KitCgMemOrder order);
 int kit_cg_atomic_is_lock_free(KitCompiler* c, KitCgMemAccess access);
 void kit_cg_atomic_load(KitCg* g, KitCgMemAccess access, KitCgMemOrder order);
 void kit_cg_atomic_store(KitCg* g, KitCgMemAccess access, KitCgMemOrder order);
 void kit_cg_atomic_rmw(KitCg* g, KitCgMemAccess access, KitCgAtomicOp op,
                        KitCgMemOrder order);
 void kit_cg_atomic_cmpxchg(KitCg* g, KitCgMemAccess access,
                            KitCgMemOrder success, KitCgMemOrder failure,
                            int weak);
 void kit_cg_atomic_fence(KitCg* g, KitCgMemOrder order);
 CGLocal* api_alloc_call_args(KitCg* g, u32 nargs);
 void api_pack_call_arg(KitCg* g, CGLocal* out, KitCgTypeId fty, u32 idx);
 CGLocal api_alloc_call_result(KitCg* g, KitCgTypeId ret_ty);
 void api_push_call_result(KitCg* g, CGLocal result, KitCgTypeId ret_ty);
 void kit_cg_call(KitCg* g, uint32_t nargs, KitCgTypeId fn_type,
                  KitCgCallAttrs attrs);
 void api_call_symbol_common(KitCg* g, KitCgSym sym, uint32_t nargs,
                             KitCgCallAttrs attrs);
 void kit_cg_call_symbol(KitCg* g, KitCgSym sym, uint32_t nargs,
                         KitCgCallAttrs attrs);
 void kit_cg_ret(KitCg* g);
 void api_stack_protector_check(KitCg* g);
 KitCgLabel kit_cg_label_new(KitCg* g);
 void kit_cg_label_place(KitCg* g, KitCgLabel label);
 void kit_cg_jump(KitCg* g, KitCgLabel label);
 void api_branch_if(KitCg* g, ApiSValue* v, int branch_when_true, Label label);
 void kit_cg_branch_true(KitCg* g, KitCgLabel label);
 void kit_cg_branch_false(KitCg* g, KitCgLabel label);
 void kit_cg_switch(KitCg* g, KitCgSwitch sw);
 void kit_cg_push_label_addr(KitCg* g, KitCgLabel label, KitCgTypeId ptr_type);
 void kit_cg_computed_goto(KitCg* g, const KitCgLabel* valid_targets,
                           uint32_t ntargets);
 
 void kit_cg_field_bits(KitCg* g, uint16_t bit_offset, uint16_t bit_width,
                        uint32_t bit_storage_size, int bit_signed);
 void kit_cg_unreachable(KitCg* g);
 KitCgScope api_scope_handle(u32 idx, u32 generation);
 ApiCgScope* api_scope_from_handle(KitCg* g, KitCgScope scope, int require_top,
                                   const char* who);
 int api_scope_has_result(const ApiCgScope* s);
 void api_scope_store_results(KitCg* g, ApiCgScope* s);
 void api_scope_push_results(KitCg* g, ApiCgScope* s);
 KitCgScope kit_cg_scope_begin(KitCg* g);
 KitCgScope kit_cg_scope_begin_value(KitCg* g, KitCgTypeId result_type);
 KitCgScope kit_cg_block_begin(KitCg* g);
 KitCgScope kit_cg_block_begin_value(KitCg* g, KitCgTypeId result_type);
 KitCgScope kit_cg_scope_begin_sig(KitCg* g, const KitCgScopeSig* sig);
 KitCgScope kit_cg_block_begin_sig(KitCg* g, const KitCgScopeSig* sig);
 void kit_cg_scope_store_params(KitCg* g, KitCgScope scope);
 void kit_cg_scope_end(KitCg* g, KitCgScope scope);
 void kit_cg_scope_end_unreachable(KitCg* g, KitCgScope scope);
 void kit_cg_break(KitCg* g, KitCgScope scope);
 void kit_cg_break_true(KitCg* g, KitCgScope scope);
 void kit_cg_break_false(KitCg* g, KitCgScope scope);
 void kit_cg_continue(KitCg* g, KitCgScope scope);
 void kit_cg_continue_true(KitCg* g, KitCgScope scope);
 void kit_cg_continue_false(KitCg* g, KitCgScope scope);
 void kit_cg_alloca(KitCg* g, uint32_t align, KitCgTypeId result_ptr_type);
 void kit_cg_vararg_start(KitCg* g);
 void kit_cg_vararg_next(KitCg* g, KitCgTypeId type);
 void kit_cg_vararg_end(KitCg* g);
 void kit_cg_vararg_copy(KitCg* g);
 void kit_cg_memcpy(KitCg* g, uint64_t size, KitCgMemAccess dst_access,
                    KitCgMemAccess src_access);
 void kit_cg_memmove(KitCg* g, uint64_t size, KitCgMemAccess dst_access,
                     KitCgMemAccess src_access);
 void kit_cg_memset(KitCg* g, uint8_t val, uint64_t size,
                    KitCgMemAccess dst_access);
 void kit_cg_data_begin(KitCg* g, KitCgSym cg_sym, KitCgDataDefAttrs attrs);
 void kit_cg_data_common(KitCg* g, KitCgSym cg_sym, uint64_t size,
                         uint32_t align);
 void kit_cg_data_align(KitCg* g, uint32_t align);
 void kit_cg_data_pad(KitCg* g, uint64_t size, uint8_t value);
 void kit_cg_data_int(KitCg* g, uint64_t value, KitCgTypeId type);
 void kit_cg_data_float(KitCg* g, double value, KitCgTypeId type);
 void kit_cg_data_bytes(KitCg* g, const uint8_t* data, size_t len);
 void kit_cg_data_zero(KitCg* g, uint64_t size);
 void api_cg_data_reloc(KitCg* g, KitCgSym target, int64_t addend,
                        uint32_t width, int pcrel);
 void kit_cg_data_addr(KitCg* g, KitCgSym target, int64_t addend, uint32_t width,
                       uint32_t address_space);
 void kit_cg_data_label_addr(KitCg* g, KitCgLabel target, int64_t addend,
                             uint32_t width, uint32_t address_space);
 void kit_cg_data_pcrel(KitCg* g, KitCgSym target, int64_t addend,
                        uint32_t width);
 void kit_cg_data_symdiff(KitCg* g, KitCgSym lhs, KitCgSym rhs, int64_t addend,
                          uint32_t width);
 void kit_cg_data_end(KitCg* g);
 ObjSymId api_emit_label_table(KitCg* g, const Label* labels, u32 n);
 DebugTypeId api_debug_type(KitCg* g, KitCgDebugType debug_type,
                            KitCgTypeId fallback_type);
 int api_local_requires_memory(KitCg* g, KitCgTypeId ty, KitCgLocalAttrs attrs);
 KitCgLocal api_local_handle(u32 index);
 int api_grow_locals(KitCg* g, u32 want);
 int api_grow_scopes(KitCg* g, u32 want);
 ApiSourceLocal* api_local_from_handle(KitCg* g, KitCgLocal local);
 CGLocal api_frame_local_storage(KitCg* g, const CGLocalDesc* d);
 KitCgLocal kit_cg_local(KitCg* g, KitCgTypeId type, KitCgLocalAttrs attrs);
 KitCgLocal kit_cg_param(KitCg* g, uint32_t index, KitCgTypeId type,
                         KitCgLocalAttrs attrs);
 void kit_cg_push_int(KitCg* g, uint64_t value, KitCgTypeId type);
 void kit_cg_push_float(KitCg* g, double value, KitCgTypeId type);
 void kit_cg_push_null(KitCg* g, KitCgTypeId ptr_type);
 KitCgSym kit_cg_const_data(KitCg* g, const uint8_t* data, size_t len,
                            uint32_t align, KitCgTypeId pointee_type);
 void api_push_local_lvalue(KitCg* g, CGLocal local, KitCgTypeId type);
 void api_push_source_local_lvalue(KitCg* g, KitCgLocal local, CGLocal storage,
                                   KitCgTypeId type);
 void kit_cg_push_local(KitCg* g, KitCgLocal local);
 void kit_cg_push_local_addr(KitCg* g, KitCgLocal local);
 void kit_cg_push_symbol_addr(KitCg* g, KitCgSym sym, int64_t addend);
 void kit_cg_load(KitCg* g, KitCgMemAccess access);
 void kit_cg_addr(KitCg* g);
 void kit_cg_deref(KitCg* g, int64_t offset);
 void kit_cg_store(KitCg* g, KitCgMemAccess access);
 void kit_cg_store_keep(KitCg* g, KitCgMemAccess access);
 void kit_cg_field_at(KitCg* g, int64_t byte_offset, KitCgTypeId field_type);
 void kit_cg_dup(KitCg* g);
 void kit_cg_dup2(KitCg* g);
 void kit_cg_swap(KitCg* g);
 void kit_cg_drop(KitCg* g);
 KitCgSlotInfo kit_cg_slot_info(KitCg* g, uint32_t depth_from_top);
 KitCgTypeId kit_cg_slot_cg_type(KitCg* g, uint32_t depth_from_top);
 const void* kit_cg_slot_lang_type(KitCg* g, uint32_t depth_from_top);
 uint16_t kit_cg_slot_lang_flags(KitCg* g, uint32_t depth_from_top);
 void kit_cg_retag_top(KitCg* g, const void* lang_type, uint16_t lang_flags);
 void kit_cg_retag_at(KitCg* g, uint32_t depth_from_top, const void* lang_type,
                      uint16_t lang_flags);
 void kit_cg_set_top_flags(KitCg* g, uint16_t set, uint16_t clear);
 uint32_t kit_cg_stack_depth(KitCg* g);
 int kit_cg_top_const_int(KitCg* g, int64_t* out_value);
 void kit_cg_rot3(KitCg* g);
 KitStatus kit_cg_new(KitCompiler* c, KitCg** cg_out);
 KitStatus kit_cg_begin(KitCg* g, KitObjBuilder* out,
                        const KitCodeOptions* opts);
 KitStatus kit_cg_begin_unit(KitCg* g, const KitCgUnitOptions* opts);
 KitStatus kit_cg_end_unit(KitCg* g);
 KitStatus kit_cg_finish(KitCg* g, const KitCgFinishOptions* opts);
 KitStatus kit_cg_detach(KitCg* g);
 KitStatus kit_cg_abort(KitCg* g);
 void kit_cg_free(KitCg* g);
 void kit_cg_set_loc(KitCg* g, KitSrcLoc loc);
 KitCgSym kit_cg_decl(KitCg* g, KitCgDecl decl);
 KitCgSym kit_cg_alias(KitCg* g, KitCgAlias alias);
 void kit_cg_func_begin(KitCg* g, KitCgSym cg_sym);
 void kit_cg_func_begin_attrs(KitCg* g, KitCgSym cg_sym, KitCgFuncAttrs attrs);
 void kit_cg_func_end(KitCg* g);
 SymBind api_map_bind(KitSymBind b);
 SymVis api_map_vis(KitCgVisibility v);
 SymKind api_decl_sym_kind(KitCgDecl decl);
 Sym api_cg_symbol_section_name(KitCg* g, Slice base, KitSym linkage_name);
 void api_remember_sym(KitCg* g, ObjSymId sym, KitCgTypeId ty, KitCgDecl decl);
 KitCgTypeId api_sym_type(KitCg* g, KitCgSym sym);
 KitCgDecl api_sym_attrs(KitCg* g, KitCgSym sym);
 int api_sym_is_tls(KitCg* g, KitCgSym sym);
 RelocKind api_data_reloc_kind(int pcrel, uint32_t width);
 SrcLoc api_no_loc(void);
 /* Type-property predicates (api_type_is_float / api_is_*_type) and the type
  * classification (api_type_class) are declared in cg/type.h. */
 
 /* Delayed cmp/arith payload pool (fold.c). alloc returns a payload from the
  * per-function freelist or the arena; free returns it for reuse; reset drops
  * the whole pool at a function boundary. */
 ApiDelayed* api_delayed_alloc(KitCg* g);
 void api_delayed_free(KitCg* g, ApiDelayed* d);
 void api_delayed_reset(KitCg* g);
 Operand api_op_imm(i64 v, KitCgTypeId ty);
 Operand api_op_local(CGLocal r, KitCgTypeId ty);
 Operand api_op_global(ObjSymId sym, i64 addend, KitCgTypeId ty);
 Operand api_op_indirect(CGLocal base, i32 ofs, KitCgTypeId ty);
 Operand api_op_indirect_indexed(CGLocal base, CGLocal index, u8 log2_scale,
                                 i32 ofs, KitCgTypeId ty);
 u8 api_residency_for(const Operand* o);
 ApiSValue api_make_sv(Operand op, KitCgTypeId ty);
 ApiSValue api_make_lv(Operand op, KitCgTypeId ty);
 ApiSValue api_make_sv_with_local_ownership(Operand op, KitCgTypeId ty,
                                            int owned);
 KitCgTypeId api_sv_type(const ApiSValue* sv);
 int api_operand_can_address(const Operand* o);
 int api_sv_op_is(const ApiSValue* sv, OpKind kind);
 int api_sv_op_is_local_or_imm(const ApiSValue* sv);
 int api_is_lvalue_sv(const ApiSValue* sv);
 int api_sv_is_bitfield(const ApiSValue* sv);
 MemAccess api_mem_for_bitfield(KitCg* g, const ApiSValue* sv,
                                const Operand* storage, KitCgTypeId field_ty);
 void api_stack_grow(KitCg* g, u32 want);
 void api_push(KitCg* g, ApiSValue v);
 ApiSValue api_pop(KitCg* g);
 int api_unevaluated(KitCg* g);
 ApiSValue api_uneval_value(KitCg* g, KitCgTypeId type);
 ApiSValue api_uneval_place(KitCg* g, KitCgTypeId type);
 ApiConstValue api_const_unknown(KitCgTypeId type);
 ApiConstValue api_const_from_sv(KitCg* g, const ApiSValue* sv);
 ApiConstValue api_const_at(KitCg* g, u32 depth);
 void api_const_set_top(KitCg* g, ApiConstValue value);
 void api_const_set_at(KitCg* g, u32 depth, ApiConstValue value);
 void api_const_copy_top_from(KitCg* g, ApiConstValue value);
 ApiConstValue api_const_for_push(KitCg* g, KitCgTypeId type,
                                  const KitCgConstInt* value);
 int api_const_fold_binop(KitCg* g, BinOp op, KitCgTypeId type, ApiConstValue a,
                          ApiConstValue b, u32 flags, ApiConstValue* out);
 int api_const_fold_unop(KitCg* g, UnOp op, KitCgTypeId type, ApiConstValue a,
                         u32 flags, ApiConstValue* out);
 int api_const_fold_cmp(KitCg* g, CmpOp op, ApiConstValue a, ApiConstValue b,
                        ApiConstValue* out);
 int api_const_fold_convert(KitCg* g, ConvKind ck, KitCgTypeId src_type,
                            KitCgTypeId dst_type, ApiConstValue in,
                            ApiConstValue* out);
 ApiConstValue api_const_int_result(KitCg* g, KitCgTypeId type, u64 lo, u64 hi,
                                    int is_signed);
 CGLocal api_local_of_sv(const ApiSValue* sv);
 void api_set_owned_local(ApiSValue* sv, CGLocal r);
 KitCgTypeId api_owned_local_type(KitCg* g, const ApiSValue* sv);
 CGLocal api_alloc_temp_local(KitCg* g, KitCgTypeId ty);
 MemAccess api_mem_for_lvalue(KitCg* g, const Operand* lv, KitCgTypeId ty);
 MemAccess api_mem_from_access_resolved(KitCg* g, const Operand* lv,
                                        KitCgMemAccess access, KitCgTypeId ty);
 MemAccess api_mem_from_access(KitCg* g, const Operand* lv,
                               KitCgMemAccess access);
 KitCgTypeId api_mem_access_type(KitCg* g, KitCgMemAccess access,
                                 KitCgTypeId fallback, const char* who);
 u32 api_mem_type_size_resolved(KitCg* g, KitCgTypeId ty, const char* who);
 u32 api_mem_type_size(KitCg* g, KitCgTypeId ty, const char* who);
 u32 api_require_scalar_mem_type_resolved(KitCg* g, const char* who,
                                          KitCgTypeId ty);
 void api_require_scalar_mem_type(KitCg* g, const char* who, KitCgTypeId ty);
 void api_require_pointer_value(KitCg* g, const char* who, KitCgTypeId ty);
 void api_validate_memory_value_resolved(KitCg* g, const char* who,
                                         KitCgTypeId access_ty,
                                         KitCgTypeId value_ty);
 void api_validate_memory_value(KitCg* g, const char* who, KitCgTypeId access_ty,
                                KitCgTypeId value_ty);
 int api_sv_owns_operand_local(const ApiSValue* sv, const Operand* op);
 void api_ensure_local(KitCg* g, ApiSValue* sv);
 Operand api_force_local(KitCg* g, ApiSValue* v, KitCgTypeId ty);
 Operand api_force_local_unless_imm(KitCg* g, ApiSValue* v, KitCgTypeId ty);
 void api_release(KitCg* g, ApiSValue* sv);
 
 /* -O0 transient liveness (see KitCg.local_refs). api_coalesce_on returns
  * whether the copy/dup coalescing + finer-reclaim mechanisms are enabled this
  * run. api_temp_dead reports whether transient `local` is provably dead right
  * now (no live value-stack entry references it): count==0 confirmed by a stack
  * scan. api_reseat_{begin,end} bracket an in-place change to a stack entry's
  * operand so its references are re-accounted. */
 int api_coalesce_on(KitCg* g);
 int api_temp_dead(KitCg* g, CGLocal local);
 /* Flag OP for the -O0 backend's eager dead-operand drop: when coalescing is on
  * and OP names a transient that api_temp_dead confirms is dead after the
  * consuming op, AND OP is distinct from that op's destination DST (so it is not
  * the result being written), return OP with OPK_FLAG_KILL set; otherwise return
  * OP unchanged. The NativeDirectTarget then drops OP's live cache register
  * after the op (no spill, no reload) instead of leaving it resident to be
  * flush-stored at the next barrier. Pass a non-OPK_LOCAL operand for DST (e.g.
  * the store's memory place) when the op has no local destination — the
  * self-guard is then a no-op. */
 Operand api_op_kill_if_dead(KitCg* g, Operand op, Operand dst);
 void api_reseat_begin(KitCg* g, const ApiSValue* sv);
 void api_reseat_end(KitCg* g, const ApiSValue* sv);
 
 BinOp api_map_int_binop(KitCgIntBinOp op);
 BinOp api_map_fp_binop(KitCgFpBinOp op);
 UnOp api_map_int_unop(KitCgIntUnOp op);
 CmpOp api_map_int_cmp(KitCgIntCmpOp op);
 CmpOp api_map_fp_cmp(KitCgFpCmpOp op);
 Operand api_lvalue_addr(KitCg* g, ApiSValue* v, KitCgTypeId pty);
 CGLocal api_f128_temp_local(KitCg* g, KitCgTypeId ty);
 u64 api_u64_from_target_bytes(KitCg* g, const u8* bytes);
 void api_store_f128_bytes(KitCg* g, CGLocal local, KitCgTypeId ty,
                           const u8 bytes[16]);
 void api_wide16_sext_imm_bytes(KitCg* g, i64 imm, u8 bytes[16]);
 ApiSValue api_make_wide16_int_const(KitCg* g, i64 value, KitCgTypeId ty);
 ApiSValue api_make_wide16_int_const_bits(KitCg* g, u64 lo, u64 hi,
                                          KitCgTypeId ty);
 void api_encode_binary128_from_double(KitCg* g, double value, u8 out[16]);
 ApiSValue api_make_f128_const(KitCg* g, double value, KitCgTypeId ty);
 ApiSValue api_wide16_materialize_lvalue(KitCg* g, ApiSValue* v, KitCgTypeId ty);
 CGLocal api_wide8_temp_local(KitCg* g, KitCgTypeId ty);
 ApiSValue api_make_wide8_const_bits(KitCg* g, u64 bits, KitCgTypeId ty);
 ApiSValue api_make_wide8_int_const(KitCg* g, i64 value, KitCgTypeId ty);
 Operand api_wide8_addr(KitCg* g, ApiSValue* v, KitCgTypeId ty);
 Operand api_wide8_load_lane(KitCg* g, Operand addr, i32 off);
 void api_wide8_store_lane(KitCg* g, Operand addr, i32 off, Operand val);
 Operand api_wide8_or_lanes(KitCg* g, ApiSValue* v, KitCgTypeId ty);
 KitCgSym api_runtime_helper(KitCg* g, const char* name, KitCgTypeId ret,
                             const KitCgTypeId* params, u32 nparams);
 void api_runtime_call_values(KitCg* g, const char* name, KitCgTypeId ret,
                              const KitCgTypeId* params, u32 nparams,
                              ApiSValue* args);
 
 /* The semantic-layer peephole optimizer: constant folding, the delayed
  * compare/arith forms, and const-local store-to-load forwarding. Included here,
  * after the operand and value types above, so its declarations can name
  * ApiSValue / ApiSourceLocal / Operand. */
 #include "cg/fold.h"
 
 #endif