value.c (26767B)
1 #include "cg/internal.h" 2 3 /* The cached wide-class tag (ApiBitField.wide_kind) reuses the bit-field 4 * member's trailing pad: ApiBitField stays at its original 12-byte footprint 5 * (u16 + u16 + u32 + u8 bit_signed + u8 wide_kind + u8 pad[2]). Inline language 6 * facts fit by packing the CG-owned state bytes into ApiSValue.flags. */ 7 _Static_assert(sizeof(ApiBitField) == 12, 8 "wide_kind must reuse ApiBitField pad, not grow it"); 9 /* The 64-byte delayed cmp/arith payload lives off the node behind a pointer 10 * (see ApiDelayed); the value-stack node must stay small. */ 11 _Static_assert(sizeof(ApiSValue) == 64, 12 "ApiSValue must stay exactly one 64-byte stack node"); 13 _Static_assert(offsetof(ApiSValue, flags) + sizeof(((ApiSValue*)0)->flags) == 14 sizeof(ApiSValue), 15 "ApiSValue flags should close the packed 64-byte node"); 16 17 /* The operand/value constructors run on the hottest per-operand codegen path. 18 * Each uses a designated compound literal rather than memset + field stores: it 19 * is value-identical (the unnamed fields zero-fill) but lets the compiler emit 20 * only the needed stores — and drop those it can see are immediately 21 * overwritten — instead of an out-of-line memset of the whole struct. Same 22 * idiom as the NativeLoc constructors in arch/native_target.h. */ 23 Operand api_op_imm(i64 v, KitCgTypeId ty) { 24 return (Operand){.kind = OPK_IMM, .type = ty, .v.imm = v}; 25 } 26 27 Operand api_op_local(CGLocal local, KitCgTypeId ty) { 28 return (Operand){.kind = OPK_LOCAL, .type = ty, .v.local = local}; 29 } 30 31 Operand api_op_global(ObjSymId sym, i64 addend, KitCgTypeId ty) { 32 return (Operand){.kind = OPK_GLOBAL, 33 .type = ty, 34 .v.global = {.sym = sym, .addend = addend}}; 35 } 36 37 Operand api_op_indirect(CGLocal base, i32 ofs, KitCgTypeId ty) { 38 return (Operand){.kind = OPK_INDIRECT, 39 .type = ty, 40 .v.ind = {.base = base, .index = CG_LOCAL_NONE, .ofs = ofs}}; 41 } 42 43 Operand api_op_indirect_indexed(CGLocal base, CGLocal index, u8 log2_scale, 44 i32 ofs, KitCgTypeId ty) { 45 return (Operand){ 46 .kind = OPK_INDIRECT, 47 .type = ty, 48 .v.ind = { 49 .base = base, .index = index, .log2_scale = log2_scale, .ofs = ofs}}; 50 } 51 52 u8 api_residency_for(const Operand* o) { 53 if (o->kind == OPK_LOCAL || o->kind == OPK_INDIRECT) return RES_LOCAL; 54 return RES_INHERENT; 55 } 56 57 ApiSValue api_make_sv(Operand op, KitCgTypeId ty) { 58 /* Designated compound literal instead of memset: the unnamed fields (the 59 * 64-byte delayed union, the bitfield rider, the flags) still zero-fill, but 60 * the compiler emits direct stores for the live fields and skips the 61 * out-of-line memset of the whole node on this hot per-operand path. */ 62 return (ApiSValue){ 63 .op = op, 64 .type = ty, 65 .source_local = KIT_CG_LOCAL_NONE, 66 .flags = API_SV_PACK(SV_OPERAND, api_residency_for(&op), 0, 0)}; 67 } 68 69 ApiSValue api_make_lv(Operand op, KitCgTypeId ty) { 70 ApiSValue sv = api_make_sv(op, ty); 71 api_sv_set_lvalue(&sv, 1); 72 return sv; 73 } 74 75 ApiSValue api_make_sv_with_local_ownership(Operand op, KitCgTypeId ty, 76 int owned) { 77 ApiSValue sv = api_make_sv(op, ty); 78 if (op.kind == OPK_LOCAL && !owned) api_sv_set_res(&sv, RES_FIXED_LOCAL); 79 return sv; 80 } 81 82 KitCgTypeId api_sv_type(const ApiSValue* sv) { 83 return sv->type ? sv->type : sv->op.type; 84 } 85 86 int api_operand_can_address(const Operand* o) { 87 return o->kind == OPK_LOCAL || o->kind == OPK_GLOBAL || 88 o->kind == OPK_INDIRECT; 89 } 90 91 int api_sv_op_is(const ApiSValue* sv, OpKind kind) { 92 return api_sv_kind(sv) == SV_OPERAND && sv->op.kind == kind; 93 } 94 95 int api_sv_op_is_local_or_imm(const ApiSValue* sv) { 96 return api_sv_kind(sv) == SV_OPERAND && 97 (sv->op.kind == OPK_IMM || sv->op.kind == OPK_LOCAL); 98 } 99 100 /* A PLACE (lvalue) is exactly an SV_OPERAND whose operand addresses storage: 101 * op.kind in { OPK_LOCAL, OPK_GLOBAL, OPK_INDIRECT } (api_operand_can_address). 102 * This is a kind-based predicate, not a heuristic: the `lvalue` flag marks the 103 * value as a place, SV_OPERAND guarantees `op` is meaningful (SV_CMP/SV_ARITH 104 * never carry lvalue=1 — see fold.c, which clears it on materialization), and 105 * the operand kind decides addressability. The former heuristic OR'd in 106 * `bitfield_lvalue` and a (source_local && OPK_LOCAL) term, both subsumed here: 107 * a bit-field place is an ordinary addressable place (it addresses the storage 108 * unit) that additionally carries a bit descriptor — see api_sv_is_bitfield and 109 * kit_cg_field — and source_local only co-occurs with OPK_LOCAL, so 110 * api_operand_can_address already covers both. */ 111 int api_is_lvalue_sv(const ApiSValue* sv) { 112 return api_sv_lvalue_flag(sv) && api_sv_kind(sv) == SV_OPERAND && 113 api_operand_can_address(&sv->op); 114 } 115 116 /* A bit-field PLACE is a PLACE that additionally carries bit geometry (set by 117 * kit_cg_field when it projects to a bit-field). The plain load/store detect 118 * this and perform the extract/insert; there is no separate bit-field memop. */ 119 int api_sv_is_bitfield(const ApiSValue* sv) { 120 return sv->bitfield.bit_width != 0; 121 } 122 123 /* Rebuild the full BitFieldAccess the extract/insert helper consumes from the 124 * geometry the place carries plus the storage operand it addresses. The storage 125 * MemAccess is derived from the operand + field type, the way the old 126 * bf_from_access did, so the place need only carry the bit geometry. */ 127 /* Build the bit-field MemAccess that rides the generic load/store: the storage 128 * unit ({type,size}) plus the bf_* bit geometry from the place's descriptor. 129 * The CgTarget impl reconstructs a BitFieldAccess via bf_from_mem. */ 130 MemAccess api_mem_for_bitfield(KitCg* g, const ApiSValue* sv, 131 const Operand* storage, KitCgTypeId field_ty) { 132 MemAccess m = api_mem_for_lvalue(g, storage, field_ty); 133 if (sv->bitfield.bit_storage_size) m.size = sv->bitfield.bit_storage_size; 134 m.bf_offset = sv->bitfield.bit_offset; 135 m.bf_width = sv->bitfield.bit_width; 136 m.bf_signed = sv->bitfield.bit_signed ? 1 : 0; 137 return m; 138 } 139 140 void api_stack_grow(KitCg* g, u32 want) { 141 Heap* h = g->c->ctx->heap; 142 u32 cap = g->cap; 143 ApiSValue* nb; 144 ApiConstValue* nc; 145 if (cap >= want) return; 146 while (cap < want) cap = cap ? cap * 2u : API_CG_STACK_INITIAL; 147 nb = (ApiSValue*)h->alloc(h, sizeof(ApiSValue) * cap, _Alignof(ApiSValue)); 148 nc = (ApiConstValue*)h->alloc(h, sizeof(ApiConstValue) * cap, 149 _Alignof(ApiConstValue)); 150 if (g->stack) { 151 memcpy(nb, g->stack, sizeof(ApiSValue) * g->sp); 152 h->free(h, g->stack, sizeof(ApiSValue) * g->cap); 153 } 154 if (g->const_stack) { 155 memcpy(nc, g->const_stack, sizeof(ApiConstValue) * g->sp); 156 h->free(h, g->const_stack, sizeof(ApiConstValue) * g->cap); 157 } 158 g->stack = nb; 159 g->const_stack = nc; 160 g->cap = cap; 161 } 162 163 /* -O0 transient liveness bookkeeping; defined below, used by push/pop/alloc. */ 164 static void api_sv_adjust_refs(KitCg* g, const ApiSValue* sv, int delta); 165 static void api_track_temp(KitCg* g, CGLocal local); 166 167 void api_push(KitCg* g, ApiSValue v) { 168 KitCgTypeId ty = api_sv_type(&v); 169 /* One memoized fetch gives both the aggregate-place property and the 170 * wide/soft-float dispatch class — both pure functions of the type, computed 171 * once per type id rather than re-derived (cg_type_get + alias-chase + ABI 172 * round trips) on every push. */ 173 u8 cls = ty ? api_type_class(g->c, ty) : 0; 174 /* An aggregate (record) can only ever be a PLACE: it is addressed, loaded, 175 * and passed by SRET/BYVAL/BYREF, never materialized as a scalar VALUE. Catch 176 * any aggregate VALUE at the point it would enter the stack. i128/f128 are 177 * scalars (not aggregates), so they remain valid VALUEs and are unaffected. 178 */ 179 CG_REQUIRE(g, !(cls & API_TYPE_CLASS_AGGREGATE) || api_is_lvalue_sv(&v), 180 "KitCg: aggregate must be a place, not a value; load the " 181 "place or pass it by reference"); 182 /* Cache the wide/soft-float dispatch class on the node; the wide and 183 * soft-float predicates in arith.c read it back instead of re-deriving it per 184 * operand. dup/swap/rot copy the whole node (type and tag together), so it 185 * stays valid for every later reader. */ 186 v.bitfield.wide_kind = cls & API_TYPE_CLASS_WIDE_MASK; 187 api_stack_grow(g, g->sp + 1); 188 g->stack[g->sp] = v; 189 g->const_stack[g->sp] = api_const_from_sv(g, &v); 190 g->sp++; 191 if (!api_unevaluated(g)) api_sv_adjust_refs(g, &v, +1); 192 } 193 194 ApiSValue api_pop(KitCg* g) { 195 ApiSValue r; 196 CG_REQUIRE(g, g->sp != 0, "KitCg: stack underflow"); 197 r = g->stack[--g->sp]; 198 if (!api_unevaluated(g)) api_sv_adjust_refs(g, &r, -1); 199 return r; 200 } 201 202 /* ---- local helpers ---- */ 203 204 CGLocal api_local_of_sv(const ApiSValue* sv) { 205 if (api_sv_kind(sv) == SV_ARITH || api_sv_kind(sv) == SV_CMP) 206 return (CGLocal)CG_LOCAL_NONE; 207 if (sv->op.kind == OPK_LOCAL) return sv->op.v.local; 208 if (sv->op.kind == OPK_INDIRECT) return sv->op.v.ind.base; 209 return (CGLocal)CG_LOCAL_NONE; 210 } 211 212 void api_set_owned_local(ApiSValue* sv, CGLocal r) { 213 if (sv->op.kind == OPK_LOCAL) 214 sv->op.v.local = r; 215 else if (sv->op.kind == OPK_INDIRECT) 216 sv->op.v.ind.base = r; 217 } 218 219 KitCgTypeId api_owned_local_type(KitCg* g, const ApiSValue* sv) { 220 if (sv->op.kind == OPK_INDIRECT) { 221 KitCgTypeId base = sv->type ? sv->type : builtin_id(KIT_CG_BUILTIN_VOID); 222 return cg_type_ptr_to(g->c, base); 223 } 224 return api_sv_type(sv); 225 } 226 227 /* ---- temporary local allocation ---- */ 228 229 CGLocal api_alloc_temp_local(KitCg* g, KitCgTypeId ty) { 230 CGLocalDesc d; 231 CGLocal local; 232 if (api_unevaluated(g)) return CG_LOCAL_NONE; 233 memset(&d, 0, sizeof d); 234 d.type = ty; 235 /* A compiler temporary: its home is reclaimable once the value stack drops it 236 * at a statement boundary (kit_cg_reclaim_temps), so the single-pass frame is 237 * bounded by max-simultaneous-live temps rather than a per-subexpression sum. 238 */ 239 d.flags |= CG_LOCAL_TRANSIENT; 240 if (ty) { 241 ABITypeInfo ti = abi_cg_type_info(g->c->abi, ty); 242 d.size = ti.size; 243 d.align = ti.align; 244 } 245 /* A split-lane 8-byte scalar temp must live in memory so its two words are 246 * addressable for lane ops and the multi-part ABI path; the allocator gives 247 * an unflagged scalar one value slot, which would truncate it. (wide16 temps 248 * are already forced via the size>word auto-home in cg_ir_lower.) */ 249 if (ty && api_is_wide8_scalar_type(g->c, ty)) 250 d.flags |= CG_LOCAL_MEMORY_REQUIRED; 251 local = g->target->local(g->target, &d); 252 if (local == CG_LOCAL_NONE) { 253 compiler_panic(g->c, g->cur_loc, 254 "KitCg: target failed to allocate temporary local"); 255 } 256 api_track_temp(g, local); /* register for -O0 transient liveness */ 257 return local; 258 } 259 260 MemAccess api_mem_for_lvalue(KitCg* g, const Operand* lv, KitCgTypeId ty) { 261 MemAccess m; 262 memset(&m, 0, sizeof m); 263 m.type = ty; 264 if (ty) { 265 ABITypeInfo ti = abi_cg_type_info(g->c->abi, ty); 266 m.size = ti.size; 267 m.align = ti.align; 268 } else { 269 m.size = 0; 270 m.align = 0; 271 } 272 m.flags = MF_NONE; 273 if (lv->kind == OPK_LOCAL) { 274 m.alias.kind = (u8)ALIAS_LOCAL; 275 m.alias.v.local_id = (i32)lv->v.local; 276 } else if (lv->kind == OPK_GLOBAL) { 277 m.alias.kind = (u8)ALIAS_GLOBAL; 278 } else { 279 m.alias.kind = (u8)ALIAS_UNKNOWN; 280 } 281 return m; 282 } 283 284 MemAccess api_mem_from_access_resolved(KitCg* g, const Operand* lv, 285 KitCgMemAccess access, KitCgTypeId ty) { 286 MemAccess m = api_mem_for_lvalue(g, lv, ty); 287 u32 natural_align = m.align; 288 if (access.align) { 289 m.align = access.align; 290 if (ty && access.align < natural_align) m.flags |= MF_UNALIGNED; 291 } 292 m.addr_space = (u16)access.address_space; 293 if (access.flags & KIT_CG_MEM_VOLATILE) m.flags |= MF_VOLATILE; 294 if (access.flags & KIT_CG_MEM_SEXT_LOAD) m.flags |= MF_SEXT_LOAD; 295 return m; 296 } 297 298 MemAccess api_mem_from_access(KitCg* g, const Operand* lv, 299 KitCgMemAccess access) { 300 return api_mem_from_access_resolved(g, lv, access, 301 resolve_type(g->c, access.type)); 302 } 303 304 KitCgTypeId api_mem_access_type(KitCg* g, KitCgMemAccess access, 305 KitCgTypeId fallback, const char* who) { 306 KitCgTypeId ty = resolve_type(g->c, access.type); 307 if (!ty) ty = resolve_type(g->c, fallback); 308 CG_REQUIRE(g, ty, "KitCg: %.*s has no value type", 309 SLICE_ARG(slice_from_cstr(who))); 310 return ty; 311 } 312 313 u32 api_mem_type_size_resolved(KitCg* g, KitCgTypeId ty, const char* who) { 314 CG_REQUIRE(g, ty, "KitCg: %.*s has invalid type", 315 SLICE_ARG(slice_from_cstr(who))); 316 CG_REQUIRE(g, !cg_type_is_void(g->c, ty), "KitCg: %.*s uses void type", 317 SLICE_ARG(slice_from_cstr(who))); 318 return abi_cg_sizeof(g->c->abi, ty); 319 } 320 321 u32 api_mem_type_size(KitCg* g, KitCgTypeId ty, const char* who) { 322 return api_mem_type_size_resolved(g, resolve_type(g->c, ty), who); 323 } 324 325 u32 api_require_scalar_mem_type_resolved(KitCg* g, const char* who, 326 KitCgTypeId ty) { 327 u8 bits = api_type_pred_bits(g->c, ty); 328 CG_REQUIRE(g, !(bits & API_PRED_AGGREGATE), 329 "KitCg: %.*s cannot use aggregate value type (size %u); " 330 "copy fields or use byte memory operations", 331 SLICE_ARG(slice_from_cstr(who)), 332 (unsigned)api_mem_type_size_resolved(g, ty, who)); 333 return api_mem_type_size_resolved(g, ty, who); 334 } 335 336 void api_require_scalar_mem_type(KitCg* g, const char* who, KitCgTypeId ty) { 337 (void)api_require_scalar_mem_type_resolved(g, who, resolve_type(g->c, ty)); 338 } 339 340 void api_require_pointer_value(KitCg* g, const char* who, KitCgTypeId ty) { 341 CG_REQUIRE(g, cg_type_pointee(g->c, ty), 342 "KitCg: %.*s operand must be a pointer", 343 SLICE_ARG(slice_from_cstr(who))); 344 } 345 346 void api_validate_memory_value_resolved(KitCg* g, const char* who, 347 KitCgTypeId access_ty, 348 KitCgTypeId value_ty) { 349 u32 access_size; 350 u32 value_size; 351 u8 access_bits; 352 u8 value_bits; 353 access_size = api_require_scalar_mem_type_resolved(g, who, access_ty); 354 CG_REQUIRE(g, value_ty, "KitCg: %.*s value has no type", 355 SLICE_ARG(slice_from_cstr(who))); 356 access_bits = api_type_pred_bits(g->c, access_ty); 357 value_bits = api_type_pred_bits(g->c, value_ty); 358 CG_REQUIRE(g, !(value_bits & API_PRED_AGGREGATE), 359 "KitCg: %.*s value is aggregate (size %u); copy fields or " 360 "use byte memory operations", 361 SLICE_ARG(slice_from_cstr(who)), 362 (unsigned)api_mem_type_size_resolved(g, value_ty, who)); 363 value_size = api_mem_type_size_resolved(g, value_ty, who); 364 CG_REQUIRE(g, 365 !(access_size != value_size || 366 ((access_bits ^ value_bits) & API_PRED_FLOAT)), 367 "KitCg: %.*s value type/size mismatch: access size %u, " 368 "value size %u", 369 SLICE_ARG(slice_from_cstr(who)), (unsigned)access_size, 370 (unsigned)value_size); 371 } 372 373 void api_validate_memory_value(KitCg* g, const char* who, KitCgTypeId access_ty, 374 KitCgTypeId value_ty) { 375 api_validate_memory_value_resolved(g, who, resolve_type(g->c, access_ty), 376 resolve_type(g->c, value_ty)); 377 } 378 379 int api_sv_owns_operand_local(const ApiSValue* sv, const Operand* op) { 380 return api_sv_res(sv) == RES_LOCAL && op->kind == OPK_LOCAL && 381 sv->op.kind == OPK_LOCAL && sv->op.v.local == op->v.local; 382 } 383 384 void api_ensure_local(KitCg* g, ApiSValue* sv) { 385 if (api_unevaluated(g)) return; 386 if (api_sv_kind(sv) == SV_CMP) { 387 KitCgTypeId ty = api_sv_type(sv); 388 Operand dst; 389 if (sv->delayed->cmp.a_owned && sv->delayed->cmp.a.kind == OPK_LOCAL && 390 sv->delayed->cmp.a.type == ty) { 391 dst = api_op_local(sv->delayed->cmp.a.v.local, ty); 392 } else if (sv->delayed->cmp.b_owned && 393 sv->delayed->cmp.b.kind == OPK_LOCAL && 394 sv->delayed->cmp.b.type == ty) { 395 dst = api_op_local(sv->delayed->cmp.b.v.local, ty); 396 } else { 397 CGLocal r = 398 api_alloc_temp_local(g, ty ? ty : builtin_id(KIT_CG_BUILTIN_I32)); 399 dst = api_op_local(r, ty); 400 } 401 api_materialize_cmp_to(g, sv, dst); 402 return; 403 } 404 if (api_sv_kind(sv) == SV_ARITH) { 405 KitCgTypeId ty = api_sv_type(sv); 406 Operand dst; 407 if (sv->delayed->arith.a_owned && sv->delayed->arith.a.kind == OPK_LOCAL && 408 sv->delayed->arith.a.type == ty) { 409 dst = api_op_local(sv->delayed->arith.a.v.local, ty); 410 } else if (api_arith_rhs_reusable(sv) && sv->delayed->arith.b_owned && 411 sv->delayed->arith.b.kind == OPK_LOCAL && 412 sv->delayed->arith.b.type == ty) { 413 dst = api_op_local(sv->delayed->arith.b.v.local, ty); 414 } else { 415 CGLocal r = 416 api_alloc_temp_local(g, ty ? ty : builtin_id(KIT_CG_BUILTIN_I32)); 417 dst = api_op_local(r, ty); 418 } 419 api_materialize_arith_to(g, sv, dst); 420 return; 421 } 422 return; 423 } 424 425 Operand api_force_local(KitCg* g, ApiSValue* v, KitCgTypeId ty) { 426 CgTarget* T = g->target; 427 ty = resolve_type(g->c, ty); 428 if (!ty) return api_op_imm(0, KIT_CG_TYPE_NONE); 429 if (api_unevaluated(g)) return api_op_imm(0, ty); 430 api_ensure_local(g, v); 431 if (v->op.kind == OPK_LOCAL && !api_is_lvalue_sv(v)) { 432 return v->op; 433 } 434 CGLocal r = api_alloc_temp_local(g, ty); 435 Operand dst = api_op_local(r, ty); 436 if (v->op.kind == OPK_IMM) { 437 T->load_imm(T, dst, v->op.v.imm); 438 } else if (api_is_lvalue_sv(v)) { 439 T->load(T, dst, v->op, api_mem_for_lvalue(g, &v->op, ty)); 440 } else if (v->op.kind == OPK_GLOBAL) { 441 T->addr_of(T, dst, v->op); 442 } else { 443 CG_BUG(g, "KitCg: cannot force operand to local"); 444 } 445 v->op = dst; 446 api_sv_set_res(v, RES_LOCAL); 447 return dst; 448 } 449 450 Operand api_force_local_unless_imm(KitCg* g, ApiSValue* v, KitCgTypeId ty) { 451 if (api_sv_op_is(v, OPK_IMM)) return v->op; 452 return api_force_local(g, v, ty); 453 } 454 455 void api_release(KitCg* g, ApiSValue* sv) { 456 if (api_sv_kind(sv) == SV_CMP) { 457 api_release_cmp(g, sv); 458 } else if (api_sv_kind(sv) == SV_ARITH) { 459 api_release_arith(g, sv); 460 } 461 api_sv_set_res(sv, RES_INHERENT); 462 } 463 464 /* ---- -O0 transient liveness (refcount + confirming scan) ---- */ 465 466 int api_coalesce_on(KitCg* g) { 467 if (!g->coalesce_known) { 468 /* Default on; KIT_NO_COALESCE=1 disables for A/B measurement. */ 469 g->coalesce = kit_debug_getenv("KIT_NO_COALESCE") ? 0u : 1u; 470 g->coalesce_known = 1u; 471 } 472 return g->coalesce; 473 } 474 475 /* Mark `local` (a fresh api_alloc_temp_local handle) as this function's temp 476 * and zero its live-reference count. */ 477 static void api_track_temp(KitCg* g, CGLocal local) { 478 Heap* h = g->c->ctx->heap; 479 u32 idx = (u32)local; 480 if (idx >= g->local_track_cap) { 481 u32 cap = g->local_track_cap ? g->local_track_cap : 64u; 482 u32* nr; 483 u32* ng; 484 while (cap <= idx) cap *= 2u; 485 nr = (u32*)h->alloc(h, sizeof(u32) * cap, _Alignof(u32)); 486 ng = (u32*)h->alloc(h, sizeof(u32) * cap, _Alignof(u32)); 487 memset(nr, 0, sizeof(u32) * cap); 488 memset(ng, 0, sizeof(u32) * cap); 489 if (g->local_refs) { 490 memcpy(nr, g->local_refs, sizeof(u32) * g->local_track_cap); 491 memcpy(ng, g->local_temp_gen, sizeof(u32) * g->local_track_cap); 492 h->free(h, g->local_refs, sizeof(u32) * g->local_track_cap); 493 h->free(h, g->local_temp_gen, sizeof(u32) * g->local_track_cap); 494 } 495 g->local_refs = nr; 496 g->local_temp_gen = ng; 497 g->local_track_cap = cap; 498 } 499 g->local_temp_gen[idx] = g->func_gen; 500 g->local_refs[idx] = 0u; 501 } 502 503 static int api_handle_is_temp(KitCg* g, CGLocal local) { 504 u32 idx = (u32)local; 505 return idx < g->local_track_cap && g->local_temp_gen[idx] == g->func_gen; 506 } 507 508 /* Visit each temp local an operand references (LOCAL, or INDIRECT base/index), 509 * adjusting its live-reference count by `delta` (clamped at 0 on decrement). */ 510 static void api_op_adjust_refs(KitCg* g, const Operand* op, int delta) { 511 CGLocal a = CG_LOCAL_NONE, b = CG_LOCAL_NONE; 512 if (op->kind == OPK_LOCAL) { 513 a = op->v.local; 514 } else if (op->kind == OPK_INDIRECT) { 515 a = op->v.ind.base; 516 b = op->v.ind.index; 517 } 518 if (a != CG_LOCAL_NONE && api_handle_is_temp(g, a)) { 519 if (delta > 0) 520 g->local_refs[a]++; 521 else if (g->local_refs[a]) 522 g->local_refs[a]--; 523 } 524 if (b != CG_LOCAL_NONE && api_handle_is_temp(g, b)) { 525 if (delta > 0) 526 g->local_refs[b]++; 527 else if (g->local_refs[b]) 528 g->local_refs[b]--; 529 } 530 } 531 532 /* All temp locals an SValue references: its operand plus, for a delayed 533 * cmp/arith, its pending operands. */ 534 static void api_sv_adjust_refs(KitCg* g, const ApiSValue* sv, int delta) { 535 if (api_sv_kind(sv) == SV_OPERAND) { 536 api_op_adjust_refs(g, &sv->op, delta); 537 } else if (sv->delayed) { 538 if (api_sv_kind(sv) == SV_CMP) { 539 api_op_adjust_refs(g, &sv->delayed->cmp.a, delta); 540 api_op_adjust_refs(g, &sv->delayed->cmp.b, delta); 541 } else if (api_sv_kind(sv) == SV_ARITH) { 542 api_op_adjust_refs(g, &sv->delayed->arith.a, delta); 543 if (sv->delayed->arith.kind == API_DELAYED_BINOP) 544 api_op_adjust_refs(g, &sv->delayed->arith.b, delta); 545 } 546 } 547 } 548 549 void api_reseat_begin(KitCg* g, const ApiSValue* sv) { 550 if (sv >= g->stack && sv < g->stack + g->sp) api_sv_adjust_refs(g, sv, -1); 551 } 552 void api_reseat_end(KitCg* g, const ApiSValue* sv) { 553 if (sv >= g->stack && sv < g->stack + g->sp) api_sv_adjust_refs(g, sv, +1); 554 } 555 556 /* Ground-truth scan: how many live value-stack entries reference temp `local`. 557 * The arbiter when the incremental count reads 0 (so a reseat-accounting gap 558 * can only cost a missed optimization, never a miscompile). */ 559 static u32 api_temp_scan_refs(KitCg* g, CGLocal local) { 560 u32 n = 0; 561 for (u32 i = 0; i < g->sp; ++i) { 562 const ApiSValue* sv = &g->stack[i]; 563 const Operand* op = &sv->op; 564 if (api_sv_kind(sv) == SV_OPERAND) { 565 if (op->kind == OPK_LOCAL && op->v.local == local) 566 n++; 567 else if (op->kind == OPK_INDIRECT && 568 (op->v.ind.base == local || op->v.ind.index == local)) 569 n++; 570 } else if (sv->delayed) { 571 const ApiDelayed* d = sv->delayed; 572 const Operand* a = api_sv_kind(sv) == SV_CMP ? &d->cmp.a : &d->arith.a; 573 const Operand* b = api_sv_kind(sv) == SV_CMP ? &d->cmp.b : &d->arith.b; 574 int has_b = 575 api_sv_kind(sv) == SV_CMP || d->arith.kind == API_DELAYED_BINOP; 576 if ((a->kind == OPK_LOCAL && a->v.local == local) || 577 (a->kind == OPK_INDIRECT && 578 (a->v.ind.base == local || a->v.ind.index == local))) 579 n++; 580 if (has_b && ((b->kind == OPK_LOCAL && b->v.local == local) || 581 (b->kind == OPK_INDIRECT && 582 (b->v.ind.base == local || b->v.ind.index == local)))) 583 n++; 584 } 585 } 586 return n; 587 } 588 589 int api_temp_dead(KitCg* g, CGLocal local) { 590 if (local == CG_LOCAL_NONE || !api_handle_is_temp(g, local)) return 0; 591 if (g->local_refs[(u32)local] > 0) 592 return 0; /* fast reject: still referenced */ 593 return api_temp_scan_refs(g, local) == 0; /* confirm before acting */ 594 } 595 596 Operand api_op_kill_if_dead(KitCg* g, Operand op, Operand dst) { 597 if (api_coalesce_on(g) && op.kind == OPK_LOCAL && 598 (dst.kind != OPK_LOCAL || op.v.local != dst.v.local) && 599 api_temp_dead(g, op.v.local)) 600 op.flags |= OPK_FLAG_KILL; 601 return op; 602 } 603 604 /* ---- BinOp / UnOp / CmpOp mapping ---- */ 605 606 BinOp api_map_int_binop(KitCgIntBinOp op) { 607 switch (op) { 608 case KIT_CG_INT_ADD: 609 return BO_IADD; 610 case KIT_CG_INT_SUB: 611 return BO_ISUB; 612 case KIT_CG_INT_MUL: 613 return BO_IMUL; 614 case KIT_CG_INT_SDIV: 615 return BO_SDIV; 616 case KIT_CG_INT_UDIV: 617 return BO_UDIV; 618 case KIT_CG_INT_SREM: 619 return BO_SREM; 620 case KIT_CG_INT_UREM: 621 return BO_UREM; 622 case KIT_CG_INT_AND: 623 return BO_AND; 624 case KIT_CG_INT_OR: 625 return BO_OR; 626 case KIT_CG_INT_XOR: 627 return BO_XOR; 628 case KIT_CG_INT_SHL: 629 return BO_SHL; 630 case KIT_CG_INT_LSHR: 631 return BO_SHR_U; 632 case KIT_CG_INT_ASHR: 633 return BO_SHR_S; 634 } 635 return BO_IADD; 636 } 637 638 BinOp api_map_fp_binop(KitCgFpBinOp op) { 639 switch (op) { 640 case KIT_CG_FP_ADD: 641 return BO_FADD; 642 case KIT_CG_FP_SUB: 643 return BO_FSUB; 644 case KIT_CG_FP_MUL: 645 return BO_FMUL; 646 case KIT_CG_FP_DIV: 647 return BO_FDIV; 648 } 649 return BO_FADD; 650 } 651 652 UnOp api_map_int_unop(KitCgIntUnOp op) { 653 switch (op) { 654 case KIT_CG_INT_NEG: 655 return UO_NEG; 656 case KIT_CG_INT_NOT: 657 return UO_NOT; 658 case KIT_CG_INT_BNOT: 659 return UO_BNOT; 660 } 661 return UO_NEG; 662 } 663 664 CmpOp api_map_int_cmp(KitCgIntCmpOp op) { 665 switch (op) { 666 case KIT_CG_INT_EQ: 667 return CMP_EQ; 668 case KIT_CG_INT_NE: 669 return CMP_NE; 670 case KIT_CG_INT_LT_S: 671 return CMP_LT_S; 672 case KIT_CG_INT_LE_S: 673 return CMP_LE_S; 674 case KIT_CG_INT_GT_S: 675 return CMP_GT_S; 676 case KIT_CG_INT_GE_S: 677 return CMP_GE_S; 678 case KIT_CG_INT_LT_U: 679 return CMP_LT_U; 680 case KIT_CG_INT_LE_U: 681 return CMP_LE_U; 682 case KIT_CG_INT_GT_U: 683 return CMP_GT_U; 684 case KIT_CG_INT_GE_U: 685 return CMP_GE_U; 686 } 687 return CMP_EQ; 688 } 689 690 /* 1:1: the internal FP block mirrors KitCgFpCmpOp's order, so the public 691 * ordered/unordered distinction is preserved all the way to the backends. */ 692 CmpOp api_map_fp_cmp(KitCgFpCmpOp op) { 693 switch (op) { 694 case KIT_CG_FP_OEQ: 695 return CMP_OEQ_F; 696 case KIT_CG_FP_ONE: 697 return CMP_ONE_F; 698 case KIT_CG_FP_OLT: 699 return CMP_OLT_F; 700 case KIT_CG_FP_OLE: 701 return CMP_OLE_F; 702 case KIT_CG_FP_OGT: 703 return CMP_OGT_F; 704 case KIT_CG_FP_OGE: 705 return CMP_OGE_F; 706 case KIT_CG_FP_UEQ: 707 return CMP_UEQ_F; 708 case KIT_CG_FP_UNE: 709 return CMP_UNE_F; 710 case KIT_CG_FP_ULT: 711 return CMP_ULT_F; 712 case KIT_CG_FP_ULE: 713 return CMP_ULE_F; 714 case KIT_CG_FP_UGT: 715 return CMP_UGT_F; 716 case KIT_CG_FP_UGE: 717 return CMP_UGE_F; 718 } 719 return CMP_OEQ_F; 720 } 721 722 Operand api_lvalue_addr(KitCg* g, ApiSValue* v, KitCgTypeId pty) { 723 CgTarget* T; 724 ApiSourceLocal* rec; 725 CGLocal r; 726 Operand dst; 727 if (api_unevaluated(g)) return api_op_imm(0, pty); 728 api_local_const_address_taken(g, v->source_local); 729 api_ensure_local(g, v); 730 CG_REQUIRE(g, api_is_lvalue_sv(v), "KitCg: addr operand is not an lvalue"); 731 T = g->target; 732 r = api_alloc_temp_local(g, pty); 733 dst = api_op_local(r, pty); 734 rec = v->source_local != KIT_CG_LOCAL_NONE 735 ? api_local_from_handle(g, v->source_local) 736 : NULL; 737 if (rec && rec->storage != CG_LOCAL_NONE && T->local_addr) 738 T->local_addr(T, dst, &rec->desc, rec->storage); 739 else 740 T->addr_of(T, dst, v->op); 741 return dst; 742 } 743 744 /* ---- C-symbol mangling ---- */