fold.c (23738B)
1 /* The semantic-layer peephole optimizer. See cg/fold.h for the contract. 2 * 3 * This is the isolated `-O0` peephole that was previously interleaved with the 4 * value-stack discipline in value.c (Track 6.2). Nothing here emits control 5 * flow or owns the stack; it folds constants, manages the delayed compare/arith 6 * forms, and tracks const-valued locals. The op families drive it through the 7 * fold.h entry points. */ 8 9 #include "cg/internal.h" 10 #include "cg/ir_eval.h" 11 12 /* ============================================================ 13 * 0. Delayed-payload pool (off-node SV_CMP / SV_ARITH state) 14 * 15 * A delayed value's cmp/arith payload lives here, behind the ApiSValue.delayed 16 * pointer, rather than inline on the value-stack node — which keeps the node 17 * small and its per-push construction free of delayed state. Payloads are 18 * bump-allocated from a per-function arena and recycled through an intrusive 19 * freelist (api_delayed_free), so within a function the arena only grows to the 20 * live-delayed-value working set. api_delayed_reset drops both arena and 21 * freelist at a function boundary (where the value stack is also dropped), so a 22 * payload never outlives its function and a missed free only enlarges the 23 * reset-per-function arena — it cannot dangle or corrupt. 24 * ============================================================ */ 25 26 ApiDelayed* api_delayed_alloc(KitCg* g) { 27 ApiDelayed* d = g->delayed_free; 28 if (d) { 29 g->delayed_free = d->next_free; 30 return d; 31 } 32 if (!g->delayed_arena_init) { 33 /* Default block size + block retention (arena_reset keeps the high-water 34 * capacity): the per-statement reset reuses one block instead of churning 35 * tiny 4 KiB blocks through the heap. */ 36 arena_init(&g->delayed_arena, (Heap*)g->c->ctx->heap, 0); 37 g->delayed_arena_init = 1; 38 } 39 return arena_new(&g->delayed_arena, ApiDelayed); 40 } 41 42 void api_delayed_free(KitCg* g, ApiDelayed* d) { 43 if (!d) return; 44 d->next_free = g->delayed_free; 45 g->delayed_free = d; 46 } 47 48 void api_delayed_reset(KitCg* g) { 49 if (g->delayed_arena_init) arena_reset(&g->delayed_arena); 50 g->delayed_free = NULL; 51 } 52 53 /* ============================================================ 54 * 1. Integer constant folding 55 * ============================================================ */ 56 57 u32 api_int_like_width(Compiler* c, KitCgTypeId id) { 58 const CgType* ty = cg_type_get(c, id); 59 if (!ty) return 0; 60 if (ty->kind == KIT_CG_TYPE_INT || ty->kind == KIT_CG_TYPE_BOOL) 61 return ty->integer.width; 62 if (ty->kind == KIT_CG_TYPE_ENUM) return (u32)(ty->size * 8u); 63 if (ty->kind == KIT_CG_TYPE_PTR) return (u32)(ty->size * 8u); 64 return 0; 65 } 66 67 int api_type_is_bool(Compiler* c, KitCgTypeId id) { 68 const CgType* ty = cg_type_get(c, id); 69 if (!ty) return 0; 70 return ty->kind == KIT_CG_TYPE_BOOL; 71 } 72 73 /* The width-arithmetic core now lives in cg/ir_eval.{c,h} (shared with opt). 74 * The fold.h surface (still consumed by arith.c / control.c) stays as thin 75 * delegators. */ 76 u64 api_width_mask(u32 width) { return kit_ir_width_mask(width); } 77 78 u64 api_mask_width(u64 v, u32 width) { return kit_ir_mask_width(v, width); } 79 80 i64 api_sign_extend_width(u64 v, u32 width) { 81 return kit_ir_sign_extend_width(v, width); 82 } 83 84 int api_foldable_int_like_type(Compiler* c, KitCgTypeId ty, u32* width_out) { 85 u32 width = api_int_like_width(c, ty); 86 if (!width || width > 64) return 0; 87 *width_out = width; 88 return 1; 89 } 90 91 int api_foldable_int_type(Compiler* c, KitCgTypeId ty, u32* width_out) { 92 if (!cg_type_is_int(c, ty)) return 0; 93 return api_foldable_int_like_type(c, ty, width_out); 94 } 95 96 i64 api_fold_result(Compiler* c, KitCgTypeId ty, u64 v, u32 width) { 97 v = api_mask_width(v, width); 98 if (api_type_is_bool(c, ty)) v = v != 0; 99 return (i64)v; 100 } 101 102 int api_try_fold_int_binop(KitCg* g, BinOp op, KitCgTypeId ty, i64 a, i64 b, 103 i64* out) { 104 u32 width; 105 i64 r; 106 /* Strict-int / PTR-aware foldability is fold's own type policy; the wrapping 107 * arithmetic is the shared core. fold_result re-masks + bool-coerces. */ 108 if (!g || !out || !api_foldable_int_type(g->c, ty, &width)) return 0; 109 if (!kit_ir_eval_binop(op, width, a, b, &r)) return 0; 110 *out = api_fold_result(g->c, ty, (u64)r, width); 111 return 1; 112 } 113 114 int api_try_fold_int_unop(KitCg* g, UnOp op, KitCgTypeId ty, i64 a, i64* out) { 115 u32 width; 116 i64 r; 117 if (!g || !out || !api_foldable_int_type(g->c, ty, &width)) return 0; 118 if (!kit_ir_eval_unop(op, width, a, &r)) return 0; 119 *out = api_fold_result(g->c, ty, (u64)r, width); 120 return 1; 121 } 122 123 int api_try_fold_int_cmp(KitCg* g, CmpOp op, KitCgTypeId ty, i64 a, i64 b, 124 i64* out) { 125 u32 width; 126 /* fold's cmp policy is int-LIKE (bool/enum/ptr admitted); the predicate eval 127 * is shared. */ 128 if (!g || !out || !api_foldable_int_like_type(g->c, ty, &width)) return 0; 129 return kit_ir_eval_cmp(op, width, a, b, out); 130 } 131 132 /* ============================================================ 133 * 2a. Delayed compare (SV_CMP) lifecycle 134 * ============================================================ */ 135 136 ApiSValue api_make_cmp(KitCg* g, CmpOp op, Operand a, Operand b, 137 KitCgTypeId result_ty, int a_owned, int b_owned) { 138 ApiDelayed* d = api_delayed_alloc(g); 139 d->cmp = (ApiDelayedCmp){.op = op, 140 .a = a, 141 .b = b, 142 .a_owned = a_owned ? 1u : 0u, 143 .b_owned = b_owned ? 1u : 0u}; 144 return (ApiSValue){.type = result_ty, 145 .source_local = KIT_CG_LOCAL_NONE, 146 .delayed = d, 147 .flags = API_SV_PACK(SV_CMP, RES_INHERENT, 0, 0)}; 148 } 149 150 CmpOp api_invert_cmp(CmpOp op) { 151 switch (op) { 152 case CMP_EQ: 153 return CMP_NE; 154 case CMP_NE: 155 return CMP_EQ; 156 case CMP_LT_S: 157 return CMP_GE_S; 158 case CMP_LE_S: 159 return CMP_GT_S; 160 case CMP_GT_S: 161 return CMP_LE_S; 162 case CMP_GE_S: 163 return CMP_LT_S; 164 case CMP_LT_U: 165 return CMP_GE_U; 166 case CMP_LE_U: 167 return CMP_GT_U; 168 case CMP_GT_U: 169 return CMP_LE_U; 170 case CMP_GE_U: 171 return CMP_LT_U; 172 /* FP: the negation of a compare must flip ordered<->unordered (the NaN 173 * outcome flips too) as well as negate the relation. The correct inverse 174 * of ordered `a<b` is *unordered* `a>=b`, not ordered `a>=b`. */ 175 case CMP_OEQ_F: 176 return CMP_UNE_F; 177 case CMP_ONE_F: 178 return CMP_UEQ_F; 179 case CMP_OLT_F: 180 return CMP_UGE_F; 181 case CMP_OLE_F: 182 return CMP_UGT_F; 183 case CMP_OGT_F: 184 return CMP_ULE_F; 185 case CMP_OGE_F: 186 return CMP_ULT_F; 187 case CMP_UEQ_F: 188 return CMP_ONE_F; 189 case CMP_UNE_F: 190 return CMP_OEQ_F; 191 case CMP_ULT_F: 192 return CMP_OGE_F; 193 case CMP_ULE_F: 194 return CMP_OGT_F; 195 case CMP_UGT_F: 196 return CMP_OLE_F; 197 case CMP_UGE_F: 198 return CMP_OLT_F; 199 } 200 return CMP_EQ; 201 } 202 203 void api_release_cmp(KitCg* g, ApiSValue* sv) { 204 api_delayed_free(g, sv->delayed); 205 sv->delayed = NULL; 206 api_sv_set_kind(sv, SV_OPERAND); 207 } 208 209 void api_materialize_cmp_to(KitCg* g, ApiSValue* sv, Operand dst) { 210 ApiDelayed* d = sv->delayed; 211 /* Flag dead-transient operands so the -O0 backend drops them after the 212 * compare instead of spilling them at the next barrier (eager dead-operand 213 * drop). */ 214 Operand a = api_op_kill_if_dead(g, d->cmp.a, dst); 215 Operand b = api_op_kill_if_dead(g, d->cmp.b, dst); 216 g->target->cmp(g->target, d->cmp.op, dst, a, b); 217 api_delayed_free(g, d); 218 sv->delayed = NULL; 219 api_sv_set_kind(sv, SV_OPERAND); 220 sv->op = dst; 221 sv->type = dst.type; 222 api_sv_set_res(sv, RES_LOCAL); 223 api_sv_set_lvalue(sv, 0); 224 } 225 226 /* ============================================================ 227 * 2b. Delayed arith (SV_ARITH) lifecycle 228 * 229 * Live: api_can_delay_int_arith admits a non-flagged foldable integer op, so an 230 * unflagged int binop/unop is held un-emitted as an SV_ARITH. A following op 231 * can then fuse it (fold an imm chain, collapse an identity) and a consumer 232 * that needs a value materializes it via api_materialize_arith_to. This was 233 * gated off while the load/store EA rider existed (Track 7 removed it); 234 * Track 6.3 flipped the gate back on. 235 * ============================================================ */ 236 237 ApiSValue api_make_arith_unop(KitCg* g, UnOp op, Operand a, KitCgTypeId ty, 238 int a_owned) { 239 ApiDelayed* d = api_delayed_alloc(g); 240 d->arith = (ApiDelayedArith){.kind = API_DELAYED_UNOP, 241 .un_op = op, 242 .a = a, 243 .a_owned = a_owned ? 1u : 0u}; 244 return (ApiSValue){.type = ty, 245 .source_local = KIT_CG_LOCAL_NONE, 246 .delayed = d, 247 .flags = API_SV_PACK(SV_ARITH, RES_INHERENT, 0, 0)}; 248 } 249 250 ApiSValue api_make_arith_binop(KitCg* g, BinOp op, Operand a, Operand b, 251 KitCgTypeId ty, int a_owned, int b_owned) { 252 ApiDelayed* d = api_delayed_alloc(g); 253 d->arith = (ApiDelayedArith){.kind = API_DELAYED_BINOP, 254 .bin_op = op, 255 .a = a, 256 .b = b, 257 .a_owned = a_owned ? 1u : 0u, 258 .b_owned = b_owned ? 1u : 0u}; 259 return (ApiSValue){.type = ty, 260 .source_local = KIT_CG_LOCAL_NONE, 261 .delayed = d, 262 .flags = API_SV_PACK(SV_ARITH, RES_INHERENT, 0, 0)}; 263 } 264 265 void api_release_arith(KitCg* g, ApiSValue* sv) { 266 api_delayed_free(g, sv->delayed); 267 sv->delayed = NULL; 268 api_sv_set_kind(sv, SV_OPERAND); 269 } 270 271 void api_materialize_arith_to(KitCg* g, ApiSValue* sv, Operand dst) { 272 ApiDelayed* d = sv->delayed; 273 if (d->arith.kind == API_DELAYED_UNOP) { 274 Operand a = api_op_kill_if_dead(g, d->arith.a, dst); 275 g->target->unop(g->target, d->arith.un_op, dst, a); 276 } else { 277 /* Flag dead-transient operands so the -O0 backend drops them after the op 278 * instead of spilling them at the next barrier (eager dead-operand drop). 279 */ 280 Operand a = api_op_kill_if_dead(g, d->arith.a, dst); 281 Operand b = api_op_kill_if_dead(g, d->arith.b, dst); 282 g->target->binop(g->target, d->arith.bin_op, dst, a, b); 283 } 284 api_delayed_free(g, d); 285 sv->delayed = NULL; 286 api_sv_set_kind(sv, SV_OPERAND); 287 sv->op = dst; 288 sv->type = dst.type; 289 api_sv_set_res(sv, RES_LOCAL); 290 api_sv_set_lvalue(sv, 0); 291 } 292 293 int api_arith_rhs_reusable(const ApiSValue* sv) { 294 if (sv->delayed->arith.kind == API_DELAYED_UNOP) return 0; 295 switch (sv->delayed->arith.bin_op) { 296 case BO_IADD: 297 case BO_IMUL: 298 case BO_AND: 299 case BO_OR: 300 case BO_XOR: 301 return 1; 302 default: 303 return 0; 304 } 305 } 306 307 int api_can_delay_int_arith(KitCg* g, KitCgTypeId ty, u32 flags) { 308 u32 width; 309 return g && !flags && api_foldable_int_type(g->c, ty, &width); 310 } 311 312 /* Strength reduction: rewrite a multiply / unsigned-divide / unsigned-remainder 313 * by a power-of-two immediate into a shift / and. Operates on the freshly 314 * popped operands (`a` = LHS, `b` = RHS) and the op; on a match it rewrites 315 * *op, *a and *b in place and returns 1. 316 * 317 * Only the cases whose plain wrapping equivalence is exact live here: 318 * x * 2^k -> x << k (multiply is commutative: the immediate may be on 319 * either side; the result is canonicalized so the 320 * variable is the shift's LHS) 321 * x u/ 2^k -> x u>> k 322 * x u% 2^k -> x & (2^k - 1) 323 * Signed division/remainder by a power of two needs a sign-bias sequence 324 * (round toward zero on negatives), so it is left to the optimizer. The caller 325 * gates this on flags==0, so trap/saturate/exact semantics never reach here. */ 326 static int api_imm_is_pow2(u64 v, u32* log2_out) { 327 u32 k; 328 if (v == 0 || (v & (v - 1u)) != 0) return 0; 329 for (k = 0; k < 64u; ++k) { 330 if (v == (1ull << k)) { 331 *log2_out = k; 332 return 1; 333 } 334 } 335 return 0; 336 } 337 338 int api_try_strength_reduce(KitCg* g, BinOp* op, KitCgTypeId ty, ApiSValue* a, 339 ApiSValue* b) { 340 u32 width; 341 u32 k = 0; 342 u64 v; 343 int a_imm, b_imm; 344 if (!g || !op || !a || !b) return 0; 345 if (!api_foldable_int_type(g->c, ty, &width)) return 0; 346 a_imm = api_sv_kind(a) == SV_OPERAND && a->op.kind == OPK_IMM; 347 b_imm = api_sv_kind(b) == SV_OPERAND && b->op.kind == OPK_IMM; 348 switch (*op) { 349 case BO_IMUL: { 350 /* Both-imm is constant-folded before we get here; need exactly one, and 351 * the variable operand is canonicalized to the shift's LHS. */ 352 ApiSValue* imm_sv; 353 int imm_on_lhs; 354 if (b_imm && !a_imm) { 355 imm_sv = b; 356 imm_on_lhs = 0; 357 } else if (a_imm && !b_imm) { 358 imm_sv = a; 359 imm_on_lhs = 1; 360 } else { 361 return 0; 362 } 363 v = api_mask_width((u64)imm_sv->op.v.imm, width); 364 if (!api_imm_is_pow2(v, &k) || k == 0) return 0; 365 if (imm_on_lhs) { 366 ApiSValue tmp = *a; 367 *a = *b; 368 *b = tmp; 369 } 370 b->op.v.imm = (i64)k; 371 b->op.type = ty; 372 *op = BO_SHL; 373 return 1; 374 } 375 case BO_UDIV: 376 case BO_UREM: 377 if (!b_imm || a_imm) return 0; /* RHS imm, LHS a real value */ 378 v = api_mask_width((u64)b->op.v.imm, width); 379 if (!api_imm_is_pow2(v, &k) || k == 0) return 0; 380 if (*op == BO_UDIV) { 381 b->op.v.imm = (i64)k; 382 *op = BO_SHR_U; 383 } else { 384 b->op.v.imm = (i64)api_mask_width(v - 1u, width); 385 *op = BO_AND; 386 } 387 b->op.type = ty; 388 return 1; 389 default: 390 return 0; 391 } 392 } 393 394 int api_op_is_int_identity(KitCg* g, BinOp op, KitCgTypeId ty, i64 imm) { 395 u32 width; 396 u64 v; 397 if (!api_foldable_int_type(g->c, ty, &width)) return 0; 398 v = api_mask_width((u64)imm, width); 399 switch (op) { 400 case BO_IADD: 401 case BO_ISUB: 402 case BO_OR: 403 case BO_XOR: 404 case BO_SHL: 405 case BO_SHR_S: 406 case BO_SHR_U: 407 return v == 0; 408 case BO_IMUL: 409 case BO_SDIV: 410 case BO_UDIV: 411 return v == 1; 412 case BO_AND: 413 return v == api_width_mask(width); 414 default: 415 return 0; 416 } 417 } 418 419 int api_try_collapse_binop_identity(KitCg* g, BinOp op, KitCgTypeId ty, 420 ApiSValue* a, ApiSValue* b, 421 ApiSValue* out) { 422 u32 width; 423 u64 av = 0; 424 u64 bv = 0; 425 if (!api_foldable_int_type(g->c, ty, &width)) return 0; 426 if (api_sv_kind(a) == SV_OPERAND && a->op.kind == OPK_IMM) 427 av = api_mask_width((u64)a->op.v.imm, width); 428 if (api_sv_kind(b) == SV_OPERAND && b->op.kind == OPK_IMM) 429 bv = api_mask_width((u64)b->op.v.imm, width); 430 431 if (api_sv_kind(b) == SV_OPERAND && b->op.kind == OPK_IMM && 432 api_sv_kind(a) == SV_OPERAND && a->op.kind != OPK_IMM && 433 api_op_is_int_identity(g, op, ty, b->op.v.imm)) { 434 *out = api_make_sv_with_local_ownership( 435 a->op, ty, api_sv_owns_operand_local(a, &a->op)); 436 api_sv_set_res(a, RES_INHERENT); 437 return 1; 438 } 439 if (api_sv_kind(b) == SV_OPERAND && b->op.kind == OPK_IMM && 440 api_sv_kind(a) == SV_OPERAND && a->op.kind != OPK_IMM && 441 (op == BO_SREM || op == BO_UREM || op == BO_IMUL || op == BO_AND || 442 op == BO_OR)) { 443 if ((op == BO_SREM || op == BO_UREM) && bv == 1) { 444 *out = api_make_sv(api_op_imm(0, ty), ty); 445 return 1; 446 } 447 if ((op == BO_IMUL || op == BO_AND) && bv == 0) { 448 *out = api_make_sv(api_op_imm(0, ty), ty); 449 return 1; 450 } 451 if (op == BO_OR && bv == api_width_mask(width)) { 452 *out = 453 api_make_sv(api_op_imm(api_fold_result(g->c, ty, bv, width), ty), ty); 454 return 1; 455 } 456 } 457 if (api_sv_kind(a) == SV_OPERAND && a->op.kind == OPK_IMM && 458 api_sv_kind(b) == SV_OPERAND && b->op.kind != OPK_IMM && 459 (op == BO_IADD || op == BO_IMUL || op == BO_OR || op == BO_XOR || 460 op == BO_AND) && 461 api_op_is_int_identity(g, op, ty, a->op.v.imm)) { 462 *out = api_make_sv_with_local_ownership( 463 b->op, ty, api_sv_owns_operand_local(b, &b->op)); 464 api_sv_set_res(b, RES_INHERENT); 465 return 1; 466 } 467 if (api_sv_kind(a) == SV_OPERAND && a->op.kind == OPK_IMM && 468 api_sv_kind(b) == SV_OPERAND && b->op.kind != OPK_IMM && 469 (op == BO_IMUL || op == BO_AND || op == BO_OR)) { 470 if ((op == BO_IMUL || op == BO_AND) && av == 0) { 471 *out = api_make_sv(api_op_imm(0, ty), ty); 472 return 1; 473 } 474 if (op == BO_OR && av == api_width_mask(width)) { 475 *out = 476 api_make_sv(api_op_imm(api_fold_result(g->c, ty, av, width), ty), ty); 477 return 1; 478 } 479 } 480 return 0; 481 } 482 483 int api_try_fold_arith_chain(KitCg* g, BinOp op, KitCgTypeId ty, ApiSValue* a, 484 ApiSValue* b, ApiSValue* out) { 485 i64 folded; 486 BinOp result_op; 487 ApiDelayed* ad; 488 if (api_sv_kind(a) != SV_ARITH || 489 a->delayed->arith.kind != API_DELAYED_BINOP || 490 a->delayed->arith.a.kind != OPK_LOCAL || 491 a->delayed->arith.b.kind != OPK_IMM || api_sv_kind(b) != SV_OPERAND || 492 b->op.kind != OPK_IMM) { 493 return 0; 494 } 495 ad = a->delayed; 496 result_op = ad->arith.bin_op; 497 switch (ad->arith.bin_op) { 498 case BO_IADD: 499 if (op == BO_IADD) { 500 if (!api_try_fold_int_binop(g, BO_IADD, ty, ad->arith.b.v.imm, 501 b->op.v.imm, &folded)) 502 return 0; 503 result_op = BO_IADD; 504 } else if (op == BO_ISUB) { 505 if (!api_try_fold_int_binop(g, BO_ISUB, ty, ad->arith.b.v.imm, 506 b->op.v.imm, &folded)) 507 return 0; 508 result_op = BO_IADD; 509 } else { 510 return 0; 511 } 512 break; 513 case BO_ISUB: 514 if (op == BO_IADD) { 515 if (!api_try_fold_int_binop(g, BO_ISUB, ty, b->op.v.imm, 516 ad->arith.b.v.imm, &folded)) 517 return 0; 518 result_op = BO_IADD; 519 } else if (op == BO_ISUB) { 520 if (!api_try_fold_int_binop(g, BO_IADD, ty, ad->arith.b.v.imm, 521 b->op.v.imm, &folded)) 522 return 0; 523 result_op = BO_ISUB; 524 } else { 525 return 0; 526 } 527 break; 528 case BO_XOR: 529 if (op != BO_XOR || 530 !api_try_fold_int_binop(g, BO_XOR, ty, ad->arith.b.v.imm, b->op.v.imm, 531 &folded)) 532 return 0; 533 result_op = BO_XOR; 534 break; 535 case BO_AND: 536 if (op != BO_AND || 537 !api_try_fold_int_binop(g, BO_AND, ty, ad->arith.b.v.imm, b->op.v.imm, 538 &folded)) 539 return 0; 540 result_op = BO_AND; 541 break; 542 case BO_OR: 543 if (op != BO_OR || 544 !api_try_fold_int_binop(g, BO_OR, ty, ad->arith.b.v.imm, b->op.v.imm, 545 &folded)) 546 return 0; 547 result_op = BO_OR; 548 break; 549 default: 550 return 0; 551 } 552 if (api_op_is_int_identity(g, result_op, ty, folded)) { 553 /* Collapses to a's input operand: out is a plain value with no payload, and 554 * a keeps its payload for the caller's api_release to reclaim. The owned 555 * local (if any) is handed to out; a's stale a_owned is never acted on 556 * (api_release_arith only frees the payload). */ 557 *out = api_make_sv_with_local_ownership(ad->arith.a, ty, ad->arith.a_owned); 558 return 1; 559 } 560 /* Chain-folds into an updated delayed binop. Move a's payload to out (a 561 * pointer transfer, not a deep copy) and null a's pointer so the caller's 562 * api_release reclaims nothing — out owns the payload until it materializes. 563 */ 564 ad->arith.bin_op = result_op; 565 ad->arith.b.v.imm = folded; 566 *out = *a; 567 a->delayed = NULL; 568 return 1; 569 } 570 571 int api_try_fold_unary_chain(ApiSValue* a, UnOp op, KitCgTypeId ty, 572 ApiSValue* out) { 573 if (op != UO_BNOT || api_sv_kind(a) != SV_ARITH || 574 a->delayed->arith.kind != API_DELAYED_UNOP || 575 a->delayed->arith.un_op != UO_BNOT || 576 a->delayed->arith.a.kind != OPK_LOCAL) { 577 return 0; 578 } 579 /* out borrows a's input operand (a plain value); a keeps its payload for the 580 * caller's api_release to reclaim. */ 581 *out = api_make_sv_with_local_ownership(a->delayed->arith.a, ty, 582 a->delayed->arith.a_owned); 583 return 1; 584 } 585 586 /* ============================================================ 587 * 3. Const-local store-to-load forwarding 588 * ============================================================ */ 589 590 void api_local_const_clear(ApiSourceLocal* rec) { 591 if (!rec) return; 592 rec->const_valid = 0; 593 rec->const_value = 0; 594 } 595 596 void api_local_const_clear_all(KitCg* g) { 597 KitCgLocal cur; 598 if (!g) return; 599 /* Walk only the locals threaded onto the const-active list, not all nlocals: 600 * clearing an already-invalid local is a no-op, so this set is exactly the 601 * one worth visiting. Boundary ops (memory/control/address-taken) are hit on 602 * every call/branch/store, so this O(#tracked) vs O(nlocals) is what keeps a 603 * function with many locals (e.g. a large zeroed aggregate) out of O(n^2). */ 604 cur = g->const_head; 605 while (cur != KIT_CG_LOCAL_NONE) { 606 ApiSourceLocal* rec = api_local_from_handle(g, cur); 607 KitCgLocal next = rec ? rec->const_next : KIT_CG_LOCAL_NONE; 608 if (rec) { 609 rec->const_valid = 0; 610 rec->const_value = 0; 611 rec->const_listed = 0; 612 rec->const_next = KIT_CG_LOCAL_NONE; 613 } 614 cur = next; 615 } 616 g->const_head = KIT_CG_LOCAL_NONE; 617 } 618 619 void api_local_const_memory_boundary(KitCg* g) { api_local_const_clear_all(g); } 620 621 void api_local_const_control_boundary(KitCg* g) { 622 api_local_const_clear_all(g); 623 } 624 625 void api_local_const_address_taken(KitCg* g, KitCgLocal local) { 626 api_local_const_clear_all(g); 627 api_local_const_clear(api_local_from_handle(g, local)); 628 } 629 630 int api_local_const_can_track(KitCg* g, const ApiSourceLocal* rec, 631 KitCgMemAccess access) { 632 u32 width; 633 KitCgTypeId ty; 634 u64 access_size; 635 u64 local_size; 636 if (!g || !rec) return 0; 637 if (rec->kind != API_SOURCE_LOCAL_AUTO) return 0; 638 if (access.flags & KIT_CG_MEM_VOLATILE) return 0; 639 ty = resolve_type(g->c, access.type); 640 if (!ty) ty = rec->type; 641 if (ty != rec->type) return 0; 642 access_size = abi_cg_sizeof(g->c->abi, ty); 643 local_size = abi_cg_sizeof(g->c->abi, rec->type); 644 if (access_size != local_size) return 0; 645 return api_foldable_int_like_type(g->c, ty, &width); 646 } 647 648 void api_local_const_store(KitCg* g, KitCgLocal local, KitCgMemAccess access, 649 i64 value) { 650 ApiSourceLocal* rec = api_local_from_handle(g, local); 651 KitCgTypeId ty; 652 u32 width; 653 if (!api_local_const_can_track(g, rec, access)) { 654 api_local_const_clear(rec); 655 return; 656 } 657 ty = resolve_type(g->c, access.type); 658 if (!ty) ty = rec->type; 659 if (!api_foldable_int_like_type(g->c, ty, &width)) { 660 api_local_const_clear(rec); 661 return; 662 } 663 rec->const_value = api_fold_result(g->c, ty, (u64)value, width); 664 rec->const_valid = 1; 665 /* Thread onto the const-active list so the next boundary can find and clear 666 * it without scanning all nlocals. A local already listed (possibly cleared 667 * since) stays linked once; membership is independent of const_valid. */ 668 if (!rec->const_listed) { 669 rec->const_next = g->const_head; 670 g->const_head = local; 671 rec->const_listed = 1; 672 } 673 } 674 675 int api_local_const_load(KitCg* g, KitCgLocal local, KitCgMemAccess access, 676 Operand* out) { 677 ApiSourceLocal* rec = api_local_from_handle(g, local); 678 KitCgTypeId ty; 679 u32 width; 680 if (!out || !api_local_const_can_track(g, rec, access)) return 0; 681 if (!rec->const_valid) return 0; 682 ty = resolve_type(g->c, access.type); 683 if (!ty) ty = rec->type; 684 if (!api_foldable_int_like_type(g->c, ty, &width)) return 0; 685 *out = 686 api_op_imm(api_fold_result(g->c, ty, (u64)rec->const_value, width), ty); 687 return 1; 688 }