opt.c (43969B)
1 #include <kit/config.h> 2 #include <string.h> 3 4 #include "abi/abi.h" 5 #include "cg/ir.h" 6 #include "cg/ir_recorder.h" 7 #include "cg/native_asm.h" 8 #include "cg/native_direct_target.h" 9 #include "cg/type.h" 10 #include "core/arena.h" 11 #include "core/core.h" 12 #include "core/diag.h" 13 #include "core/hashmap.h" 14 #include "core/metrics.h" 15 #include "core/pool.h" 16 #include "core/slice.h" 17 #include "core/strbuf.h" 18 #include "debug/debug.h" 19 #include "obj/symresolve.h" 20 #include "opt/opt_internal.h" 21 22 #undef Operand 23 #undef CGCallDesc 24 #undef CGFuncDesc 25 #undef CGParamDesc 26 #undef CGScopeDesc 27 28 /* Fixpoint bound for the whole-program inliner. opt_inline internally clamps to 29 * 4; an inlined straightline body introduces no new call sites, so a small 30 * bound converges. */ 31 #define OPT_WHOLE_PROGRAM_INLINE_ITERS 4 32 33 typedef struct OptImpl { 34 Compiler* c; 35 CgTarget* target; 36 NativeTarget* native; 37 int level; 38 /* Whole-program (LTO) mode: defer all per-function emission to finalize so 39 * the module-wide sweep can GC dead symbols and run cross-function inlining 40 * over the full reachable set. Enabled whenever the optimizer runs (-O1 and 41 * above; see opt_cgtarget_new). The ARM64 path already defers 42 * unconditionally; this generalizes that to every arch and adds the inliner. 43 */ 44 int whole_program; 45 CgFinishPolicy finish_policy; 46 Writer* dump_writer; 47 /* Registry of functions recorded so far, for tiny-inline callee lookup. 48 * `lowered_cache` is parallel to `cg_by_sym`: a lazily re-lowered 49 * pre-machinize Func for the callee, reused across all call sites. All on 50 * c->tu (persist for the whole translation unit). */ 51 CgIrFunc** cg_by_sym; 52 Func** lowered_cache; 53 u32 ncg; 54 u32 cg_cap; 55 } OptImpl; 56 57 HASHMAP_DEFINE(OptFuncIndex, ObjSymId, u32, hash_u32); 58 59 /* Section-indexed view of the relocatable-data symbols, built once so the 60 * data-reloc rooting pass can find the symbol that storage-contains a reloc in 61 * O(syms-in-section) rather than scanning every symbol per reloc. `entries` is 62 * packed by section (stable iteration order within a section preserved so the 63 * "first containing symbol" choice matches the old symiter walk); `by_section` 64 * maps a section id to its [start, start+count) slice of `entries`. */ 65 typedef struct OptDataSymEntry { 66 ObjSymId id; 67 u64 begin; /* containment range start (s->value) */ 68 u64 end; /* containment range end (begin + max(size, 1)) */ 69 int exported; 70 } OptDataSymEntry; 71 typedef struct OptSecSlice { 72 u32 start; 73 u32 count; 74 } OptSecSlice; 75 HASHMAP_DEFINE(OptSecSymIndex, ObjSecId, OptSecSlice, hash_u32); 76 typedef struct OptDataSymTable { 77 OptDataSymEntry* entries; 78 OptSecSymIndex by_section; 79 } OptDataSymTable; 80 81 /* A symbol whose definition can be replaced at link time must not have its body 82 * inlined — the inlined copy would defeat the override. Weak definitions are 83 * interposable in every output kind, so they are never safe to inline. (The 84 * broader default-visibility interposition under -shared is governed by the 85 * preserved set; see doc/plan/LTO.md §5/§9.) */ 86 static int opt_cg_func_interposable(OptImpl* o, const CgIrFunc* cg) { 87 const ObjSym* s; 88 if (!cg || cg->desc.sym == OBJ_SYM_NONE) return 0; 89 if (cg->desc.sym_bind == SB_WEAK) return 1; 90 s = obj_symbol_get(o->target->obj, cg->desc.sym); 91 return s && s->bind == SB_WEAK; 92 } 93 94 /* Lower a recorded function to the pre-machinize Func used by the inliners, and 95 * mark interposable definitions INLINE_NEVER so neither the streaming 96 * tiny-inliner nor the whole-program inliner fuses their bodies into callers. 97 * Marking the callee's policy is honored by effective_inline_policy in both. */ 98 static Func* opt_lower_for_inline(OptImpl* o, const CgIrFunc* cg) { 99 Func* f = opt_func_from_cg_ir(o->c, cg); 100 if (f && opt_cg_func_interposable(o, cg)) 101 f->desc.inline_policy = KIT_CG_INLINE_NEVER; 102 return f; 103 } 104 105 /* Lazily re-lower (and cache) the pre-machinize Func for a recorded callee 106 * symbol. Returns NULL for forward-defined callees not yet recorded. */ 107 static Func* opt_tiny_callee_lookup(void* ctx, ObjSymId sym) { 108 OptImpl* o = (OptImpl*)ctx; 109 for (u32 i = 0; i < o->ncg; ++i) { 110 if (o->cg_by_sym[i]->desc.sym != sym) continue; 111 if (!o->lowered_cache[i]) 112 o->lowered_cache[i] = opt_lower_for_inline(o, o->cg_by_sym[i]); 113 return o->lowered_cache[i]; 114 } 115 return NULL; 116 } 117 118 static void opt_registry_add(OptImpl* o, CgIrFunc* f) { 119 if (o->ncg == o->cg_cap) { 120 u32 ncap = o->cg_cap ? o->cg_cap * 2u : 16u; 121 CgIrFunc** ncg = arena_array(o->c->tu, CgIrFunc*, ncap); 122 Func** nlc = arena_zarray(o->c->tu, Func*, ncap); 123 if (o->ncg) { 124 memcpy(ncg, o->cg_by_sym, sizeof(ncg[0]) * o->ncg); 125 memcpy(nlc, o->lowered_cache, sizeof(nlc[0]) * o->ncg); 126 } 127 o->cg_by_sym = ncg; 128 o->lowered_cache = nlc; 129 o->cg_cap = ncap; 130 } 131 o->cg_by_sym[o->ncg++] = f; 132 } 133 134 static void opt_dbg_dump(OptImpl* o, Func* f, const char* tag) { 135 Func view; 136 Func* graph = f; 137 const char* s = kit_debug_getenv("KIT_DUMP"); 138 KitWriter* w = NULL; 139 size_t len = 0; 140 const uint8_t* bytes; 141 if (!s) return; 142 if (strcmp(s, "1") != 0 && strcmp(s, tag) != 0) return; 143 if (opt_mir_view(f, &view)) graph = &view; 144 kit_writer_mem(o->c->ctx->heap, &w); 145 opt_ir_dump(graph, w); 146 bytes = kit_writer_mem_bytes(w, &len); 147 diag_emit(o->c->ctx->diag, KIT_DIAG_NOTE, f->desc.loc, "DUMP %s:\n%.*s", tag, 148 (int)len, (const char*)bytes); 149 } 150 151 /* CFG-prep prefix shared by the streaming and whole-program pipelines: lower's 152 * raw blocks -> built CFG -> jump cleanup -> rebuilt CFG -> local simplify. In 153 * whole-program mode this runs on every reachable function before opt_inline 154 * sees the FuncSet, so the inliner observes the same block shape the streaming 155 * path does. */ 156 static void opt_o1_native_prepare(OptImpl* o, Func* f) { 157 if (!o->native) 158 compiler_panic(o->c, f ? f->desc.loc : (SrcLoc){0, 0, 0}, 159 "O1 optimizer requires a native target"); 160 opt_dbg_dump(o, f, "entry"); 161 162 metrics_count(o->c, "opt.funcs", 1); 163 metrics_count(o->c, "opt.blocks", f->nblocks); 164 metrics_count(o->c, "opt.pregs", f->npregs); 165 166 metrics_scope_begin(o->c, "opt.cfg.build_1"); 167 opt_build_cfg(f); 168 metrics_scope_end(o->c, "opt.cfg.build_1"); 169 metrics_scope_begin(o->c, "opt.cfg.jump_cleanup_cfg"); 170 opt_jump_cleanup(f, OPT_JUMP_CLEANUP_CFG); 171 metrics_scope_end(o->c, "opt.cfg.jump_cleanup_cfg"); 172 metrics_scope_begin(o->c, "opt.cfg.build_2"); 173 opt_build_cfg(f); 174 metrics_scope_end(o->c, "opt.cfg.build_2"); 175 /* O1.md W9+W10: one-pass branch cleanup (collapse same-target + forward 176 * pass-through blocks once) plus constant cmp_branch folding. Linear: a single 177 * memoized forwarding scan and at most one CFG rebuild (inside the helper). 178 * Folded into the build_2/simplify_local metric span; it is a single linear 179 * scan that does not move sqlite -O1 timing. */ 180 opt_jump_cleanup_o1(f); 181 metrics_scope_begin(o->c, "opt.cfg.simplify_local"); 182 opt_simplify_local(f); 183 metrics_scope_end(o->c, "opt.cfg.simplify_local"); 184 } 185 186 /* The machinize-through-emit suffix. `cfg_dirty` is set by the whole-program 187 * path: opt_inline mutated the caller's blocks in place and left CFG analysis 188 * stale, so the CFG must be rebuilt before tiny-inline/verify/machinize. The 189 * streaming path passes 0 (prepare just built it). */ 190 static void opt_o1_native_finish(OptImpl* o, Func* f, int cfg_dirty) { 191 OptLiveInfo live; 192 OptLiveInfo regalloc_live; 193 194 if (cfg_dirty) { 195 /* opt_inline maintains succ + emit_order only; rebuild preds/CFG and merge 196 * the BR-glue chains back into straight-line blocks (build_cfg -> 197 * jump_cleanup -> build_cfg) before any pass that needs the analysis. */ 198 opt_build_cfg(f); 199 opt_jump_cleanup(f, OPT_JUMP_CLEANUP_CFG); 200 opt_build_cfg(f); 201 } 202 203 metrics_scope_begin(o->c, "opt.o1.tiny_inline"); 204 int inlined = opt_try_tiny_inline(f, opt_tiny_callee_lookup, o); 205 metrics_scope_end(o->c, "opt.o1.tiny_inline"); 206 if (inlined) { 207 /* inline_call_site invalidated CFG and left preds stale (it maintains 208 * succ + emit_order only); rebuilding is required before verify/machinize. 209 * Then merge the BR-glue chain (pre -> body -> cont) back into 210 * straight-line blocks so regalloc sees no artificial boundaries, mirroring 211 * the prologue's build_cfg -> jump_cleanup -> build_cfg idiom. */ 212 opt_build_cfg(f); 213 opt_jump_cleanup(f, OPT_JUMP_CLEANUP_CFG); 214 opt_build_cfg(f); 215 } 216 217 metrics_scope_begin(o->c, "opt.cfg.verify"); 218 opt_verify(f, "lowering-cfg"); 219 metrics_scope_end(o->c, "opt.cfg.verify"); 220 221 metrics_scope_begin(o->c, "opt.machinize"); 222 opt_machinize_native(f, o->native); 223 metrics_scope_end(o->c, "opt.machinize"); 224 metrics_scope_begin(o->c, "opt.machinize.verify"); 225 opt_verify(f, "lowering-machinize"); 226 metrics_scope_end(o->c, "opt.machinize.verify"); 227 228 metrics_scope_begin(o->c, "opt.o1.addr_xform_pregs"); 229 opt_addr_xform_pregs(f); 230 metrics_scope_end(o->c, "opt.o1.addr_xform_pregs"); 231 metrics_scope_begin(o->c, "opt.o1.addr_xform.verify"); 232 opt_verify(f, "o1-addr-xform"); 233 metrics_scope_end(o->c, "opt.o1.addr_xform.verify"); 234 metrics_scope_begin(o->c, "opt.o1.promote_scalar_locals"); 235 opt_promote_scalar_locals(f); 236 metrics_scope_end(o->c, "opt.o1.promote_scalar_locals"); 237 metrics_scope_begin(o->c, "opt.o1.promote_scalar.verify"); 238 opt_verify(f, "o1-promote-scalar"); 239 metrics_scope_end(o->c, "opt.o1.promote_scalar.verify"); 240 metrics_scope_begin(o->c, "opt.o1.addr_of_global_cse"); 241 opt_addr_of_global_cse(f); 242 metrics_scope_end(o->c, "opt.o1.addr_of_global_cse"); 243 metrics_scope_begin(o->c, "opt.o1.addr_of_global.verify"); 244 opt_verify(f, "o1-addr-global-cse"); 245 metrics_scope_end(o->c, "opt.o1.addr_of_global.verify"); 246 247 metrics_scope_begin(o->c, "opt.build_loop_tree"); 248 opt_build_loop_tree(f); 249 metrics_scope_end(o->c, "opt.build_loop_tree"); 250 metrics_scope_begin(o->c, "opt.o1.lower_loop_imm"); 251 opt_lower_loop_imm_operands(f, o->native); 252 metrics_scope_end(o->c, "opt.o1.lower_loop_imm"); 253 opt_verify(f, "o1-lower-loop-imm"); 254 metrics_scope_begin(o->c, "opt.o1.hoist_loop_consts"); 255 opt_hoist_loop_consts(f); 256 metrics_scope_end(o->c, "opt.o1.hoist_loop_consts"); 257 opt_verify(f, "o1-hoist-loop-consts"); 258 259 /* Machinization derives fixed-register effects from instruction shape. The 260 * loop-immediate pass above can turn an immediate shift into a register shift 261 * (x64 then uses/clobbers RCX), and the hoister is the last HIR shape mutator 262 * before allocation. Refresh once at this explicit boundary so liveness and 263 * regalloc consume effects for the final HIR, not the pre-transform form. */ 264 metrics_scope_begin(o->c, "opt.machine_effects.refresh"); 265 opt_refresh_machine_clobbers(f, o->native); 266 metrics_scope_end(o->c, "opt.machine_effects.refresh"); 267 268 metrics_scope_begin(o->c, "opt.live_blocks.pre_dde"); 269 memset(&live, 0, sizeof live); 270 opt_live_blocks(f, &live); 271 metrics_count(o->c, "opt.live_words", f->opt_live_words); 272 metrics_scope_end(o->c, "opt.live_blocks.pre_dde"); 273 metrics_scope_begin(o->c, "opt.dead_def_elim"); 274 opt_dead_def_elim_with_live(f, &live); 275 metrics_scope_end(o->c, "opt.dead_def_elim"); 276 277 metrics_scope_begin(o->c, "opt.regalloc"); 278 memset(®alloc_live, 0, sizeof regalloc_live); 279 opt_regalloc_locations(f, ®alloc_live); 280 metrics_scope_end(o->c, "opt.regalloc"); 281 metrics_scope_begin(o->c, "opt.regalloc.verify"); 282 opt_analysis_invalidate(f, OPT_ANALYSIS_DEF_USE); 283 opt_verify(f, "post-regalloc"); 284 metrics_scope_end(o->c, "opt.regalloc.verify"); 285 286 metrics_scope_begin(o->c, "opt.lower_mir"); 287 opt_lower_to_mir(f, ®alloc_live); 288 metrics_scope_end(o->c, "opt.lower_mir"); 289 metrics_scope_begin(o->c, "opt.lower_mir.verify"); 290 opt_mir_verify(f, "lower-mir"); 291 metrics_scope_end(o->c, "opt.lower_mir.verify"); 292 metrics_scope_begin(o->c, "opt.combine"); 293 opt_mir_combine(f, o->native); 294 metrics_scope_end(o->c, "opt.combine"); 295 metrics_scope_begin(o->c, "opt.combine.verify"); 296 opt_mir_verify(f, "post-mir-combine"); 297 metrics_scope_end(o->c, "opt.combine.verify"); 298 metrics_scope_begin(o->c, "opt.dce"); 299 opt_mir_dce(f); 300 metrics_scope_end(o->c, "opt.dce"); 301 metrics_scope_begin(o->c, "opt.dce.verify"); 302 opt_mir_verify(f, "post-mir-dce"); 303 metrics_scope_end(o->c, "opt.dce.verify"); 304 metrics_scope_begin(o->c, "opt.post_ra.jump_cleanup_cfg"); 305 opt_mir_jump_cleanup(f, OPT_JUMP_CLEANUP_CFG); 306 metrics_scope_end(o->c, "opt.post_ra.jump_cleanup_cfg"); 307 metrics_scope_begin(o->c, "opt.post_ra.build_cfg"); 308 opt_mir_build_cfg(f); 309 metrics_scope_end(o->c, "opt.post_ra.build_cfg"); 310 metrics_scope_begin(o->c, "opt.post_ra.verify"); 311 opt_mir_verify(f, "post-mir-jump-cfg"); 312 metrics_scope_end(o->c, "opt.post_ra.verify"); 313 metrics_scope_begin(o->c, "opt.post_ra.jump_cleanup_layout"); 314 opt_mir_jump_cleanup(f, OPT_JUMP_CLEANUP_LAYOUT); 315 metrics_scope_end(o->c, "opt.post_ra.jump_cleanup_layout"); 316 metrics_scope_begin(o->c, "opt.post_ra.layout.verify"); 317 opt_mir_verify(f, "post-mir-layout"); 318 metrics_scope_end(o->c, "opt.post_ra.layout.verify"); 319 320 opt_dbg_dump(o, f, "pre-emit"); 321 metrics_scope_begin(o->c, "opt.emit"); 322 if (o->native->mc && o->native->mc->debug) 323 debug_func_select(o->native->mc->debug, f->desc.sym); 324 opt_emit_native(o->c, f, o->native); 325 if (o->native->mc && o->native->mc->debug) 326 debug_func_end(o->native->mc->debug); 327 metrics_scope_end(o->c, "opt.emit"); 328 } 329 330 /* Streaming pipeline for one function: prepare + finish, back to back. Used by 331 * the eager per-function path (x64/rv64 below -O2) and by the ARM64/-O2 sweep 332 * for functions with no cross-function inlining to do. */ 333 static void opt_run_o1_native(OptImpl* o, Func* f) { 334 metrics_scope_begin(o->c, "opt.o1.total"); 335 opt_o1_native_prepare(o, f); 336 opt_o1_native_finish(o, f, /*cfg_dirty=*/0); 337 metrics_scope_end(o->c, "opt.o1.total"); 338 } 339 340 /* Sibling of opt_run_o1_native for the threaded interpreter. Runs the maximal 341 * target-independent subset and stops before opt_machinize_native: no regalloc, 342 * no MIR, no native emission. The result is a Func with virtual PRegs that the 343 * interpreter loader consumes directly. opt_verify is intentionally omitted — 344 * it is a debug aid and some checks assume the machinize-completed shape. */ 345 Func* opt_run_o1_interp(Compiler* c, const CgIrFunc* cg) { 346 Func* f; 347 OptLiveInfo live; 348 metrics_scope_begin(c, "opt.interp.total"); 349 f = opt_func_from_cg_ir(c, cg); 350 opt_build_cfg(f); 351 opt_jump_cleanup(f, OPT_JUMP_CLEANUP_CFG); 352 opt_build_cfg(f); 353 opt_simplify_local(f); 354 /* Target-independent local/escape optimizations. These run after machinize 355 * in the native pipeline but depend only on the PReg/frame-slot view, not on 356 * physical-register pools, so they are safe here and shrink the work the 357 * interpreter does (promote scalar locals to PRegs, fold addr-of-local, 358 * CSE addr-of-global). */ 359 opt_addr_xform_pregs(f); 360 opt_promote_scalar_locals(f); 361 opt_addr_of_global_cse(f); 362 opt_build_loop_tree(f); 363 memset(&live, 0, sizeof live); 364 opt_live_blocks(f, &live); 365 opt_dead_def_elim_with_live(f, &live); 366 metrics_scope_end(c, "opt.interp.total"); 367 { 368 if (kit_debug_getenv("KIT_DUMP_INTERP")) { 369 KitWriter* w = NULL; 370 size_t len = 0; 371 const uint8_t* bytes; 372 kit_writer_mem(c->ctx->heap, &w); 373 opt_ir_dump(f, w); 374 bytes = kit_writer_mem_bytes(w, &len); 375 diag_emit(c->ctx->diag, KIT_DIAG_NOTE, (SrcLoc){0, 0, 0}, 376 "INTERP IR:\n%.*s", (int)len, (const char*)bytes); 377 } 378 } 379 return f; 380 } 381 382 #if KIT_INTERP_ENABLED 383 /* Defined in src/interp/lower.c. Lowers a post-opt_run_o1_interp Func into the 384 * program's bytecode and registers it by symbol (ObjSymId, for internal calls) 385 * and by unmangled C name (for entry lookup). Declared here (rather than 386 * including the interp header into opt) to keep the dependency one-way. */ 387 void interp_capture_func(void* program, Func* f, ObjSymId sym, const char* name, 388 u32 name_len, const ObjBuilder* obj); 389 390 static void opt_maybe_capture_interp(OptImpl* o, const CgIrFunc* cg) { 391 ObjSymId sym; 392 Slice name; 393 if (!o->c->interp_sink) return; 394 sym = cg->desc.sym; 395 name = 396 (sym != OBJ_SYM_NONE) 397 ? pool_slice(o->c->global, obj_symbol_get(o->target->obj, sym)->name) 398 : SLICE_NULL; 399 interp_capture_func(o->c->interp_sink, opt_run_o1_interp(o->c, cg), sym, 400 name.s, (u32)name.len, o->target->obj); 401 } 402 #else 403 static void opt_maybe_capture_interp(OptImpl* o, const CgIrFunc* cg) { 404 (void)o; 405 (void)cg; 406 } 407 #endif 408 409 static void opt_dbg_dump_cg(OptImpl* o, const CgIrFunc* f) { 410 KitWriter* w = NULL; 411 size_t len = 0; 412 const uint8_t* bytes; 413 if (!kit_debug_getenv("KIT_DUMPCG")) return; 414 kit_writer_mem(o->c->ctx->heap, &w); 415 cg_ir_func_dump(f, w); 416 bytes = kit_writer_mem_bytes(w, &len); 417 diag_emit(o->c->ctx->diag, KIT_DIAG_NOTE, f->desc.loc, "CGIR:\n%.*s", 418 (int)len, (const char*)bytes); 419 } 420 421 static void opt_on_func(void* user, CgIrFunc* cg_func) { 422 OptImpl* o = (OptImpl*)user; 423 Func* f; 424 /* Register before lowering so later callers can resolve this (complete) 425 * function as a tiny-inline callee. */ 426 opt_registry_add(o, cg_func); 427 opt_dbg_dump_cg(o, cg_func); 428 /* The dump writer renders the semantic CG IR tape — the IR as recorded, 429 * before lowering to the optimizer's CFG form. */ 430 if (o->dump_writer) cg_ir_func_dump(cg_func, o->dump_writer); 431 /* Defer emission to the finalize sweep whenever whole-program mode is on — 432 * the same path for every arch. The sweep does GC + cross-function inlining 433 * over the full reachable set. The eager emit below is the fallback for a 434 * (currently unreachable) non-whole-program configuration. */ 435 if (o->whole_program) return; 436 metrics_scope_begin(o->c, "opt.o1.cg_ir_lower"); 437 f = opt_func_from_cg_ir(o->c, cg_func); 438 metrics_scope_end(o->c, "opt.o1.cg_ir_lower"); 439 opt_run_o1_native(o, f); 440 opt_maybe_capture_interp(o, cg_func); 441 } 442 443 static int opt_module_has_asm(const CgIrModule* module) { 444 if (!module) return 0; 445 if (module->nfile_scope_asms) return 1; 446 for (u32 i = 0; i < module->nfuncs; ++i) { 447 const CgIrFunc* f = module->funcs[i]; 448 if (!f || f->removed) continue; 449 for (u32 k = 0; k < f->ninsts; ++k) 450 if (f->insts[k].op == CG_IR_ASM_BLOCK) return 1; 451 } 452 return 0; 453 } 454 455 static int opt_sym_in_preserved_section(OptImpl* o, ObjSymId sym, 456 const ObjSym* s) { 457 const Section* sec; 458 const ObjAtom* atom; 459 ObjAtomId aid; 460 if (!o || !s || s->section_id == OBJ_SEC_NONE) return 0; 461 sec = obj_section_get(o->target->obj, s->section_id); 462 if (sec && ((sec->flags & SF_RETAIN) || sec->sem == SSEM_INIT_ARRAY || 463 sec->sem == SSEM_FINI_ARRAY || sec->sem == SSEM_PREINIT_ARRAY)) 464 return 1; 465 aid = obj_atom_find_symbol(o->target->obj, sym); 466 atom = obj_atom_get(o->target->obj, aid); 467 return atom && (atom->flags & OBJ_ATOM_RETAIN); 468 } 469 470 static void opt_build_preserved_set(OptImpl* o, ObjSymSet* preserved) { 471 for (u32 i = 0; i < o->finish_policy.npreserved_symbols; ++i) { 472 ObjSymId sym = o->finish_policy.preserved_symbols[i]; 473 if (sym != OBJ_SYM_NONE) (void)ObjSymSet_set(preserved, sym, 1); 474 } 475 } 476 477 static int opt_sym_must_stay_external(OptImpl* o, int module_has_asm, 478 const ObjSymSet* preserved, ObjSymId sym, 479 const ObjSym* s) { 480 if (!s || s->removed) return 1; 481 if (s->bind == SB_LOCAL) return 1; 482 if (o->finish_policy.output_kind != KIT_CG_OUTPUT_EXECUTABLE) return 1; 483 if (o->finish_policy.interposition_policy == 484 KIT_CG_INTERPOSITION_DEFAULT_VISIBILITY) 485 return 1; 486 if (ObjSymSet_get(preserved, sym)) return 1; 487 if (s->bind == SB_WEAK) return 1; 488 if (s->kind == SK_IFUNC) return 1; 489 if (s->flags & KIT_CG_SYM_USED) return 1; 490 if (module_has_asm) return 1; 491 if (opt_sym_in_preserved_section(o, sym, s)) return 1; 492 return 0; 493 } 494 495 static int opt_sym_internalizable(const ObjSym* s) { 496 if (!s || s->removed || s->bind == SB_LOCAL) return 0; 497 if (!symresolve_sym_is_def(s)) return 0; 498 switch ((SymKind)s->kind) { 499 case SK_FUNC: 500 case SK_OBJ: 501 case SK_TLS: 502 return 1; 503 default: 504 return 0; 505 } 506 } 507 508 static void opt_internalize_non_preserved(OptImpl* o, const CgIrModule* module, 509 const ObjSymSet* preserved) { 510 ObjSymIter* it; 511 ObjSymEntry ent; 512 int module_has_asm; 513 if (!o || !o->target || !o->target->obj) return; 514 if (o->finish_policy.output_kind != KIT_CG_OUTPUT_EXECUTABLE) return; 515 module_has_asm = opt_module_has_asm(module); 516 it = obj_symiter_new(o->target->obj); 517 while (it && obj_symiter_next(it, &ent)) { 518 const ObjSym* s = ent.sym; 519 if (!opt_sym_internalizable(s)) continue; 520 if (opt_sym_must_stay_external(o, module_has_asm, preserved, ent.id, s)) 521 continue; 522 obj_symbol_set_bind(o->target->obj, ent.id, SB_LOCAL); 523 obj_symbol_set_vis(o->target->obj, ent.id, SV_HIDDEN); 524 } 525 if (it) obj_symiter_free(it); 526 } 527 528 static int opt_func_is_root(OptImpl* o, const CgIrFunc* f) { 529 const ObjSym* s; 530 if (!f || f->removed || f->desc.sym == OBJ_SYM_NONE) return 0; 531 s = obj_symbol_get(o->target->obj, f->desc.sym); 532 if (!s || s->removed) return 0; 533 if (s->bind != SB_LOCAL) return 1; 534 if (s->flags & KIT_CG_SYM_USED) return 1; 535 return 0; 536 } 537 538 static SymAttrs opt_func_sym_attrs(OptImpl* o, const CgIrFunc* f) { 539 SymAttrs a; 540 const ObjSym* s; 541 memset(&a, 0, sizeof a); 542 if (!f) return a; 543 s = obj_symbol_get(o->target->obj, f->desc.sym); 544 a.bind = f->desc.sym_bind ? f->desc.sym_bind : (s ? s->bind : SB_GLOBAL); 545 a.kind = f->desc.sym_kind ? f->desc.sym_kind : SK_FUNC; 546 a.size = s ? s->size : 0; 547 a.common_align = 0; 548 a.in_comdat = 0; 549 return a; 550 } 551 552 static void opt_resolve_duplicate_funcs(OptImpl* o, const CgIrModule* module, 553 OptFuncIndex* index) { 554 for (u32 i = 0; i < module->nfuncs; ++i) { 555 CgIrFunc* incoming = module->funcs[i]; 556 u32* existing_idx; 557 if (!incoming || incoming->removed || incoming->desc.sym == OBJ_SYM_NONE) 558 continue; 559 existing_idx = OptFuncIndex_get(index, incoming->desc.sym); 560 if (!existing_idx) { 561 (void)OptFuncIndex_set(index, incoming->desc.sym, i); 562 continue; 563 } 564 { 565 CgIrFunc* existing = module->funcs[*existing_idx]; 566 SymMergeResult mr = symresolve_merge(opt_func_sym_attrs(o, existing), 567 opt_func_sym_attrs(o, incoming)); 568 switch (mr.kind) { 569 case SYM_MERGE_REPLACE: 570 if (existing) existing->removed = 1; 571 (void)OptFuncIndex_set(index, incoming->desc.sym, i); 572 if (incoming->desc.sym_bind) 573 obj_symbol_set_bind(o->target->obj, incoming->desc.sym, 574 (SymBind)incoming->desc.sym_bind); 575 break; 576 case SYM_MERGE_KEEP_EXISTING: 577 case SYM_MERGE_COMDAT_DISCARD: 578 incoming->removed = 1; 579 if (existing && existing->desc.sym_bind) 580 obj_symbol_set_bind(o->target->obj, incoming->desc.sym, 581 (SymBind)existing->desc.sym_bind); 582 break; 583 case SYM_MERGE_COMMON: 584 incoming->removed = 1; 585 break; 586 case SYM_MERGE_ODR_ERROR: 587 compiler_panic(o->c, incoming->desc.loc, 588 "duplicate definition of symbol"); 589 } 590 } 591 } 592 } 593 594 static void opt_mark_func(u8* reachable, u8* queued, u32* queue, u32* qtail, 595 u32 idx) { 596 if (reachable[idx]) return; 597 reachable[idx] = 1; 598 if (!queued[idx]) { 599 queued[idx] = 1; 600 queue[(*qtail)++] = idx; 601 } 602 } 603 604 static void opt_mark_sym(OptFuncIndex* index, u8* reachable, u8* queued, 605 u32* queue, u32* qtail, ObjSymId sym) { 606 u32* slot; 607 if (sym == OBJ_SYM_NONE) return; 608 slot = OptFuncIndex_get(index, sym); 609 if (slot) opt_mark_func(reachable, queued, queue, qtail, *slot); 610 } 611 612 static void opt_mark_symset(OptFuncIndex* index, u8* reachable, u8* queued, 613 u32* queue, u32* qtail, const ObjSymSet* refs) { 614 if (!refs || !refs->cap) return; 615 for (u32 i = 0; i < refs->cap; ++i) { 616 ObjSymId sym = refs->slots[i].k; 617 if (sym != OBJ_SYM_NONE) 618 opt_mark_sym(index, reachable, queued, queue, qtail, sym); 619 } 620 } 621 622 static int opt_sym_is_relocatable_data(const ObjSym* s) { 623 if (!s || s->removed || s->section_id == OBJ_SEC_NONE) return 0; 624 return s->kind == SK_OBJ || s->kind == SK_TLS || s->kind == SK_COMMON; 625 } 626 627 static int opt_reloc_inside_sym(const Reloc* r, const ObjSym* s) { 628 u64 begin, end; 629 if (!r || r->removed || !opt_sym_is_relocatable_data(s)) return 0; 630 if (r->section_id != s->section_id) return 0; 631 begin = s->value; 632 end = begin + s->size; 633 if (s->size == 0) end = begin + 1u; 634 return (u64)r->offset >= begin && (u64)r->offset < end; 635 } 636 637 static void opt_enqueue_data_sym(OptImpl* o, ObjSymSet* seen, ObjSymId* queue, 638 u32* qtail, ObjSymId sym) { 639 const ObjSym* s; 640 if (sym == OBJ_SYM_NONE || ObjSymSet_get(seen, sym)) return; 641 s = obj_symbol_get(o->target->obj, sym); 642 if (!opt_sym_is_relocatable_data(s)) return; 643 (void)ObjSymSet_set(seen, sym, 1); 644 queue[(*qtail)++] = sym; 645 } 646 647 static void opt_enqueue_data_symset(OptImpl* o, ObjSymSet* seen, 648 ObjSymId* queue, u32* qtail, 649 const ObjSymSet* refs) { 650 if (!refs || !refs->cap) return; 651 for (u32 i = 0; i < refs->cap; ++i) { 652 ObjSymId sym = refs->slots[i].k; 653 if (sym != OBJ_SYM_NONE) opt_enqueue_data_sym(o, seen, queue, qtail, sym); 654 } 655 } 656 657 static void opt_mark_data_reloc_graph(OptImpl* o, OptFuncIndex* index, 658 u8* reachable, u8* queued, 659 u32* func_queue, u32* func_qtail, 660 ObjSymSet* data_seen, 661 ObjSymId* data_queue, u32* data_qhead, 662 u32* data_qtail) { 663 u32 nrel = obj_reloc_total(o->target->obj); 664 while (*data_qhead < *data_qtail) { 665 ObjSymId data_sym = data_queue[(*data_qhead)++]; 666 const ObjSym* data = obj_symbol_get(o->target->obj, data_sym); 667 if (!opt_sym_is_relocatable_data(data)) continue; 668 for (u32 i = 0; i < nrel; ++i) { 669 const Reloc* r = obj_reloc_at(o->target->obj, i); 670 if (!opt_reloc_inside_sym(r, data)) continue; 671 opt_mark_sym(index, reachable, queued, func_queue, func_qtail, r->sym); 672 opt_enqueue_data_sym(o, data_seen, data_queue, data_qtail, r->sym); 673 } 674 } 675 } 676 677 /* Build the section-indexed table of relocatable-data symbols in two passes 678 * (count per section, then fill), keeping per-section iteration order stable so 679 * downstream "first containing symbol" lookups match the historical symiter 680 * walk. `nsym` is the symbol-iterator upper bound already computed by the 681 * caller. */ 682 static void opt_data_sym_table_build(OptImpl* o, OptDataSymTable* t, u32 nsym) { 683 ObjSymIter* it; 684 ObjSymEntry ent; 685 OptSecSymIndex_init_cap(&t->by_section, o->c->ctx->heap, 0); 686 t->entries = nsym ? arena_array(o->c->tu, OptDataSymEntry, nsym) : NULL; 687 /* Pass 1: count relocatable-data symbols per section. */ 688 it = obj_symiter_new(o->target->obj); 689 while (it && obj_symiter_next(it, &ent)) { 690 const ObjSym* s = ent.sym; 691 OptSecSlice* slot; 692 if (!opt_sym_is_relocatable_data(s)) continue; 693 slot = OptSecSymIndex_get(&t->by_section, s->section_id); 694 if (slot) { 695 slot->count++; 696 } else { 697 OptSecSlice sl = {0, 1}; 698 (void)OptSecSymIndex_set(&t->by_section, s->section_id, sl); 699 } 700 } 701 if (it) obj_symiter_free(it); 702 /* Assign each section a contiguous [start, start+count) slice and reset the 703 * counts so pass 2 can use them as fill cursors. */ 704 { 705 u32 next = 0; 706 for (u32 i = 0; i < t->by_section.cap; ++i) { 707 if (!t->by_section.slots[i].k) continue; 708 t->by_section.slots[i].v.start = next; 709 next += t->by_section.slots[i].v.count; 710 t->by_section.slots[i].v.count = 0; 711 } 712 } 713 /* Pass 2: place symbols in their section slice, preserving iteration order. 714 */ 715 it = obj_symiter_new(o->target->obj); 716 while (it && obj_symiter_next(it, &ent)) { 717 const ObjSym* s = ent.sym; 718 OptSecSlice* slot; 719 OptDataSymEntry* e; 720 if (!opt_sym_is_relocatable_data(s)) continue; 721 slot = OptSecSymIndex_get(&t->by_section, s->section_id); 722 if (!slot) continue; /* unreachable: pass 1 inserted every such section */ 723 e = &t->entries[slot->start + slot->count++]; 724 e->id = ent.id; 725 e->begin = s->value; 726 e->end = s->value + (s->size ? s->size : 1u); 727 e->exported = s->bind != SB_LOCAL || (s->flags & KIT_CG_SYM_USED) ? 1 : 0; 728 } 729 if (it) obj_symiter_free(it); 730 } 731 732 static void opt_data_sym_table_fini(OptDataSymTable* t) { 733 OptSecSymIndex_fini(&t->by_section); 734 } 735 736 /* Find the data symbol whose storage contains reloc `r`, restricted to roots: 737 * if the reloc's section is RETAIN any containing symbol roots it, otherwise 738 * only an exported symbol does. Returns the first match in the section's stable 739 * order (matching the old per-symbol scan). */ 740 static ObjSymId opt_data_reloc_exported_root_sym(OptImpl* o, 741 const OptDataSymTable* t, 742 const Reloc* r) { 743 const Section* sec; 744 const OptSecSlice* slice; 745 int retained; 746 if (!r || r->removed || r->section_id == OBJ_SEC_NONE) return OBJ_SYM_NONE; 747 sec = obj_section_get(o->target->obj, r->section_id); 748 if (!sec || sec->removed || sec->kind == SEC_TEXT) return OBJ_SYM_NONE; 749 slice = OptSecSymIndex_get(&t->by_section, r->section_id); 750 if (!slice) return OBJ_SYM_NONE; 751 retained = (sec->flags & SF_RETAIN) ? 1 : 0; 752 for (u32 i = 0; i < slice->count; ++i) { 753 const OptDataSymEntry* e = &t->entries[slice->start + i]; 754 if ((u64)r->offset < e->begin || (u64)r->offset >= e->end) continue; 755 if (retained) return e->id; 756 if (!e->exported) continue; 757 return e->id; 758 } 759 return OBJ_SYM_NONE; 760 } 761 762 static void opt_root_exported_data_relocs(OptImpl* o, const OptDataSymTable* t, 763 ObjSymSet* data_seen, 764 ObjSymId* data_queue, 765 u32* data_qtail) { 766 u32 nrel = obj_reloc_total(o->target->obj); 767 for (u32 i = 0; i < nrel; ++i) { 768 const Reloc* r = obj_reloc_at(o->target->obj, i); 769 ObjSymId sym = opt_data_reloc_exported_root_sym(o, t, r); 770 if (sym != OBJ_SYM_NONE) 771 opt_enqueue_data_sym(o, data_seen, data_queue, data_qtail, sym); 772 } 773 } 774 775 static void opt_root_aliases(OptImpl* o, const CgIrModule* module, 776 OptFuncIndex* index, u8* reachable, u8* queued, 777 u32* queue, u32* qtail) { 778 for (u32 i = 0; module && i < module->naliases; ++i) { 779 const CgIrAlias* a = &module->aliases[i]; 780 const ObjSym* s = obj_symbol_get(o->target->obj, a->alias_sym); 781 if (!s || s->removed) continue; 782 if (s->bind != SB_LOCAL || (s->flags & KIT_CG_SYM_USED)) 783 opt_mark_sym(index, reachable, queued, queue, qtail, a->target_sym); 784 } 785 } 786 787 static void opt_refresh_or_prune_aliases(OptImpl* o, const CgIrModule* module, 788 OptFuncIndex* index, 789 const u8* reachable) { 790 for (u32 i = 0; module && i < module->naliases; ++i) { 791 const CgIrAlias* a = &module->aliases[i]; 792 const ObjSym* ts; 793 const ObjSym* as; 794 u32* target_idx = OptFuncIndex_get(index, a->target_sym); 795 if (!target_idx || !reachable[*target_idx]) { 796 as = obj_symbol_get(o->target->obj, a->alias_sym); 797 if (as && as->bind == SB_LOCAL) 798 obj_symbol_remove(o->target->obj, a->alias_sym); 799 continue; 800 } 801 ts = obj_symbol_get(o->target->obj, a->target_sym); 802 if (ts && !ts->removed && ts->section_id != OBJ_SEC_NONE) 803 obj_symbol_define(o->target->obj, a->alias_sym, ts->section_id, ts->value, 804 ts->size); 805 } 806 } 807 808 static void opt_prune_debug(OptImpl* o) { 809 if (o->native && o->native->mc && o->native->mc->debug) 810 debug_prune_removed_funcs(o->native->mc->debug); 811 } 812 813 /* Whole-module finalize: seed roots, walk the call/use + data-reloc graph, 814 * remove unreachable local symbols, then lower + optimize + emit only the live 815 * set. Arch-independent — the ARM64 path has always finalized this way; -O2 now 816 * routes every arch through here (see opt_on_finalize). When `do_inline` is set 817 * the live functions are lowered into a FuncSet and run through the 818 * whole-program inliner before the per-function machinize/emit suffix. */ 819 static void opt_whole_module_finalize(OptImpl* o, const CgIrModule* module, 820 int do_inline) { 821 OptFuncIndex index; 822 ObjSymSet preserved; 823 ObjSymSet data_seen; 824 OptDataSymTable data_syms; 825 u8* reachable; 826 u8* queued; 827 u32* queue; 828 ObjSymId* data_queue; 829 u32 qhead = 0; 830 u32 qtail = 0; 831 u32 data_qhead = 0; 832 u32 data_qtail = 0; 833 u32 nsym = 1; 834 if (!module || !module->nfuncs) return; 835 OptFuncIndex_init_cap(&index, o->c->ctx->heap, 0); 836 ObjSymSet_init_cap(&preserved, o->c->ctx->heap, 0); 837 ObjSymSet_init_cap(&data_seen, o->c->ctx->heap, 0); 838 reachable = arena_zarray(o->c->tu, u8, module->nfuncs); 839 queued = arena_zarray(o->c->tu, u8, module->nfuncs); 840 queue = arena_array(o->c->tu, u32, module->nfuncs); 841 { 842 ObjSymIter* it = obj_symiter_new(o->target->obj); 843 ObjSymEntry ent; 844 while (it && obj_symiter_next(it, &ent)) ++nsym; 845 if (it) obj_symiter_free(it); 846 } 847 data_queue = arena_array(o->c->tu, ObjSymId, nsym); 848 opt_data_sym_table_build(o, &data_syms, nsym); 849 opt_resolve_duplicate_funcs(o, module, &index); 850 opt_build_preserved_set(o, &preserved); 851 opt_internalize_non_preserved(o, module, &preserved); 852 for (u32 i = 0; i < module->nfuncs; ++i) { 853 if (module->funcs[i] && module->funcs[i]->removed) continue; 854 if (module->nfile_scope_asms || opt_func_is_root(o, module->funcs[i])) 855 opt_mark_func(reachable, queued, queue, &qtail, i); 856 } 857 opt_root_aliases(o, module, &index, reachable, queued, queue, &qtail); 858 opt_root_exported_data_relocs(o, &data_syms, &data_seen, data_queue, 859 &data_qtail); 860 opt_mark_data_reloc_graph(o, &index, reachable, queued, queue, &qtail, 861 &data_seen, data_queue, &data_qhead, &data_qtail); 862 while (qhead < qtail) { 863 CgIrFunc* f = module->funcs[queue[qhead++]]; 864 opt_mark_symset(&index, reachable, queued, queue, &qtail, &f->call_refs); 865 opt_mark_symset(&index, reachable, queued, queue, &qtail, &f->global_refs); 866 opt_enqueue_data_symset(o, &data_seen, data_queue, &data_qtail, 867 &f->global_refs); 868 opt_mark_data_reloc_graph(o, &index, reachable, queued, queue, &qtail, 869 &data_seen, data_queue, &data_qhead, &data_qtail); 870 } 871 opt_data_sym_table_fini(&data_syms); 872 for (u32 i = 0; i < module->nfuncs; ++i) { 873 CgIrFunc* cg_func = module->funcs[i]; 874 if (cg_func && cg_func->removed) continue; 875 if (reachable[i]) continue; 876 if (cg_func && cg_func->desc.sym != OBJ_SYM_NONE) { 877 const ObjSym* s = obj_symbol_get(o->target->obj, cg_func->desc.sym); 878 if (s && s->bind == SB_LOCAL) 879 obj_symbol_remove(o->target->obj, cg_func->desc.sym); 880 } 881 } 882 opt_prune_debug(o); 883 if (!do_inline) { 884 /* Streaming emit: lower and run the full per-function pipeline in place. 885 * Preserves the historical ARM64 -O1 behavior exactly. */ 886 for (u32 i = 0; i < module->nfuncs; ++i) { 887 Func* f; 888 if (!reachable[i]) continue; 889 metrics_scope_begin(o->c, "opt.o1.cg_ir_lower"); 890 f = opt_func_from_cg_ir(o->c, module->funcs[i]); 891 metrics_scope_end(o->c, "opt.o1.cg_ir_lower"); 892 opt_run_o1_native(o, f); 893 opt_maybe_capture_interp(o, module->funcs[i]); 894 } 895 } else { 896 /* Whole-program inline: lower + CFG-prep every live function into one 897 * FuncSet so the inliner can resolve direct callees by symbol across the 898 * module, inline under the growth-gated cost model, then run the 899 * machinize/emit suffix on each (cfg_dirty=1 because opt_inline left the 900 * caller CFGs stale). Functions and their source CgIrFuncs are tracked in 901 * parallel so the interp-capture re-lowers the right body. */ 902 FuncSet fs; 903 CgIrFunc** cg_srcs; 904 u32 nlive = 0; 905 for (u32 i = 0; i < module->nfuncs; ++i) 906 if (reachable[i] && module->funcs[i] && !module->funcs[i]->removed) 907 ++nlive; 908 memset(&fs, 0, sizeof fs); 909 fs.c = o->c; 910 fs.arena = o->c->tu; 911 fs.funcs = arena_array(o->c->tu, Func*, nlive ? nlive : 1u); 912 fs.cap = nlive; 913 cg_srcs = arena_array(o->c->tu, CgIrFunc*, nlive ? nlive : 1u); 914 for (u32 i = 0; i < module->nfuncs; ++i) { 915 Func* f; 916 if (!reachable[i] || !module->funcs[i] || module->funcs[i]->removed) 917 continue; 918 metrics_scope_begin(o->c, "opt.o1.cg_ir_lower"); 919 f = opt_lower_for_inline(o, module->funcs[i]); 920 metrics_scope_end(o->c, "opt.o1.cg_ir_lower"); 921 opt_o1_native_prepare(o, f); 922 cg_srcs[fs.nfuncs] = module->funcs[i]; 923 fs.funcs[fs.nfuncs++] = f; 924 } 925 metrics_scope_begin(o->c, "opt.inline.total"); 926 opt_inline(&fs, OPT_WHOLE_PROGRAM_INLINE_ITERS); 927 metrics_scope_end(o->c, "opt.inline.total"); 928 for (u32 k = 0; k < fs.nfuncs; ++k) { 929 metrics_scope_begin(o->c, "opt.o1.total"); 930 opt_o1_native_finish(o, fs.funcs[k], /*cfg_dirty=*/1); 931 metrics_scope_end(o->c, "opt.o1.total"); 932 opt_maybe_capture_interp(o, cg_srcs[k]); 933 } 934 } 935 opt_refresh_or_prune_aliases(o, module, &index, reachable); 936 ObjSymSet_fini(&data_seen); 937 ObjSymSet_fini(&preserved); 938 OptFuncIndex_fini(&index); 939 } 940 941 static void opt_on_finalize(void* user, const CgIrModule* module) { 942 OptImpl* o = (OptImpl*)user; 943 /* File-scope asm blocks are captured during recording (no live target then) 944 * and replayed here, before finalize. Emission order relative to functions 945 * does not matter: each block selects its own sections (.data/.text/...). */ 946 if (o->native && o->native->file_scope_asm && module) { 947 for (u32 i = 0; i < module->nfile_scope_asms; ++i) 948 o->native->file_scope_asm(o->native, module->file_scope_asms[i].src, 949 module->file_scope_asms[i].len); 950 } 951 /* Whole-program mode finalizes through the module sweep — one path for every 952 * arch: GC + cross-function inlining over the full reachable set. If it were 953 * off, every arch would have emitted eagerly in opt_on_func and the sweep 954 * would find nothing, so it is skipped. */ 955 if (o->whole_program) 956 opt_whole_module_finalize(o, module, /*do_inline=*/o->whole_program); 957 if (o->native && o->native->finalize) o->native->finalize(o->native); 958 } 959 960 static void opt_on_destroy(void* user) { 961 OptImpl* o = (OptImpl*)user; 962 if (o->native && o->native->destroy) o->native->destroy(o->native); 963 (void)o; 964 } 965 966 static int opt_on_local_static_data_begin(void* user, 967 const CGLocalStaticDataDesc* desc) { 968 (void)desc; 969 OptImpl* o = (OptImpl*)user; 970 return o && o->target && o->target->local_static_data_begin && 971 o->target->local_static_data_write && 972 o->target->local_static_data_label_addr && 973 o->target->local_static_data_end; 974 } 975 976 static const char* opt_on_tail_call_unrealizable_reason( 977 void* user, const struct CGFuncDesc* caller, const CGCallDesc* call) { 978 OptImpl* o = (OptImpl*)user; 979 int callee_va = 0, caller_va = 0; 980 u32 callee_nparams = 0; 981 u32 outgoing, incoming; 982 if (!o || !o->native || !o->native->signature_stack_bytes) return NULL; 983 /* A realized sibling call reuses the area this function received for its own 984 * incoming stack arguments to hold the callee's outgoing stack arguments, so 985 * the callee's must fit. If they don't, the tail isn't realizable and CG 986 * falls back to an ordinary call + return. */ 987 outgoing = o->native->signature_stack_bytes(o->native, call->fn_type, 988 &callee_va, &callee_nparams); 989 incoming = o->native->signature_stack_bytes(o->native, caller->fn_type, 990 &caller_va, NULL); 991 /* A variadic caller's incoming layout includes the register save / overflow 992 * area, which isn't a simple reusable parameter slab — don't realize. And if 993 * the call actually passes variadic (beyond-fixed) arguments, their stack 994 * footprint isn't captured by the fixed-parameter sizing above, so be 995 * conservative and fall back. A variadic callee invoked with only its fixed 996 * arguments (e.g. `musttail sink(x)`) is fine and uses the byte comparison. 997 */ 998 if (caller_va) return "variadic caller cannot host a sibling call"; 999 if (callee_va && call->nargs > callee_nparams) 1000 return "variadic tail call arguments are not realizable as a sibling call"; 1001 /* A backend may need tail-only stable storage in the reusable incoming 1002 * window in addition to the ABI argument slots themselves (x64 indirect / 1003 * byval payloads are one example). Once variadic-extra arguments have been 1004 * excluded above, synthesize the pure NativeCallDesc shape from the fixed 1005 * signature so call_stack_bytes can include that target-specific tail cost. 1006 */ 1007 if (o->native->call_stack_bytes) { 1008 NativeCallDesc d; 1009 NativeLoc* args = NULL; 1010 memset(&d, 0, sizeof d); 1011 if (call->nargs) 1012 args = arena_zarray(o->c->tu, NativeLoc, call->nargs); 1013 for (u32 i = 0; i < call->nargs; ++i) 1014 args[i].type = cg_type_func_param_id(o->c, call->fn_type, i); 1015 d.fn_type = call->fn_type; 1016 d.args = args; 1017 d.nargs = call->nargs; 1018 d.flags = CG_CALL_TAIL; 1019 outgoing = o->native->call_stack_bytes(o->native, &d); 1020 } 1021 if (outgoing > incoming) 1022 return "tail call stack arguments exceed the caller's parameter area"; 1023 return NULL; 1024 } 1025 1026 static int opt_on_asm_is_reg_constraint(void* user, const char* constraint) { 1027 OptImpl* o = (OptImpl*)user; 1028 return native_asm_constraint_is_reg(o ? o->native : NULL, constraint); 1029 } 1030 1031 CgTarget* opt_cgtarget_new(Compiler* c, CgTarget* target, int level) { 1032 if (!target) 1033 compiler_panic(c, (SrcLoc){0, 0, 0}, "opt_cgtarget_new: target is NULL"); 1034 if (level < 1) 1035 compiler_panic(c, (SrcLoc){0, 0, 0}, 1036 "opt_cgtarget_new: level %d out of range", level); 1037 OptImpl* o = arena_znew(c->tu, OptImpl); 1038 o->c = c; 1039 o->target = target; 1040 o->native = native_direct_target_native(target); 1041 o->level = level; 1042 /* Whenever the optimizer is engaged (-O1 and above) we run whole-program 1043 * optimization: deferred emission plus the module-wide reachability sweep and 1044 * cross-function inliner. The optimizer recorder only exists at level >= 1 1045 * (see kit_cg_begin), so this is effectively "on whenever optimizing". 1046 * -O0 uses the single-pass direct target and never reaches this code. */ 1047 o->whole_program = (level >= 1) ? 1 : 0; 1048 1049 CgIrRecorderConfig cfg; 1050 memset(&cfg, 0, sizeof cfg); 1051 cfg.func_recorded = opt_on_func; 1052 cfg.finalize = opt_on_finalize; 1053 cfg.destroy = opt_on_destroy; 1054 cfg.local_static_data_begin = opt_on_local_static_data_begin; 1055 cfg.tail_call_unrealizable_reason = opt_on_tail_call_unrealizable_reason; 1056 cfg.asm_is_reg_constraint = opt_on_asm_is_reg_constraint; 1057 cfg.user = o; 1058 return cg_ir_recorder_new(c, target->obj, &cfg); 1059 } 1060 1061 void opt_set_dump_writer(CgTarget* t, Writer* w) { 1062 CgIrRecorder* rec = cg_ir_recorder_from_target(t); 1063 OptImpl* o = rec ? (OptImpl*)cg_ir_recorder_user(rec) : NULL; 1064 if (o) o->dump_writer = w; 1065 } 1066 1067 void opt_set_finish_policy(CgTarget* t, const CgFinishPolicy* policy) { 1068 CgIrRecorder* rec = cg_ir_recorder_from_target(t); 1069 OptImpl* o = rec ? (OptImpl*)cg_ir_recorder_user(rec) : NULL; 1070 if (!o) return; 1071 memset(&o->finish_policy, 0, sizeof(o->finish_policy)); 1072 if (policy) o->finish_policy = *policy; 1073 }