cg_ir_lower.c (45249B)
1 #include <string.h> 2 3 #include "cg/ir.h" 4 #include "cg/type.h" 5 #include "opt/opt_internal.h" 6 7 #undef Operand 8 #undef CGParamDesc 9 #undef CGCallDesc 10 #undef CGFuncDesc 11 #undef CGLocalStorage 12 #undef FrameSlotDesc 13 14 typedef struct OptLocalMap { 15 OptCGLocalStorage storage; 16 NativeFrameSlot home_slot; 17 KitCgTypeId type; 18 u32 size; 19 u32 align; 20 u8 cls; 21 u8 address_taken; 22 u8 pad[2]; 23 } OptLocalMap; 24 25 /* Per-instruction record of pointer locals whose value was loaded from their 26 * frame home into a fresh PReg so they can serve as an indirect-addressing base 27 * (see frame_indirect_base_reg). Reset for each lowered instruction. */ 28 #define CG_IR_LOWER_MAX_MAT 8u 29 typedef struct CgIrLower { 30 Compiler* c; 31 const CgIrFunc* src; 32 Func* f; 33 OptLocalMap* locals; 34 u32 nlocals; 35 u8* local_addr_used; 36 const CgIrParam** param_by_local; 37 u32* label_block; 38 u32 nlabels; 39 u32* inst_block; 40 u32* fallthrough_by_block; 41 u8* leader; 42 CGLocal mat_local[CG_IR_LOWER_MAX_MAT]; 43 u8 mat_role[CG_IR_LOWER_MAX_MAT]; 44 Reg mat_reg[CG_IR_LOWER_MAX_MAT]; 45 u32 nmat; 46 } CgIrLower; 47 48 typedef enum CgIrMatRole { 49 CG_IR_MAT_BASE = 0, 50 CG_IR_MAT_INDEX = 1, 51 } CgIrMatRole; 52 53 static _Noreturn void lower_panic(CgIrLower* l, SrcLoc loc, const char* msg) { 54 compiler_panic(l->c, loc, "opt cg-ir lower: %s", msg); 55 } 56 57 static u8 local_reg_class(Compiler* c, KitCgTypeId ty) { 58 return opt_value_reg_class(c, ty); 59 } 60 61 static OptCGFuncDesc lower_func_desc(Arena* a, const struct CGFuncDesc* in) { 62 OptCGFuncDesc out; 63 memset(&out, 0, sizeof out); 64 if (!in) return out; 65 out.sym = in->sym; 66 out.text_section_id = in->text_section_id; 67 out.group_id = in->group_id; 68 out.fn_type = in->fn_type; 69 out.result_type = in->result_type; 70 out.nparams = in->nparams; 71 out.loc = in->loc; 72 out.flags = in->flags; 73 out.inline_policy = in->inline_policy; 74 out.atomize = in->atomize; 75 if (in->nparams && in->params) { 76 OptCGParamDesc* params = arena_zarray(a, OptCGParamDesc, in->nparams); 77 for (u32 i = 0; i < in->nparams; ++i) { 78 params[i].index = in->params[i].index; 79 params[i].name = in->params[i].name; 80 params[i].type = in->params[i].type; 81 params[i].size = in->params[i].size; 82 params[i].align = in->params[i].align; 83 params[i].flags = in->params[i].flags; 84 params[i].loc = in->params[i].loc; 85 } 86 out.params = params; 87 } 88 return out; 89 } 90 91 static NativeFrameSlotDesc local_slot_desc(const CgIrLocal* in, u8 kind) { 92 NativeFrameSlotDesc out; 93 memset(&out, 0, sizeof out); 94 out.type = in->desc.type; 95 out.name = in->desc.name; 96 out.loc = in->desc.loc; 97 out.size = in->desc.size; 98 out.align = in->desc.align; 99 out.kind = kind; 100 if (in->address_taken || (in->desc.flags & CG_LOCAL_ADDR_TAKEN)) 101 out.flags |= FSF_ADDR_TAKEN; 102 if (in->desc.flags & CG_LOCAL_MEMORY_REQUIRED) 103 out.flags |= FSF_MEMORY_REQUIRED; 104 return out; 105 } 106 107 static OptLocalMap* local_map(CgIrLower* l, CGLocal id, SrcLoc loc) { 108 if (id == CG_LOCAL_NONE || id > l->nlocals) 109 lower_panic(l, loc, "bad semantic local"); 110 return &l->locals[id - 1u]; 111 } 112 113 static int local_needs_home(const CgIrLocal* in) { 114 return in->address_taken || 115 (in->desc.flags & (CG_LOCAL_ADDR_TAKEN | CG_LOCAL_MEMORY_REQUIRED)); 116 } 117 118 /* AGG_COPY/AGG_SET take their dest/src as *pointer values* to the aggregate — 119 * the emitter derefs an OPK_LOCAL pointer operand via pointer_addr_from_operand 120 * (it loads the pointer; it does not address the local's own slot). So a 121 * pointer-typed local operand of an aggregate op uses the local's VALUE, not 122 * its address, and must not force the local to a frame home. Only a non-pointer 123 * operand (the aggregate-typed local itself) genuinely addresses its storage. 124 * (STORE/LOAD/ADDR_OF use addr_from_operand, where an OPK_LOCAL always 125 * addresses the slot, so they keep the direct-address marking below.) */ 126 static void note_local_addr_use(CgIrLower* l, const Operand* op) { 127 if (!op || op->kind != OPK_LOCAL) return; 128 CGLocal local = op->v.local; 129 if (local == CG_LOCAL_NONE || local > l->nlocals) return; 130 l->local_addr_used[local] = 1; 131 } 132 133 static void note_local_agg_addr_use(CgIrLower* l, const Operand* op) { 134 if (!op || op->kind != OPK_LOCAL || cg_type_is_ptr(l->c, op->type)) return; 135 CGLocal local = op->v.local; 136 if (local == CG_LOCAL_NONE || local > l->nlocals) return; 137 l->local_addr_used[local] = 1; 138 } 139 140 static void mark_local_address_uses(CgIrLower* l) { 141 const CgIrFunc* f = l->src; 142 l->local_addr_used = arena_zarray(l->f->arena, u8, l->nlocals + 1u); 143 for (u32 i = 0; i < f->ninsts; ++i) { 144 const CgIrInst* in = &f->insts[i]; 145 switch ((CgIrOp)in->op) { 146 case CG_IR_LOAD: 147 case CG_IR_BITFIELD_LOAD: 148 if (in->nopnds > 1u) note_local_addr_use(l, &in->opnds[1]); 149 break; 150 case CG_IR_STORE: 151 case CG_IR_BITFIELD_STORE: 152 if (in->nopnds > 0u) note_local_addr_use(l, &in->opnds[0]); 153 break; 154 case CG_IR_AGG_SET: 155 if (in->nopnds > 0u) note_local_agg_addr_use(l, &in->opnds[0]); 156 break; 157 case CG_IR_ADDR_OF: 158 if (in->nopnds > 1u) note_local_addr_use(l, &in->opnds[1]); 159 break; 160 case CG_IR_AGG_COPY: 161 if (in->nopnds > 0u) note_local_agg_addr_use(l, &in->opnds[0]); 162 if (in->nopnds > 1u) note_local_agg_addr_use(l, &in->opnds[1]); 163 break; 164 /* VA_START/VA_ARG/VA_END/VA_COPY consume a pointer *value* (the address 165 * of the va_list, produced by an earlier ADDR_OF); they do not take the 166 * address of their pointer operand, so they must not force it to a frame 167 * slot. */ 168 default: 169 break; 170 } 171 } 172 } 173 174 static void lower_locals(CgIrLower* l) { 175 l->nlocals = l->src->nlocals; 176 l->locals = 177 arena_zarray(l->f->arena, OptLocalMap, l->nlocals ? l->nlocals : 1u); 178 mark_local_address_uses(l); 179 for (u32 i = 0; i < l->src->nlocals; ++i) { 180 const CgIrLocal* in = &l->src->locals[i]; 181 OptLocalMap* m; 182 if (in->id == CG_LOCAL_NONE || in->id > l->src->nlocals) 183 lower_panic(l, in->desc.loc, "non-dense semantic local table"); 184 m = &l->locals[in->id - 1u]; 185 m->type = in->desc.type; 186 m->size = in->desc.size; 187 m->align = in->desc.align; 188 m->cls = local_reg_class(l->c, in->desc.type); 189 /* Aggregates and oversized scalars cannot live in a single PReg; they need 190 * a memory home regardless of whether their address is taken. "Oversized" 191 * is wider than the machine word (ptr_size): 8 on rv64/x64/aa64, 4 on rv32 192 * — so an 8-byte i64/double on rv32 is homed in memory like an i128 is on a 193 * 64-bit target (the cg layer also flags these CG_LOCAL_MEMORY_REQUIRED). 194 */ 195 m->address_taken = 196 local_needs_home(in) || 197 (in->id <= l->nlocals && l->local_addr_used[in->id]) || 198 cg_type_is_aggregate(l->c, in->desc.type) || 199 cg_type_size(l->c, in->desc.type) > (u64)l->c->target.ptr_size; 200 201 PReg r = ir_alloc_preg(l->f, in->desc.type, m->cls); 202 if (m->address_taken) { 203 m->storage.kind = CG_LOCAL_STORAGE_FRAME; 204 } else { 205 m->storage.kind = CG_LOCAL_STORAGE_REG; 206 m->storage.v.reg = (Reg)r; 207 } 208 209 if (m->address_taken) { 210 NativeFrameSlotDesc fsd = 211 local_slot_desc(in, in->is_param ? FS_PARAM : FS_LOCAL); 212 m->home_slot = ir_frame_slot_new(l->f, &fsd); 213 m->storage.v.frame_slot = m->home_slot; 214 } else { 215 m->home_slot = FRAME_SLOT_NONE; 216 } 217 (void)ir_local_add(l->f, &in->desc, m->storage); 218 l->f->locals[l->f->nlocals - 1u].address_taken = m->address_taken; 219 l->f->locals[l->f->nlocals - 1u].home_slot = m->home_slot; 220 } 221 } 222 223 static void build_param_by_local(CgIrLower* l) { 224 l->param_by_local = 225 arena_zarray(l->f->arena, const CgIrParam*, l->nlocals + 1u); 226 for (u32 i = 0; i < l->src->nparams; ++i) { 227 const CgIrParam* p = &l->src->params[i]; 228 if (p->local == CG_LOCAL_NONE || p->local > l->nlocals) continue; 229 l->param_by_local[p->local] = p; 230 } 231 } 232 233 static void lower_params(CgIrLower* l) { 234 /* Resolve the function-level ABI info once so we can attach per-param 235 * ABIArgInfo to each IRParam. Consumers (set_preg_pref_for_params, the 236 * native bind_param emit path) read p->abi without going through 237 * f->desc.abi, so this stays scoped to the param plumbing and does not 238 * activate the dormant f->desc.abi-gated passes (e.g. 239 * apply_param_incoming_register_hazards, opt_verify_alloc's incoming 240 * check), which have known issues with tail-call shuffles. */ 241 const ABIFuncInfo* fi = NULL; 242 if (l->c && l->c->abi && l->f->desc.fn_type) 243 fi = abi_cg_func_info(l->c->abi, l->f->desc.fn_type); 244 build_param_by_local(l); 245 for (u32 i = 0; i < l->src->nlocals; ++i) { 246 const CgIrLocal* loc = &l->src->locals[i]; 247 if (!loc->is_param) continue; 248 const CgIrParam* p = 249 (loc->id <= l->nlocals) ? l->param_by_local[loc->id] : NULL; 250 OptLocalMap* m = local_map(l, loc->id, loc->desc.loc); 251 OptCGParamDesc d; 252 memset(&d, 0, sizeof d); 253 if (p) { 254 d.index = p->desc.index; 255 d.name = p->desc.name; 256 d.type = p->desc.type; 257 d.size = p->desc.size; 258 d.align = p->desc.align; 259 d.flags = p->desc.flags; 260 d.loc = p->desc.loc; 261 } else { 262 d.index = loc->param_index; 263 d.name = loc->desc.name; 264 d.type = loc->desc.type; 265 d.size = loc->desc.size; 266 d.align = loc->desc.align; 267 d.flags = loc->desc.flags; 268 d.loc = loc->desc.loc; 269 } 270 d.storage = m->storage; 271 if (fi && d.index < fi->nparams) d.abi = &fi->params[d.index]; 272 ir_param_add(l->f, &d); 273 } 274 } 275 276 static int cg_inst_terminates(const CgIrInst* in) { 277 if (!in) return 0; 278 switch ((CgIrOp)in->op) { 279 case CG_IR_BR: 280 case CG_IR_RET: 281 case CG_IR_UNREACHABLE: 282 case CG_IR_CMP_BRANCH: 283 case CG_IR_SWITCH: 284 case CG_IR_INDIRECT_BRANCH: 285 case CG_IR_BREAK_TO: 286 case CG_IR_CONTINUE_TO: 287 return 1; 288 case CG_IR_INTRINSIC: { 289 const CgIrIntrinsicAux* aux = (const CgIrIntrinsicAux*)in->extra.aux; 290 return aux && (aux->kind == INTRIN_LONGJMP || aux->kind == INTRIN_TRAP); 291 } 292 default: 293 return 0; 294 } 295 } 296 297 static u32 label_id_max(const CgIrFunc* f) { 298 u32 max = 0; 299 for (u32 i = 0; i < f->nlabels; ++i) 300 if (f->labels[i].id > max) max = f->labels[i].id; 301 return max; 302 } 303 304 static void mark_label_leader(CgIrLower* l, Label label, const u32* place) { 305 if (label == LABEL_NONE || label > l->nlabels || place[label] == UINT32_MAX) 306 return; 307 l->leader[place[label]] = 1; 308 } 309 310 static void mark_leaders(CgIrLower* l, u32* label_place) { 311 const CgIrFunc* f = l->src; 312 for (u32 i = 0; i <= f->ninsts; ++i) l->leader[i] = 0; 313 if (f->ninsts) l->leader[0] = 1; 314 for (u32 i = 0; i < f->ninsts; ++i) { 315 const CgIrInst* in = &f->insts[i]; 316 if ((CgIrOp)in->op == CG_IR_LABEL) { 317 Label label = (Label)in->extra.imm; 318 l->leader[i] = 1; 319 if (label && label <= l->nlabels && label_place[label] == UINT32_MAX) 320 label_place[label] = i; 321 } 322 } 323 for (u32 i = 0; i < f->ninsts; ++i) { 324 const CgIrInst* in = &f->insts[i]; 325 if (cg_inst_terminates(in) && i + 1u < f->ninsts) l->leader[i + 1u] = 1; 326 switch ((CgIrOp)in->op) { 327 case CG_IR_BR: 328 case CG_IR_LOAD_LABEL_ADDR: 329 mark_label_leader(l, (Label)in->extra.imm, label_place); 330 break; 331 case CG_IR_CMP_BRANCH: { 332 CgIrCmpBranchAux* aux = (CgIrCmpBranchAux*)in->extra.aux; 333 if (i + 1u < f->ninsts) l->leader[i + 1u] = 1; 334 if (aux) mark_label_leader(l, aux->target, label_place); 335 break; 336 } 337 case CG_IR_SWITCH: { 338 CgIrSwitchAux* aux = (CgIrSwitchAux*)in->extra.aux; 339 if (i + 1u < f->ninsts) l->leader[i + 1u] = 1; 340 if (aux) { 341 mark_label_leader(l, aux->default_label, label_place); 342 for (u32 c = 0; c < aux->ncases; ++c) 343 mark_label_leader(l, aux->cases[c].label, label_place); 344 } 345 break; 346 } 347 case CG_IR_INDIRECT_BRANCH: { 348 CgIrIndirectAux* aux = (CgIrIndirectAux*)in->extra.aux; 349 if (aux) { 350 for (u32 t = 0; t < aux->ntargets; ++t) 351 mark_label_leader(l, aux->targets[t], label_place); 352 } 353 break; 354 } 355 case CG_IR_SCOPE_BEGIN: 356 if (i + 1u < f->ninsts) l->leader[i + 1u] = 1; 357 break; 358 case CG_IR_SCOPE_END: 359 l->leader[i] = 1; 360 if (i + 1u < f->ninsts) l->leader[i + 1u] = 1; 361 break; 362 default: 363 break; 364 } 365 } 366 } 367 368 static void make_blocks(CgIrLower* l, const u32* label_place) { 369 const CgIrFunc* f = l->src; 370 u32 cur = UINT32_MAX; 371 l->inst_block = arena_zarray(l->f->arena, u32, f->ninsts ? f->ninsts : 1u); 372 for (u32 i = 0; i < f->ninsts; ++i) { 373 if (l->leader[i] || cur == UINT32_MAX) { 374 cur = ir_block_new(l->f); 375 ir_note_emit(l->f, cur); 376 if (l->f->nblocks == 1u) l->f->entry = cur; 377 } 378 l->inst_block[i] = cur; 379 } 380 l->label_block = 381 arena_zarray(l->f->arena, u32, l->nlabels ? l->nlabels + 1u : 1u); 382 for (u32 i = 0; i <= l->nlabels; ++i) l->label_block[i] = UINT32_MAX; 383 for (u32 label = 1; label <= l->nlabels; ++label) { 384 if (label_place[label] != UINT32_MAX) { 385 u32 place = label_place[label]; 386 l->label_block[label] = (place + 1u < f->ninsts) 387 ? l->inst_block[place + 1u] 388 : l->inst_block[place]; 389 } else { 390 l->label_block[label] = ir_block_new(l->f); 391 } 392 } 393 if (!l->f->nblocks) { 394 l->f->entry = ir_block_new(l->f); 395 ir_note_emit(l->f, l->f->entry); 396 } 397 } 398 399 static void emit_param_decls(CgIrLower* l) { 400 if (!l->f->nparams || l->f->entry >= l->f->nblocks) return; 401 /* Emit the IR_PARAM_DECL phantom defs into a dedicated prologue block that 402 * falls through to the body, and make it the function entry. This keeps the 403 * parameter defs out of the body's first block, which matters when the body 404 * begins with a loop: that first block is then the loop header and the 405 * back-edge targets it. With the param_decls in the header, liveness reads 406 * each parameter as redefined every iteration (killing the liveness of an 407 * induction variable carried in a parameter register), and because the entry 408 * block's label is not placed by the emitter the back-edge resolves to a 409 * branch-to-self. Both miscompile loop-first functions at -O1. The prologue 410 * block emits no code (param_decls are markers, the fall-through is free, and 411 * the entry label is elided), so this is free in the common case. */ 412 u32 prologue = ir_block_new(l->f); 413 l->f->entry = prologue; 414 ir_note_emit(l->f, prologue); 415 for (u32 i = l->f->emit_order_n - 1u; i > 0; --i) 416 l->f->emit_order[i] = l->f->emit_order[i - 1u]; 417 l->f->emit_order[0] = prologue; 418 for (u32 i = 0; i < l->f->nparams; ++i) { 419 IRParam* p = &l->f->params[i]; 420 Inst* in = ir_emit(l->f, prologue, IR_PARAM_DECL); 421 IRParamDeclAux* aux = arena_znew(l->f->arena, IRParamDeclAux); 422 in->loc = p->loc; 423 in->type = p->type; 424 if (p->storage.kind == CG_LOCAL_STORAGE_REG) in->def = p->storage.v.reg; 425 memset(aux, 0, sizeof *aux); 426 aux->desc.index = p->index; 427 aux->desc.name = p->name; 428 aux->desc.type = p->type; 429 aux->desc.size = p->size; 430 aux->desc.align = p->align; 431 aux->desc.flags = p->flags; 432 aux->desc.loc = p->loc; 433 aux->desc.storage = p->storage; 434 aux->desc.abi = p->abi; 435 in->extra.aux = aux; 436 } 437 } 438 439 static u32 block_for_label(CgIrLower* l, Label label, SrcLoc loc) { 440 if (label == LABEL_NONE || label > l->nlabels || 441 l->label_block[label] == UINT32_MAX) 442 lower_panic(l, loc, "bad label"); 443 return l->label_block[label]; 444 } 445 446 static u32 fallthrough_block(CgIrLower* l, u32 inst_index) { 447 if (inst_index + 1u >= l->src->ninsts) return UINT32_MAX; 448 return l->inst_block[inst_index + 1u]; 449 } 450 451 static void set_succ1(CgIrLower* l, u32 block, u32 succ) { 452 if (succ == UINT32_MAX) { 453 l->f->blocks[block].nsucc = 0; 454 return; 455 } 456 l->f->blocks[block].succ[0] = succ; 457 l->f->blocks[block].nsucc = 1; 458 } 459 460 static OptOperand* dup_opt_ops(CgIrLower* l, const OptOperand* ops, u32 n) { 461 if (!n) return NULL; 462 OptOperand* out = arena_array(l->f->arena, OptOperand, n); 463 memcpy(out, ops, sizeof(*out) * n); 464 return out; 465 } 466 467 static OptOperand opt_reg_operand(OptLocalMap* m) { 468 OptOperand out; 469 memset(&out, 0, sizeof out); 470 out.kind = OPK_REG; 471 out.cls = m->cls; 472 out.type = m->type; 473 out.v.reg = m->storage.v.reg; 474 return out; 475 } 476 477 static OptOperand opt_frame_operand(OptLocalMap* m) { 478 OptOperand out; 479 memset(&out, 0, sizeof out); 480 out.kind = OPK_LOCAL; 481 out.cls = RC_INT; 482 out.type = m->type; 483 out.v.frame_slot = m->home_slot; 484 return out; 485 } 486 487 /* Base/index register for an OPK_INDIRECT whose base is a local. A REG-storage 488 * local supplies its value register directly. A FRAME-storage local (its 489 * address was taken, e.g. `int **q = &p; p->f = ...`) holds the pointer value 490 * in its frame home, so storage.v.reg is meaningless; load the home into a 491 * fresh PReg. prematerialize_indirect_bases emits that load before the using 492 * instruction; here we just look the result up (l->mat_*). */ 493 static Reg resolve_materialized_reg(CgIrLower* l, CGLocal local, 494 CgIrMatRole role, SrcLoc loc) { 495 OptLocalMap* m = local_map(l, local, loc); 496 if (m->storage.kind == CG_LOCAL_STORAGE_REG) return m->storage.v.reg; 497 for (u32 i = 0; i < l->nmat; ++i) 498 if (l->mat_local[i] == local && l->mat_role[i] == (u8)role) 499 return l->mat_reg[i]; 500 lower_panic(l, loc, 501 role == CG_IR_MAT_INDEX ? "indirect index local not materialized" 502 : "indirect base local not materialized"); 503 } 504 505 static KitCgTypeId pointer_sized_int_type(CgIrLower* l) { 506 return builtin_id(l->c->target.ptr_size <= 4u ? KIT_CG_BUILTIN_I32 507 : KIT_CG_BUILTIN_I64); 508 } 509 510 static void remember_materialized_reg(CgIrLower* l, CGLocal local, 511 CgIrMatRole role, Reg r, SrcLoc loc) { 512 if (l->nmat >= CG_IR_LOWER_MAX_MAT) 513 lower_panic(l, loc, "too many frame indirect operands in one instruction"); 514 l->mat_local[l->nmat] = local; 515 l->mat_role[l->nmat] = (u8)role; 516 l->mat_reg[l->nmat] = r; 517 l->nmat++; 518 } 519 520 static int materialized_reg_exists(CgIrLower* l, CGLocal local, 521 CgIrMatRole role) { 522 for (u32 i = 0; i < l->nmat; ++i) 523 if (l->mat_local[i] == local && l->mat_role[i] == (u8)role) return 1; 524 return 0; 525 } 526 527 static OptOperand opt_frame_operand_as(OptLocalMap* m, KitCgTypeId type) { 528 OptOperand out = opt_frame_operand(m); 529 out.type = type ? type : m->type; 530 return out; 531 } 532 533 /* Emit the pre-materialization needed for a FRAME-storage local used as an 534 * OPK_INDIRECT base. A pointer-typed local holds the base pointer value and is 535 * loaded. A non-pointer local names storage, so its frame address is the base. 536 */ 537 static void materialize_frame_base(CgIrLower* l, u32 block, CGLocal local, 538 SrcLoc loc) { 539 OptLocalMap* m = local_map(l, local, loc); 540 if (m->storage.kind == CG_LOCAL_STORAGE_REG) return; 541 if (materialized_reg_exists(l, local, CG_IR_MAT_BASE)) return; 542 KitCgTypeId base_type = 543 cg_type_is_ptr(l->c, m->type) ? m->type : cg_type_ptr_to(l->c, m->type); 544 if (!base_type) 545 lower_panic(l, loc, "cannot construct indirect frame-base pointer type"); 546 PReg r = ir_alloc_preg(l->f, base_type, RC_INT); 547 OptOperand ops[2]; 548 ops[1] = opt_frame_operand(m); 549 if (cg_type_is_ptr(l->c, m->type)) { 550 /* The local *holds* a pointer; load that value to use as the base. */ 551 Inst* ld = ir_emit(l->f, block, IR_LOAD); 552 ld->loc = loc; 553 memset(&ops[0], 0, sizeof ops[0]); 554 ops[0].kind = OPK_REG; 555 ops[0].cls = RC_INT; 556 ops[0].type = base_type; 557 ops[0].v.reg = (Reg)r; 558 ld->opnds = dup_opt_ops(l, ops, 2); 559 ld->nopnds = 2; 560 ld->def = (Val)r; 561 ld->type = m->type; 562 memset(&ld->extra.mem, 0, sizeof ld->extra.mem); 563 ld->extra.mem.type = m->type; 564 ld->extra.mem.size = m->size ? m->size : 8u; 565 ld->extra.mem.align = m->align ? m->align : 8u; 566 } else { 567 /* The local *is* the storage; its frame address is the base. */ 568 Inst* ao = ir_emit(l->f, block, IR_ADDR_OF); 569 ao->loc = loc; 570 memset(&ops[0], 0, sizeof ops[0]); 571 ops[0].kind = OPK_REG; 572 ops[0].cls = RC_INT; 573 ops[0].type = base_type; 574 ops[0].v.reg = (Reg)r; 575 ao->opnds = dup_opt_ops(l, ops, 2); 576 ao->nopnds = 2; 577 ao->def = (Val)r; 578 ao->type = base_type; 579 } 580 remember_materialized_reg(l, local, CG_IR_MAT_BASE, (Reg)r, loc); 581 } 582 583 /* Emit `r = load <local home>` for a FRAME-storage local used as an 584 * OPK_INDIRECT index. Unlike a non-pointer base, an index always needs the 585 * local's value. On rv32, Toy indexes are i64 and therefore memory-backed; the 586 * address calculation only consumes the pointer-width low word. */ 587 static void materialize_frame_index(CgIrLower* l, u32 block, CGLocal local, 588 SrcLoc loc) { 589 OptLocalMap* m = local_map(l, local, loc); 590 if (m->storage.kind == CG_LOCAL_STORAGE_REG) return; 591 if (materialized_reg_exists(l, local, CG_IR_MAT_INDEX)) return; 592 KitCgTypeId idx_ty = pointer_sized_int_type(l); 593 PReg r = ir_alloc_preg(l->f, idx_ty, RC_INT); 594 OptOperand ops[2]; 595 Inst* ld = ir_emit(l->f, block, IR_LOAD); 596 ld->loc = loc; 597 memset(&ops[0], 0, sizeof ops[0]); 598 ops[0].kind = OPK_REG; 599 ops[0].cls = RC_INT; 600 ops[0].type = idx_ty; 601 ops[0].v.reg = (Reg)r; 602 ops[1] = opt_frame_operand_as(m, idx_ty); 603 ld->opnds = dup_opt_ops(l, ops, 2); 604 ld->nopnds = 2; 605 ld->def = (Val)r; 606 ld->type = idx_ty; 607 memset(&ld->extra.mem, 0, sizeof ld->extra.mem); 608 ld->extra.mem.type = idx_ty; 609 ld->extra.mem.size = l->c->target.ptr_size; 610 ld->extra.mem.align = m->align && m->align < l->c->target.ptr_size 611 ? m->align 612 : l->c->target.ptr_size; 613 remember_materialized_reg(l, local, CG_IR_MAT_INDEX, (Reg)r, loc); 614 } 615 616 /* Scan the CG instruction's operands for OPK_INDIRECT bases/indices that are 617 * FRAME-storage locals and pre-load them (see materialize_frame_base). */ 618 static void prematerialize_indirect_bases(CgIrLower* l, const CgIrInst* in, 619 u32 block) { 620 l->nmat = 0; 621 for (u32 i = 0; i < in->nopnds; ++i) { 622 const Operand* op = &in->opnds[i]; 623 if (op->kind != OPK_INDIRECT) continue; 624 materialize_frame_base(l, block, op->v.ind.base, in->loc); 625 if (op->v.ind.index != CG_LOCAL_NONE) 626 materialize_frame_index(l, block, op->v.ind.index, in->loc); 627 } 628 } 629 630 static OptOperand lower_operand_value(CgIrLower* l, const Operand* in, 631 SrcLoc loc); 632 633 static OptOperand lower_operand_addr(CgIrLower* l, const Operand* in, 634 SrcLoc loc) { 635 OptOperand out; 636 memset(&out, 0, sizeof out); 637 if (!in) return out; 638 out.type = in->type; 639 switch ((OpKind)in->kind) { 640 case OPK_LOCAL: { 641 OptLocalMap* m = local_map(l, in->v.local, loc); 642 if (m->home_slot == FRAME_SLOT_NONE) { 643 const CgIrLocal* src = &l->src->locals[in->v.local - 1u]; 644 NativeFrameSlotDesc fsd = 645 local_slot_desc(src, src->is_param ? FS_PARAM : FS_LOCAL); 646 m->home_slot = ir_frame_slot_new(l->f, &fsd); 647 m->address_taken = 1; 648 if (in->v.local - 1u < l->f->nlocals) { 649 l->f->locals[in->v.local - 1u].address_taken = 1; 650 l->f->locals[in->v.local - 1u].home_slot = m->home_slot; 651 } 652 } 653 return opt_frame_operand(m); 654 } 655 case OPK_GLOBAL: 656 out.kind = OPK_GLOBAL; 657 out.cls = RC_INT; 658 out.v.global.sym = in->v.global.sym; 659 out.v.global.addend = in->v.global.addend; 660 return out; 661 case OPK_INDIRECT: { 662 out.kind = OPK_INDIRECT; 663 out.cls = RC_INT; 664 out.v.ind.base = 665 resolve_materialized_reg(l, in->v.ind.base, CG_IR_MAT_BASE, loc); 666 out.v.ind.base_type = opt_reg_type(l->f, (PReg)out.v.ind.base); 667 out.v.ind.index = REG_NONE; 668 if (in->v.ind.index != CG_LOCAL_NONE) { 669 out.v.ind.index = 670 resolve_materialized_reg(l, in->v.ind.index, CG_IR_MAT_INDEX, loc); 671 out.v.ind.index_type = opt_reg_type(l->f, (PReg)out.v.ind.index); 672 } 673 out.v.ind.log2_scale = in->v.ind.log2_scale; 674 out.v.ind.ofs = in->v.ind.ofs; 675 return out; 676 } 677 case OPK_IMM: 678 default: 679 lower_panic(l, loc, "operand is not addressable"); 680 } 681 } 682 683 static OptOperand lower_operand_value(CgIrLower* l, const Operand* in, 684 SrcLoc loc) { 685 OptOperand out; 686 memset(&out, 0, sizeof out); 687 if (!in) return out; 688 out.type = in->type; 689 switch ((OpKind)in->kind) { 690 case OPK_IMM: 691 out.kind = OPK_IMM; 692 out.cls = RC_INT; 693 out.v.imm = in->v.imm; 694 return out; 695 case OPK_LOCAL: { 696 OptLocalMap* m = local_map(l, in->v.local, loc); 697 return m->address_taken ? opt_frame_operand(m) : opt_reg_operand(m); 698 } 699 case OPK_GLOBAL: 700 out.kind = OPK_GLOBAL; 701 out.cls = RC_INT; 702 out.v.global.sym = in->v.global.sym; 703 out.v.global.addend = in->v.global.addend; 704 return out; 705 case OPK_INDIRECT: 706 return lower_operand_addr(l, in, loc); 707 default: 708 lower_panic(l, loc, "bad operand kind"); 709 } 710 } 711 712 static void set_inst_def(Inst* out, const OptOperand* op) { 713 if (op && op->kind == OPK_REG) { 714 out->def = (Val)op->v.reg; 715 out->type = op->type; 716 } 717 } 718 719 /* Lower `n` value operands. When `defs_first` is set, opnds[0] is the 720 * instruction's destination (def); otherwise all operands are uses. Branch 721 * terminators (CMP_BRANCH, SWITCH, INDIRECT_BRANCH) read their first operand 722 * and define nothing, so they must pass defs_first=0 -- otherwise dead-def 723 * elimination treats the branch as a redefinition of the tested value and 724 * removes the real producer. */ 725 static void lower_value_ops_ex(CgIrLower* l, Inst* out, const CgIrInst* in, 726 u32 n, int defs_first) { 727 OptOperand tmp[5]; 728 if (n > 5u) lower_panic(l, in->loc, "too many operands"); 729 for (u32 i = 0; i < n; ++i) 730 tmp[i] = lower_operand_value(l, &in->opnds[i], in->loc); 731 out->opnds = dup_opt_ops(l, tmp, n); 732 out->nopnds = n; 733 if (n && defs_first) set_inst_def(out, &out->opnds[0]); 734 } 735 736 static void lower_value_ops(CgIrLower* l, Inst* out, const CgIrInst* in, 737 u32 n) { 738 lower_value_ops_ex(l, out, in, n, 1); 739 } 740 741 static void lower_use_ops(CgIrLower* l, Inst* out, const CgIrInst* in, u32 n) { 742 lower_value_ops_ex(l, out, in, n, 0); 743 } 744 745 static void lower_addr_value_ops(CgIrLower* l, Inst* out, const CgIrInst* in, 746 u32 naddr, u32 nvalue) { 747 OptOperand tmp[5]; 748 u32 n = naddr + nvalue; 749 if (n > 5u) lower_panic(l, in->loc, "too many operands"); 750 for (u32 i = 0; i < naddr; ++i) 751 tmp[i] = lower_operand_addr(l, &in->opnds[i], in->loc); 752 for (u32 i = 0; i < nvalue; ++i) 753 tmp[naddr + i] = lower_operand_value(l, &in->opnds[naddr + i], in->loc); 754 out->opnds = dup_opt_ops(l, tmp, n); 755 out->nopnds = n; 756 } 757 758 static OptCGABIValue abi_value_for_local(CgIrLower* l, CGLocal local, 759 SrcLoc loc) { 760 OptCGABIValue out; 761 memset(&out, 0, sizeof out); 762 OptLocalMap* m = local_map(l, local, loc); 763 out.type = m->type; 764 out.storage = m->address_taken ? opt_frame_operand(m) : opt_reg_operand(m); 765 return out; 766 } 767 768 static void lower_call(CgIrLower* l, Inst* out, const CgIrInst* in) { 769 const CgIrCallAux* src = (const CgIrCallAux*)in->extra.aux; 770 IRCallAux* aux = arena_znew(l->f->arena, IRCallAux); 771 memset(aux, 0, sizeof *aux); 772 if (!src) { 773 out->extra.aux = aux; 774 return; 775 } 776 aux->desc.fn_type = src->desc.fn_type; 777 aux->desc.callee = lower_operand_value(l, &src->desc.callee, in->loc); 778 aux->desc.nargs = src->desc.nargs; 779 aux->desc.flags = src->desc.flags; 780 aux->desc.tail_policy = src->desc.tail_policy; 781 aux->desc.inline_policy = src->desc.inline_policy; 782 /* Cache the function ABI on the desc so downstream passes (e.g. the 783 * regalloc hint pass that steers call-arg sources toward their ABI dest 784 * register) don't have to re-derive it per call. abi_cg_func_info is the 785 * canonical lookup. */ 786 if (l->f->c && l->f->c->abi) 787 aux->desc.abi = abi_cg_func_info(l->f->c->abi, src->desc.fn_type); 788 if (src->desc.nargs) { 789 aux->desc.args = arena_zarray(l->f->arena, OptCGABIValue, src->desc.nargs); 790 for (u32 i = 0; i < src->desc.nargs; ++i) 791 aux->desc.args[i] = abi_value_for_local(l, src->desc.args[i], in->loc); 792 } 793 if (src->desc.result != CG_LOCAL_NONE) { 794 aux->desc.ret = abi_value_for_local(l, src->desc.result, in->loc); 795 set_inst_def(out, &aux->desc.ret.storage); 796 } 797 out->type = src->desc.fn_type; 798 out->extra.aux = aux; 799 } 800 801 static void lower_ret(CgIrLower* l, Inst* out, const CgIrInst* in) { 802 const CgIrRetAux* src = (const CgIrRetAux*)in->extra.aux; 803 IRRetAux* aux = arena_znew(l->f->arena, IRRetAux); 804 if (src && src->present) { 805 aux->present = 1; 806 aux->val = abi_value_for_local(l, src->value, in->loc); 807 } 808 out->extra.aux = aux; 809 } 810 811 static void lower_intrinsic(CgIrLower* l, Inst* out, const CgIrInst* in) { 812 const CgIrIntrinsicAux* src = (const CgIrIntrinsicAux*)in->extra.aux; 813 IRIntrinAux* aux = arena_znew(l->f->arena, IRIntrinAux); 814 if (src) { 815 aux->kind = src->kind; 816 aux->ndst = src->ndst; 817 aux->narg = src->narg; 818 aux->dsts = 819 src->ndst ? arena_array(l->f->arena, OptOperand, src->ndst) : NULL; 820 aux->args = 821 src->narg ? arena_array(l->f->arena, OptOperand, src->narg) : NULL; 822 for (u32 i = 0; i < src->ndst; ++i) 823 aux->dsts[i] = lower_operand_value(l, &src->dsts[i], in->loc); 824 for (u32 i = 0; i < src->narg; ++i) 825 aux->args[i] = lower_operand_value(l, &src->args[i], in->loc); 826 if (src->ndst) { 827 u32 ndefs = 0; 828 for (u32 i = 0; i < src->ndst; ++i) 829 if (aux->dsts[i].kind == OPK_REG) ++ndefs; 830 if (ndefs) { 831 u32 d = 0; 832 out->ndefs = ndefs; 833 out->defs = arena_array(l->f->arena, Val, ndefs); 834 for (u32 i = 0; i < src->ndst; ++i) 835 if (aux->dsts[i].kind == OPK_REG) out->defs[d++] = aux->dsts[i].v.reg; 836 out->def = out->defs[0]; 837 } 838 out->type = aux->dsts[0].type; 839 } 840 } 841 out->extra.aux = aux; 842 } 843 844 static void lower_asm(CgIrLower* l, Inst* out, const CgIrInst* in) { 845 const CgIrAsmAux* src = (const CgIrAsmAux*)in->extra.aux; 846 IRAsmAux* aux = arena_znew(l->f->arena, IRAsmAux); 847 if (src) { 848 aux->tmpl = src->tmpl; 849 aux->outs = src->outs; 850 aux->ins = src->ins; 851 aux->clobbers = src->clobbers; 852 aux->nout = src->nout; 853 aux->nin = src->nin; 854 aux->nclob = src->nclob; 855 aux->clobber_abi_sets = src->clobber_abi_sets; 856 aux->out_ops = 857 src->nout ? arena_array(l->f->arena, OptOperand, src->nout) : NULL; 858 aux->in_ops = 859 src->nin ? arena_array(l->f->arena, OptOperand, src->nin) : NULL; 860 for (u32 i = 0; i < src->nout; ++i) 861 aux->out_ops[i] = lower_operand_value(l, &src->out_ops[i], in->loc); 862 for (u32 i = 0; i < src->nin; ++i) 863 aux->in_ops[i] = lower_operand_value(l, &src->in_ops[i], in->loc); 864 if (src->nout) { 865 u32 ndefs = 0; 866 for (u32 i = 0; i < src->nout; ++i) 867 if (aux->out_ops[i].kind == OPK_REG) ++ndefs; 868 if (ndefs) { 869 u32 d = 0; 870 out->ndefs = ndefs; 871 out->defs = arena_array(l->f->arena, Val, ndefs); 872 for (u32 i = 0; i < src->nout; ++i) 873 if (aux->out_ops[i].kind == OPK_REG) 874 out->defs[d++] = aux->out_ops[i].v.reg; 875 out->def = out->defs[0]; 876 } 877 out->type = aux->out_ops[0].type; 878 } 879 } 880 out->extra.aux = aux; 881 } 882 883 static void lower_one_inst(CgIrLower* l, u32 idx) { 884 const CgIrInst* in = &l->src->insts[idx]; 885 u32 block = l->inst_block[idx]; 886 Inst* out = NULL; 887 IROp op = IR_NOP; 888 switch ((CgIrOp)in->op) { 889 case CG_IR_LABEL: 890 return; 891 case CG_IR_LOAD_IMM: 892 op = IR_LOAD_IMM; 893 break; 894 case CG_IR_LOAD_CONST: 895 op = IR_LOAD_CONST; 896 break; 897 case CG_IR_COPY: 898 op = IR_COPY; 899 break; 900 case CG_IR_LOAD: 901 op = IR_LOAD; 902 break; 903 case CG_IR_STORE: 904 op = IR_STORE; 905 break; 906 case CG_IR_ADDR_OF: 907 op = IR_ADDR_OF; 908 break; 909 case CG_IR_TLS_ADDR_OF: 910 op = IR_TLS_ADDR_OF; 911 break; 912 case CG_IR_AGG_COPY: 913 op = IR_AGG_COPY; 914 break; 915 case CG_IR_AGG_SET: 916 op = IR_AGG_SET; 917 break; 918 case CG_IR_BITFIELD_LOAD: 919 op = IR_BITFIELD_LOAD; 920 break; 921 case CG_IR_BITFIELD_STORE: 922 op = IR_BITFIELD_STORE; 923 break; 924 case CG_IR_BINOP: 925 op = IR_BINOP; 926 break; 927 case CG_IR_UNOP: 928 op = IR_UNOP; 929 break; 930 case CG_IR_CMP: 931 op = IR_CMP; 932 break; 933 case CG_IR_CONVERT: 934 op = IR_CONVERT; 935 break; 936 case CG_IR_CALL: 937 op = IR_CALL; 938 break; 939 case CG_IR_RET: 940 op = IR_RET; 941 break; 942 case CG_IR_UNREACHABLE: 943 op = IR_UNREACHABLE; 944 break; 945 case CG_IR_BR: 946 op = IR_BR; 947 break; 948 case CG_IR_CMP_BRANCH: 949 op = IR_CMP_BRANCH; 950 break; 951 case CG_IR_SWITCH: 952 op = IR_SWITCH; 953 break; 954 case CG_IR_INDIRECT_BRANCH: 955 op = IR_INDIRECT_BRANCH; 956 break; 957 case CG_IR_LOAD_LABEL_ADDR: 958 op = IR_LOAD_LABEL_ADDR; 959 break; 960 case CG_IR_LOCAL_STATIC_DATA_BEGIN: 961 op = IR_LOCAL_STATIC_DATA_BEGIN; 962 break; 963 case CG_IR_LOCAL_STATIC_DATA_WRITE: 964 op = IR_LOCAL_STATIC_DATA_WRITE; 965 break; 966 case CG_IR_LOCAL_STATIC_DATA_LABEL_ADDR: 967 op = IR_LOCAL_STATIC_DATA_LABEL_ADDR; 968 break; 969 case CG_IR_LOCAL_STATIC_DATA_END: 970 op = IR_LOCAL_STATIC_DATA_END; 971 break; 972 case CG_IR_SCOPE_BEGIN: 973 op = IR_SCOPE_BEGIN; 974 break; 975 case CG_IR_SCOPE_END: 976 op = IR_SCOPE_END; 977 break; 978 case CG_IR_BREAK_TO: 979 op = IR_BREAK_TO; 980 break; 981 case CG_IR_CONTINUE_TO: 982 op = IR_CONTINUE_TO; 983 break; 984 case CG_IR_ALLOCA: 985 op = IR_ALLOCA; 986 break; 987 case CG_IR_VA_START: 988 op = IR_VA_START; 989 break; 990 case CG_IR_VA_ARG: 991 op = IR_VA_ARG; 992 break; 993 case CG_IR_VA_END: 994 op = IR_VA_END; 995 break; 996 case CG_IR_VA_COPY: 997 op = IR_VA_COPY; 998 break; 999 case CG_IR_ATOMIC_LOAD: 1000 op = IR_ATOMIC_LOAD; 1001 break; 1002 case CG_IR_ATOMIC_STORE: 1003 op = IR_ATOMIC_STORE; 1004 break; 1005 case CG_IR_ATOMIC_RMW: 1006 op = IR_ATOMIC_RMW; 1007 break; 1008 case CG_IR_ATOMIC_CAS: 1009 op = IR_ATOMIC_CAS; 1010 break; 1011 case CG_IR_FENCE: 1012 op = IR_FENCE; 1013 break; 1014 case CG_IR_INTRINSIC: 1015 op = IR_INTRINSIC; 1016 break; 1017 case CG_IR_ASM_BLOCK: 1018 op = IR_ASM_BLOCK; 1019 break; 1020 default: 1021 op = IR_NOP; 1022 break; 1023 } 1024 /* Pre-load any FRAME-resident pointer locals used as indirect bases so the 1025 * load dominates this instruction (which is emitted next). */ 1026 prematerialize_indirect_bases(l, in, block); 1027 out = ir_emit(l->f, block, op); 1028 out->loc = in->loc; 1029 switch ((CgIrOp)in->op) { 1030 case CG_IR_LOAD_IMM: 1031 lower_value_ops(l, out, in, 1); 1032 out->extra.imm = in->extra.imm; 1033 break; 1034 case CG_IR_LOAD_CONST: 1035 lower_value_ops(l, out, in, 1); 1036 out->extra.cbytes = in->extra.cbytes; 1037 break; 1038 case CG_IR_COPY: 1039 case CG_IR_BINOP: 1040 case CG_IR_UNOP: 1041 case CG_IR_CMP: 1042 case CG_IR_CONVERT: 1043 case CG_IR_ALLOCA: 1044 case CG_IR_VA_ARG: 1045 lower_value_ops(l, out, in, in->nopnds); 1046 out->extra.imm = in->extra.imm; 1047 break; 1048 case CG_IR_LOAD: 1049 case CG_IR_BITFIELD_LOAD: { 1050 OptOperand ops[2]; 1051 ops[0] = lower_operand_value(l, &in->opnds[0], in->loc); 1052 ops[1] = lower_operand_addr(l, &in->opnds[1], in->loc); 1053 out->opnds = dup_opt_ops(l, ops, 2); 1054 out->nopnds = 2; 1055 set_inst_def(out, &out->opnds[0]); 1056 if ((CgIrOp)in->op == CG_IR_LOAD) 1057 out->extra.mem = in->extra.mem; 1058 else 1059 out->extra.aux = in->extra.aux; 1060 break; 1061 } 1062 case CG_IR_ATOMIC_LOAD: { 1063 OptOperand ops[2]; 1064 ops[0] = lower_operand_value(l, &in->opnds[0], in->loc); 1065 ops[1] = lower_operand_value(l, &in->opnds[1], in->loc); 1066 out->opnds = dup_opt_ops(l, ops, 2); 1067 out->nopnds = 2; 1068 set_inst_def(out, &out->opnds[0]); 1069 out->extra.aux = in->extra.aux; 1070 break; 1071 } 1072 case CG_IR_STORE: 1073 case CG_IR_BITFIELD_STORE: 1074 lower_addr_value_ops(l, out, in, 1, in->nopnds - 1u); 1075 if ((CgIrOp)in->op == CG_IR_STORE) 1076 out->extra.mem = in->extra.mem; 1077 else 1078 out->extra.aux = in->extra.aux; 1079 break; 1080 case CG_IR_AGG_COPY: 1081 case CG_IR_AGG_SET: 1082 /* Aggregate ops take their operands as pointer *values* to the aggregates 1083 * (the emitter derefs them via pointer_addr_from_operand). Lowering them 1084 * as values keeps a pointer local in its register instead of forcing a 1085 * frame home — the home would otherwise break the local's other uses as 1086 * an indirect base, whose lowering reads storage.v.reg. */ 1087 lower_use_ops(l, out, in, in->nopnds); 1088 out->extra.aux = in->extra.aux; 1089 break; 1090 case CG_IR_ATOMIC_STORE: { 1091 OptOperand ops[2]; 1092 ops[0] = lower_operand_value(l, &in->opnds[0], in->loc); 1093 ops[1] = lower_operand_value(l, &in->opnds[1], in->loc); 1094 out->opnds = dup_opt_ops(l, ops, 2); 1095 out->nopnds = 2; 1096 out->extra.aux = in->extra.aux; 1097 break; 1098 } 1099 case CG_IR_ADDR_OF: { 1100 OptOperand ops[2]; 1101 ops[0] = lower_operand_value(l, &in->opnds[0], in->loc); 1102 ops[1] = lower_operand_addr(l, &in->opnds[1], in->loc); 1103 out->opnds = dup_opt_ops(l, ops, 2); 1104 out->nopnds = 2; 1105 set_inst_def(out, &out->opnds[0]); 1106 break; 1107 } 1108 case CG_IR_TLS_ADDR_OF: 1109 lower_value_ops(l, out, in, 1); 1110 out->extra.aux = in->extra.aux; 1111 break; 1112 case CG_IR_CALL: 1113 lower_call(l, out, in); 1114 break; 1115 case CG_IR_RET: 1116 lower_ret(l, out, in); 1117 l->f->blocks[block].nsucc = 0; 1118 break; 1119 case CG_IR_UNREACHABLE: 1120 /* Terminator with no successors: control does not leave this block. */ 1121 l->f->blocks[block].nsucc = 0; 1122 break; 1123 case CG_IR_BR: 1124 out->extra.imm = block_for_label(l, (Label)in->extra.imm, in->loc); 1125 set_succ1(l, block, (u32)out->extra.imm); 1126 break; 1127 case CG_IR_CMP_BRANCH: { 1128 CgIrCmpBranchAux* aux = (CgIrCmpBranchAux*)in->extra.aux; 1129 lower_use_ops(l, out, in, 2); 1130 out->extra.imm = aux ? aux->op : CMP_NE; 1131 ir_block_set_nsucc(l->f, block, 2); 1132 l->f->blocks[block].succ[0] = 1133 aux ? block_for_label(l, aux->target, in->loc) : UINT32_MAX; 1134 l->f->blocks[block].succ[1] = fallthrough_block(l, idx); 1135 break; 1136 } 1137 case CG_IR_SWITCH: { 1138 CgIrSwitchAux* src = (CgIrSwitchAux*)in->extra.aux; 1139 IRSwitchAux* aux = arena_znew(l->f->arena, IRSwitchAux); 1140 lower_use_ops(l, out, in, 1); 1141 if (src) { 1142 aux->selector_type = src->selector_type; 1143 aux->ncases = src->ncases; 1144 aux->hint = src->hint; 1145 aux->has_default = src->default_label != LABEL_NONE; 1146 aux->default_block = 1147 aux->has_default ? block_for_label(l, src->default_label, in->loc) 1148 : fallthrough_block(l, idx); 1149 if (src->ncases) { 1150 aux->cases = arena_array(l->f->arena, IRSwitchAuxCase, src->ncases); 1151 for (u32 i = 0; i < src->ncases; ++i) { 1152 aux->cases[i].value = src->cases[i].value; 1153 aux->cases[i].block = 1154 block_for_label(l, src->cases[i].label, in->loc); 1155 } 1156 } 1157 ir_block_set_nsucc(l->f, block, src->ncases + 1u); 1158 for (u32 i = 0; i < src->ncases; ++i) 1159 l->f->blocks[block].succ[i] = aux->cases[i].block; 1160 l->f->blocks[block].succ[src->ncases] = aux->default_block; 1161 } 1162 out->extra.aux = aux; 1163 break; 1164 } 1165 case CG_IR_INDIRECT_BRANCH: { 1166 CgIrIndirectAux* src = (CgIrIndirectAux*)in->extra.aux; 1167 IRIndirectAux* aux = arena_znew(l->f->arena, IRIndirectAux); 1168 lower_use_ops(l, out, in, 1); 1169 if (src && src->ntargets) { 1170 aux->ntargets = src->ntargets; 1171 aux->targets = arena_array(l->f->arena, u32, src->ntargets); 1172 ir_block_set_nsucc(l->f, block, src->ntargets); 1173 for (u32 i = 0; i < src->ntargets; ++i) { 1174 aux->targets[i] = block_for_label(l, src->targets[i], in->loc); 1175 l->f->blocks[block].succ[i] = aux->targets[i]; 1176 } 1177 } 1178 out->extra.aux = aux; 1179 break; 1180 } 1181 case CG_IR_LOAD_LABEL_ADDR: 1182 lower_value_ops(l, out, in, 1); 1183 out->extra.imm = block_for_label(l, (Label)in->extra.imm, in->loc); 1184 break; 1185 case CG_IR_LOCAL_STATIC_DATA_BEGIN: 1186 out->extra.aux = in->extra.aux; 1187 break; 1188 case CG_IR_LOCAL_STATIC_DATA_WRITE: 1189 out->extra.aux = in->extra.aux; 1190 break; 1191 case CG_IR_LOCAL_STATIC_DATA_LABEL_ADDR: { 1192 CgIrLocalStaticLabelAux* src = (CgIrLocalStaticLabelAux*)in->extra.aux; 1193 CgIrLocalStaticLabelAux* aux = 1194 arena_znew(l->f->arena, CgIrLocalStaticLabelAux); 1195 if (src) { 1196 *aux = *src; 1197 aux->target = (Label)block_for_label(l, src->target, in->loc); 1198 } 1199 out->extra.aux = aux; 1200 break; 1201 } 1202 case CG_IR_LOCAL_STATIC_DATA_END: 1203 break; 1204 case CG_IR_SCOPE_BEGIN: { 1205 CgIrScopeAux* src = (CgIrScopeAux*)in->extra.aux; 1206 IRScopeAux* aux = arena_znew(l->f->arena, IRScopeAux); 1207 if (src) { 1208 aux->scope_id = src->scope; 1209 aux->desc.kind = src->desc.kind; 1210 aux->desc.break_label = src->desc.break_label; 1211 aux->desc.continue_label = src->desc.continue_label; 1212 aux->desc.result_type = src->desc.result_type; 1213 } 1214 out->extra.aux = aux; 1215 break; 1216 } 1217 case CG_IR_SCOPE_END: 1218 case CG_IR_BREAK_TO: 1219 case CG_IR_CONTINUE_TO: 1220 out->extra.imm = in->extra.imm; 1221 break; 1222 case CG_IR_VA_START: 1223 case CG_IR_VA_END: 1224 /* The operand is a pointer value (the address of the va_list object), 1225 * produced by an earlier ADDR_OF. Lower as a value so it can live in a 1226 * register; the backend va hook consumes the pointer. */ 1227 lower_use_ops(l, out, in, 1); 1228 break; 1229 case CG_IR_VA_COPY: 1230 lower_use_ops(l, out, in, 2); 1231 break; 1232 case CG_IR_ATOMIC_RMW: 1233 lower_value_ops(l, out, in, 3); 1234 out->extra.aux = in->extra.aux; 1235 break; 1236 case CG_IR_ATOMIC_CAS: 1237 lower_value_ops(l, out, in, 5); 1238 out->ndefs = 2; 1239 out->defs = arena_array(l->f->arena, Val, 2); 1240 out->defs[0] = out->opnds[0].v.reg; 1241 out->defs[1] = out->opnds[1].v.reg; 1242 out->def = out->defs[0]; 1243 out->type = out->opnds[0].type; 1244 { 1245 const CgIrAtomicAux* src = (const CgIrAtomicAux*)in->extra.aux; 1246 IRCasAux* aux = arena_znew(l->f->arena, IRCasAux); 1247 if (src) { 1248 aux->mem = src->mem; 1249 aux->success = src->order; 1250 aux->failure = src->failure; 1251 } 1252 out->extra.aux = aux; 1253 } 1254 break; 1255 case CG_IR_FENCE: 1256 out->extra.imm = in->extra.imm; 1257 break; 1258 case CG_IR_INTRINSIC: 1259 lower_intrinsic(l, out, in); 1260 break; 1261 case CG_IR_ASM_BLOCK: 1262 lower_asm(l, out, in); 1263 break; 1264 default: 1265 out->extra.aux = in->extra.aux; 1266 break; 1267 } 1268 } 1269 1270 static void add_fallthrough_succs(CgIrLower* l) { 1271 l->fallthrough_by_block = 1272 arena_array(l->f->arena, u32, l->f->nblocks ? l->f->nblocks : 1u); 1273 for (u32 b = 0; b < l->f->nblocks; ++b) l->fallthrough_by_block[b] = UINT32_MAX; 1274 for (u32 i = 0; i + 1u < l->f->emit_order_n; ++i) { 1275 u32 b = l->f->emit_order[i]; 1276 if (b < l->f->nblocks) l->fallthrough_by_block[b] = l->f->emit_order[i + 1u]; 1277 } 1278 for (u32 b = 0; b < l->f->nblocks; ++b) { 1279 Block* bl = &l->f->blocks[b]; 1280 if (bl->nsucc) continue; 1281 if (bl->ninsts) { 1282 Inst* last = &bl->insts[bl->ninsts - 1u]; 1283 switch ((IROp)last->op) { 1284 case IR_BR: 1285 case IR_CONDBR: 1286 case IR_CMP_BRANCH: 1287 case IR_SWITCH: 1288 case IR_INDIRECT_BRANCH: 1289 case IR_RET: 1290 case IR_UNREACHABLE: 1291 case IR_BREAK_TO: 1292 case IR_CONTINUE_TO: 1293 continue; 1294 case IR_INTRINSIC: { 1295 IRIntrinAux* aux = (IRIntrinAux*)last->extra.aux; 1296 if (aux && (aux->kind == INTRIN_LONGJMP || aux->kind == INTRIN_TRAP)) 1297 continue; 1298 break; 1299 } 1300 default: 1301 break; 1302 } 1303 } 1304 set_succ1(l, b, l->fallthrough_by_block[b]); 1305 } 1306 } 1307 1308 Func* opt_func_from_cg_ir(Compiler* c, const CgIrFunc* src) { 1309 if (!c || !src) return NULL; 1310 OptCGFuncDesc desc = lower_func_desc(c->tu, &src->desc); 1311 Func* f = ir_func_new(c, &desc); 1312 CgIrLower l; 1313 memset(&l, 0, sizeof l); 1314 l.c = c; 1315 l.src = src; 1316 l.f = f; 1317 l.nlabels = label_id_max(src); 1318 u32* label_place = 1319 arena_array(f->arena, u32, l.nlabels ? l.nlabels + 1u : 1u); 1320 for (u32 i = 0; i <= l.nlabels; ++i) label_place[i] = UINT32_MAX; 1321 l.leader = arena_zarray(f->arena, u8, src->ninsts + 1u); 1322 lower_locals(&l); 1323 lower_params(&l); 1324 mark_leaders(&l, label_place); 1325 make_blocks(&l, label_place); 1326 emit_param_decls(&l); 1327 for (u32 i = 0; i < src->ninsts; ++i) lower_one_inst(&l, i); 1328 add_fallthrough_succs(&l); 1329 opt_build_cfg(f); 1330 return f; 1331 }