pass_machinize.c (11731B)
1 #include <string.h> 2 3 #include "cg/native_asm.h" 4 #include "cg/type.h" 5 #include "core/pool.h" 6 #include "core/slice.h" 7 #include "opt/opt_internal.h" 8 9 static int native_resolve_reg(NativeTarget* target, Slice name, Reg* out, 10 RegClass* cls_out) { 11 NativeAllocClass cls; 12 if (!target || !target->regs || !target->regs->resolve_name) return 1; 13 if (target->regs->resolve_name(target->regs, name, out, &cls) != 0) return 1; 14 if (cls_out) *cls_out = (RegClass)cls; 15 return 0; 16 } 17 18 static void asm_prepare_reg_requirement(Func* f, NativeTarget* target, 19 const AsmConstraint* constraint, 20 IRAsmRegRequirement* req) { 21 NativeAsmRegPin pin; 22 NativeAsmRegPinStatus status; 23 NativeAsmConstraintInfo info; 24 memset(req, 0, sizeof *req); 25 req->fixed_reg = -1; 26 27 status = native_asm_resolve_pin(target, constraint->reg, constraint->str, 28 &pin); 29 if (status == NATIVE_ASM_REG_PIN_OK) { 30 req->present = 1; 31 req->cls = (u8)pin.cls; 32 req->fixed_reg = (i32)pin.reg; 33 return; 34 } 35 if (status != NATIVE_ASM_REG_PIN_ABSENT) 36 compiler_panic(f->c, (SrcLoc){0, 0, 0}, "opt asm: %s", 37 native_asm_pin_status_message(status)); 38 39 memset(&info, 0, sizeof info); 40 if (!native_asm_constraint_reg_info(target, constraint->str, &info)) return; 41 req->present = 1; 42 req->cls = (u8)info.cls; 43 req->allowed_mask = info.allowed_mask; 44 if (info.fixed_reg != REG_NONE) req->fixed_reg = (i32)info.fixed_reg; 45 } 46 47 static void asm_prepare_constraints(Func* f, NativeTarget* target, 48 IRAsmAux* aux) { 49 if (!aux) return; 50 for (u32 c = 0; c < OPT_REG_CLASSES; ++c) aux->clobber_mask[c] = 0; 51 aux->has_memory_constraint = 0; 52 if (aux->nout && !aux->out_reg_reqs) 53 aux->out_reg_reqs = 54 arena_zarray(f->arena, IRAsmRegRequirement, aux->nout); 55 if (aux->nin && !aux->in_reg_reqs) 56 aux->in_reg_reqs = 57 arena_zarray(f->arena, IRAsmRegRequirement, aux->nin); 58 for (u32 i = 0; i < aux->nclob; ++i) { 59 Reg r; 60 RegClass cls; 61 Slice nm = pool_slice(f->c->global, aux->clobbers[i]); 62 if (native_resolve_reg(target, nm, &r, &cls) != 0) continue; 63 if ((u32)cls < OPT_REG_CLASSES && r < OPT_MAX_HARD_REGS) 64 aux->clobber_mask[cls] |= 1u << r; 65 } 66 if (aux->clobber_abi_sets) { 67 u32 int_mask, fp_mask; 68 native_asm_abi_clobber_masks(target, aux->clobber_abi_sets, &int_mask, 69 &fp_mask); 70 if (NATIVE_REG_INT < OPT_REG_CLASSES) 71 aux->clobber_mask[NATIVE_REG_INT] |= int_mask; 72 if (NATIVE_REG_FP < OPT_REG_CLASSES) 73 aux->clobber_mask[NATIVE_REG_FP] |= fp_mask; 74 } 75 for (u32 i = 0; i < aux->nout; ++i) { 76 asm_prepare_reg_requirement(f, target, &aux->outs[i], 77 &aux->out_reg_reqs[i]); 78 if (native_asm_constraint_body(aux->outs[i].str)[0] == 'm') 79 aux->has_memory_constraint = 1; 80 } 81 for (u32 i = 0; i < aux->nin; ++i) { 82 asm_prepare_reg_requirement(f, target, &aux->ins[i], 83 &aux->in_reg_reqs[i]); 84 if (native_asm_constraint_body(aux->ins[i].str)[0] == 'm') 85 aux->has_memory_constraint = 1; 86 } 87 } 88 89 static void machinize_reset(Func* f, NativeTarget* target) { 90 f->opt_target = target->c->target; 91 f->opt_has_target = 1; 92 for (u32 c = 0; c < OPT_REG_CLASSES; ++c) { 93 f->opt_hard_reg_count[c] = 0; 94 f->opt_phys_reg_count[c] = 0; 95 f->emit_temp_reg_count[c] = 0; 96 f->asm_temp_mask[c] = 0; 97 f->opt_caller_saved[c] = 0; 98 f->opt_callee_saved[c] = 0; 99 f->opt_reserved_regs[c] = 0; 100 f->opt_arg_regs[c] = 0; 101 f->opt_ret_regs[c] = 0; 102 } 103 } 104 105 static void machinize_prepare_insts(Func* f, NativeTarget* target) { 106 for (u32 b = 0; b < f->nblocks; ++b) { 107 Block* bl = &f->blocks[b]; 108 for (u32 i = 0; i < bl->ninsts; ++i) { 109 Inst* in = &bl->insts[i]; 110 if ((IROp)in->op == IR_ASM_BLOCK) 111 asm_prepare_constraints(f, target, (IRAsmAux*)in->extra.aux); 112 } 113 } 114 } 115 116 static void collect_class(Func* f, NativeTarget* target, 117 const NativeAllocClassInfo* ci) { 118 u32 cls = ci->cls; 119 u32 emit_mask = 0; 120 u32 asm_mask = 0; 121 if (cls >= OPT_REG_CLASSES) 122 compiler_panic(f->c, (SrcLoc){0, 0, 0}, 123 "opt_machinize: invalid register class %u", (unsigned)cls); 124 if (ci->nemit_temps > OPT_MAX_EMIT_TEMPS) 125 compiler_panic(f->c, (SrcLoc){0, 0, 0}, 126 "opt_machinize: class %u needs %u emit temps (capacity %u)", 127 (unsigned)cls, (unsigned)ci->nemit_temps, 128 (unsigned)OPT_MAX_EMIT_TEMPS); 129 for (u32 i = 0; i < ci->nemit_temps; ++i) 130 emit_mask |= 1u << ci->emit_temps[i]; 131 for (u32 i = 0; i < ci->nasm_temps; ++i) 132 asm_mask |= 1u << ci->asm_temps[i]; 133 f->opt_caller_saved[cls] = 134 native_target_caller_saved_mask(target, (NativeAllocClass)cls); 135 f->opt_callee_saved[cls] = 136 native_target_callee_saved_mask(target, (NativeAllocClass)cls); 137 if (emit_mask & ~f->opt_caller_saved[cls]) 138 compiler_panic(f->c, (SrcLoc){0, 0, 0}, 139 "opt_machinize: class %u emit temp bank is not entirely " 140 "caller-saved", 141 (unsigned)cls); 142 /* Static reservations come from register roles. The emitter temp bank is 143 * separately policy-owned, so reserve it for this function here too. */ 144 f->opt_reserved_regs[cls] = 145 native_target_reserved_mask(target, (NativeAllocClass)cls) | emit_mask; 146 f->asm_temp_mask[cls] = asm_mask; 147 f->opt_arg_regs[cls] = native_target_arg_mask(target, (NativeAllocClass)cls); 148 f->opt_ret_regs[cls] = native_target_ret_mask(target, (NativeAllocClass)cls); 149 for (u32 i = 0; i < ci->nphys; ++i) { 150 const NativePhysRegInfo* src = &ci->phys[i]; 151 CGPhysRegInfo* dst = &f->opt_phys_regs[cls][f->opt_phys_reg_count[cls]++]; 152 memset(dst, 0, sizeof *dst); 153 dst->reg = src->reg; 154 dst->cls = src->cls; 155 dst->abi_index = src->abi_index; 156 dst->flags = src->flags; 157 if ((src->flags & CG_REG_ALLOCABLE) && !(src->flags & CG_REG_RESERVED) && 158 f->opt_hard_reg_count[cls] < OPT_MAX_HARD_REGS) 159 f->opt_hard_regs[cls][f->opt_hard_reg_count[cls]++] = src->reg; 160 } 161 for (u32 i = 0; i < ci->nemit_temps && i < OPT_MAX_EMIT_TEMPS; ++i) 162 f->emit_temp_regs[cls][f->emit_temp_reg_count[cls]++] = 163 ci->emit_temps[i]; 164 } 165 166 static void machinize_collect_regs(Func* f, NativeTarget* target) { 167 u32 seen = 0; 168 if (!target || !target->regs) return; 169 native_reg_info_validate(f->c, target->regs); 170 for (u32 i = 0; i < target->regs->nclasses; ++i) { 171 u32 cls = target->regs->classes[i].cls; 172 if (cls >= OPT_REG_CLASSES || (seen & (1u << cls))) 173 compiler_panic(f->c, (SrcLoc){0, 0, 0}, 174 "opt_machinize: duplicate/invalid register class %u", 175 (unsigned)cls); 176 seen |= 1u << cls; 177 collect_class(f, target, &target->regs->classes[i]); 178 } 179 } 180 181 static void machinize_check_overlap(Func* f) { 182 for (u32 c = 0; c < OPT_REG_CLASSES; ++c) { 183 for (u32 i = 0; i < f->opt_hard_reg_count[c]; ++i) { 184 Reg hr = f->opt_hard_regs[c][i]; 185 for (u32 s = 0; s < f->emit_temp_reg_count[c]; ++s) { 186 if (f->emit_temp_regs[c][s] == hr) { 187 compiler_panic(f->c, (SrcLoc){0, 0, 0}, 188 "opt_machinize: hard reg %u overlaps emit temp " 189 "in class %u", 190 (unsigned)hr, (unsigned)c); 191 } 192 } 193 } 194 } 195 } 196 197 static u32 machinize_known_phys_mask(const Func* f, u32 cls) { 198 u32 mask = 0; 199 if (!f || cls >= OPT_REG_CLASSES) return 0; 200 for (u32 i = 0; i < f->opt_phys_reg_count[cls]; ++i) { 201 Reg reg = f->opt_phys_regs[cls][i].reg; 202 if (reg < OPT_MAX_HARD_REGS) mask |= 1u << reg; 203 } 204 return mask; 205 } 206 207 static void machinize_validate_clobber_masks(Func* f, const Inst* in, 208 const u32* masks) { 209 for (u32 cls = 0; cls < OPT_REG_CLASSES; ++cls) { 210 u32 unknown = masks[cls] & ~machinize_known_phys_mask(f, cls); 211 if (unknown) 212 compiler_panic( 213 f->c, in->loc, 214 "opt_machinize: class %u machine clobber mask names unknown " 215 "physical register %#x", 216 (unsigned)cls, (unsigned)unknown); 217 } 218 } 219 220 /* Record, per instruction, every optimizer-visible register the target's 221 * encoding clobbers as a side effect (division, variable shifts, bitfield RMW, 222 * atomics, intrinsics, and format-dependent TLS among the current cases). The 223 * allocator keeps live-across MIR values out of those registers, and native 224 * emission uses the same exhaustive effect to invalidate clean frame-cache 225 * entries in cache-capable emit temps. Pure backend-private registers stay out 226 * of the table. A NULL hook promises no such clobbers, so the derived side 227 * table stays empty. */ 228 void opt_refresh_machine_clobbers(Func* f, NativeTarget* target) { 229 if (!f || !target) return; 230 /* Target effects are derived from the current instruction shapes, not 231 * persistent IR state. Replace the table on every refresh so a shape-changing 232 * pass, target change, or instruction-id growth cannot retain stale effects. 233 * The arena owns the superseded allocation. */ 234 f->inst_clobbers = NULL; 235 f->inst_clobbers_cap = 0; 236 if (!target->machine_op_clobbers || !f->next_inst_id) return; 237 for (u32 b = 0; b < f->nblocks; ++b) { 238 Block* bl = &f->blocks[b]; 239 for (u32 i = 0; i < bl->ninsts; ++i) { 240 Inst* in = &bl->insts[i]; 241 NativeMachineOp mop; 242 u32 mask[NATIVE_REG_CLASS_COUNT]; 243 int has_clobbers; 244 memset(&mop, 0, sizeof mop); 245 switch ((IROp)in->op) { 246 case IR_BINOP: 247 mop.kind = NATIVE_MOP_BINOP; 248 mop.binop = (u8)in->extra.imm; 249 mop.second_is_reg = 250 (u8)(in->nopnds > 2u && in->opnds[2].kind == OPK_REG); 251 break; 252 case IR_BITFIELD_LOAD: 253 mop.kind = NATIVE_MOP_BITFIELD_LOAD; 254 break; 255 case IR_BITFIELD_STORE: 256 mop.kind = NATIVE_MOP_BITFIELD_STORE; 257 break; 258 case IR_VA_START: 259 mop.kind = NATIVE_MOP_VA_START; 260 break; 261 case IR_VA_ARG: 262 mop.kind = NATIVE_MOP_VA_ARG; 263 mop.result_is_fp = (u8)(in->nopnds > 0u && 264 cg_type_is_float(f->c, in->opnds[0].type)); 265 break; 266 case IR_ATOMIC_CAS: 267 mop.kind = NATIVE_MOP_ATOMIC_CAS; 268 break; 269 case IR_ATOMIC_RMW: 270 mop.kind = NATIVE_MOP_ATOMIC_RMW; 271 break; 272 case IR_TLS_ADDR_OF: 273 mop.kind = NATIVE_MOP_TLS_ADDR; 274 break; 275 case IR_INTRINSIC: { 276 const IRIntrinAux* aux = (const IRIntrinAux*)in->extra.aux; 277 if (!aux) continue; 278 mop.kind = NATIVE_MOP_INTRINSIC; 279 mop.intrin = (u8)aux->kind; 280 break; 281 } 282 default: 283 continue; 284 } 285 mask[0] = mask[1] = mask[2] = 0; 286 has_clobbers = target->machine_op_clobbers(target, &mop, mask); 287 machinize_validate_clobber_masks(f, in, mask); 288 if (!has_clobbers) continue; 289 if (in->id == INST_ID_NONE) continue; 290 if (!f->inst_clobbers) { 291 f->inst_clobbers_cap = f->next_inst_id; 292 f->inst_clobbers = 293 arena_zarray(f->arena, OptInstClobberMask, f->inst_clobbers_cap); 294 } 295 if (in->id < f->inst_clobbers_cap) 296 for (u32 c = 0; c < OPT_REG_CLASSES; ++c) 297 f->inst_clobbers[in->id][c] = mask[c]; 298 } 299 } 300 } 301 302 void opt_machinize_native(Func* f, NativeTarget* target) { 303 machinize_reset(f, target); 304 machinize_prepare_insts(f, target); 305 machinize_collect_regs(f, target); 306 machinize_check_overlap(f); 307 opt_refresh_machine_clobbers(f, target); 308 }