native_emit_frame_dst_test.c (70024B)
1 #include <stdio.h> 2 #include <string.h> 3 4 #include "lib/kit_unit.h" 5 #include "opt/opt.h" 6 7 /* NativeTarget's vtable was declared against the semantic descriptor before 8 * opt/ir.h introduced its optimizer-private compatibility macro. Keep the 9 * callback signature on that original boundary; use OptCGFuncDesc explicitly 10 * when constructing the optimizer Func below. */ 11 #undef CGFuncDesc 12 13 typedef struct MockNative { 14 NativeTarget base; 15 KitUnit *unit; 16 NativeAllocClassInfo classes[2]; 17 NativePhysRegInfo int_phys[1]; 18 NativeRegInfo regs; 19 Reg int_temps[4]; 20 NativeCallPhaseMove call_arg; 21 NativeCallPhaseRet call_ret; 22 NativeCallPhaseMove phase_call_args[2]; 23 NativeCallPhaseRet ret_parts[4]; 24 u32 callback_mask; 25 u32 writeback_slots; 26 u32 load_imms; 27 u32 loads; 28 u32 moves; 29 u32 load_addrs; 30 u32 frame_loads; 31 u32 reg_address_loads; 32 u32 in_place_reg_address_loads; 33 u32 reg_address_stores; 34 u32 in_place_reg_address_bitfield_loads; 35 u32 atomic_loads; 36 u32 atomic_stores; 37 Reg frame_load_dsts[16]; 38 u32 nframe_load_dsts; 39 u32 reg_values[32]; 40 u32 phase_callee_value; 41 u32 phase_arg_values[2]; 42 u32 calls; 43 Reg machine_clobber_reg; 44 u8 report_bitfield_store_clobber; 45 u8 plan_two_arg_indirect_call; 46 u8 plan_four_part_ret; 47 u8 reject_indexed_addr; 48 MCLabel next_label; 49 } MockNative; 50 51 enum { 52 SAW_LOAD_CONST = 1u << 0, 53 SAW_TLS_ADDR = 1u << 1, 54 SAW_BITFIELD_LOAD = 1u << 2, 55 SAW_LABEL_ADDR = 1u << 3, 56 SAW_ALLOCA = 1u << 4, 57 SAW_INTRINSIC = 1u << 5, 58 SAW_BINOP = 1u << 6, 59 SAW_ATOMIC_RMW = 1u << 7, 60 SAW_ATOMIC_CAS = 1u << 8, 61 SAW_BITFIELD_STORE = 1u << 9, 62 }; 63 64 static MockNative *mock_of(NativeTarget *target) { 65 return (MockNative *)target; 66 } 67 68 static void expect_reg(MockNative *mock, NativeLoc loc, const char *what) { 69 CU_EXPECT(mock->unit, loc.kind == NATIVE_LOC_REG, 70 "%s must receive a physical register, got location kind %u", what, 71 (unsigned)loc.kind); 72 } 73 74 static NativeAllocClass mock_class_for_type(NativeTarget *target, 75 KitCgTypeId type) { 76 return cg_type_is_float(target->c, type) ? NATIVE_REG_FP : NATIVE_REG_INT; 77 } 78 79 static int mock_addr_legal(NativeTarget *target, const NativeAddr *addr, 80 MemAccess mem) { 81 (void)mem; 82 return !mock_of(target)->reject_indexed_addr || 83 addr->index_kind == NATIVE_ADDR_INDEX_NONE; 84 } 85 86 static void mock_func_begin_known_frame(NativeTarget *target, 87 const CGFuncDesc *desc, 88 const NativeKnownFrameDesc *frame, 89 NativeFrameSlot *out_slots) { 90 (void)target; 91 (void)desc; 92 for (u32 i = 0; i < frame->nslots; ++i) 93 out_slots[i] = i + 1u; 94 } 95 96 static void mock_func_end(NativeTarget *target) { (void)target; } 97 98 static MCLabel mock_label_new(NativeTarget *target) { 99 return ++mock_of(target)->next_label; 100 } 101 102 static void mock_label_place(NativeTarget *target, MCLabel label) { 103 (void)target; 104 (void)label; 105 } 106 107 static void mock_load_imm(NativeTarget *target, NativeLoc dst, i64 imm) { 108 MockNative *mock = mock_of(target); 109 expect_reg(mock, dst, "load_imm destination"); 110 if (dst.v.reg < 32u) mock->reg_values[dst.v.reg] = (u32)imm; 111 ++mock->load_imms; 112 } 113 114 static void mock_move(NativeTarget *target, NativeLoc dst, NativeLoc src) { 115 MockNative *mock = mock_of(target); 116 expect_reg(mock, dst, "move destination"); 117 expect_reg(mock, src, "move source"); 118 if (dst.v.reg < 32u && src.v.reg < 32u) 119 mock->reg_values[dst.v.reg] = mock->reg_values[src.v.reg]; 120 ++mock->moves; 121 } 122 123 static void mock_load_const(NativeTarget *target, NativeLoc dst, 124 ConstBytes bytes) { 125 MockNative *mock = mock_of(target); 126 expect_reg(mock, dst, "load_const destination"); 127 CU_EXPECT(mock->unit, bytes.size == 8u, 128 "load_const byte payload should remain intact"); 129 mock->callback_mask |= SAW_LOAD_CONST; 130 } 131 132 static void mock_load(NativeTarget *target, NativeLoc dst, NativeAddr addr, 133 MemAccess mem) { 134 MockNative *mock = mock_of(target); 135 expect_reg(mock, dst, "frame materialization destination"); 136 CU_EXPECT(mock->unit, mem.size != 0u, 137 "frame materialization should retain its access width"); 138 if (addr.base_kind == NATIVE_ADDR_BASE_FRAME) { 139 if (mock->nframe_load_dsts < 140 sizeof mock->frame_load_dsts / sizeof mock->frame_load_dsts[0]) 141 mock->frame_load_dsts[mock->nframe_load_dsts++] = dst.v.reg; 142 ++mock->frame_loads; 143 if (dst.v.reg < 32u) 144 mock->reg_values[dst.v.reg] = addr.base.frame; 145 } else if (addr.base_kind == NATIVE_ADDR_BASE_REG) { 146 ++mock->reg_address_loads; 147 if (dst.v.reg == addr.base.reg) ++mock->in_place_reg_address_loads; 148 } else { 149 CU_EXPECT(mock->unit, 0, 150 "mock load requires a direct frame or register address, got %u", 151 (unsigned)addr.base_kind); 152 } 153 ++mock->loads; 154 } 155 156 static void mock_load_addr(NativeTarget *target, NativeLoc dst, 157 NativeAddr addr) { 158 MockNative *mock = mock_of(target); 159 u32 value = 0; 160 expect_reg(mock, dst, "address materialization destination"); 161 CU_EXPECT(mock->unit, 162 addr.base_kind == NATIVE_ADDR_BASE_REG && 163 addr.index_kind == NATIVE_ADDR_INDEX_REG, 164 "pressure regression must materialize a register base+index"); 165 CU_EXPECT(mock->unit, dst.v.reg == addr.base.reg, 166 "scoped address base should be reusable when no third temp exists"); 167 if (addr.base_kind == NATIVE_ADDR_BASE_REG && addr.base.reg < 32u) 168 value = mock->reg_values[addr.base.reg]; 169 if (addr.index_kind == NATIVE_ADDR_INDEX_REG && addr.index.reg < 32u) 170 value += mock->reg_values[addr.index.reg] << addr.log2_scale; 171 value += (u32)addr.offset; 172 if (dst.v.reg < 32u) mock->reg_values[dst.v.reg] = value; 173 ++mock->load_addrs; 174 } 175 176 static void mock_store(NativeTarget *target, NativeAddr addr, NativeLoc src, 177 MemAccess mem) { 178 MockNative *mock = mock_of(target); 179 expect_reg(mock, src, "frame writeback source"); 180 CU_EXPECT(mock->unit, mem.size != 0u, 181 "frame writeback should retain its access width"); 182 if (addr.base_kind == NATIVE_ADDR_BASE_FRAME) { 183 CU_EXPECT(mock->unit, addr.base.frame > 0u && addr.base.frame < 32u, 184 "frame writeback slot should be mapped and in test range"); 185 if (addr.base.frame < 32u) 186 mock->writeback_slots |= 1u << addr.base.frame; 187 } else if (addr.base_kind == NATIVE_ADDR_BASE_REG) { 188 CU_EXPECT(mock->unit, src.v.reg != addr.base.reg, 189 "store value must remain distinct from its address base"); 190 ++mock->reg_address_stores; 191 } else { 192 CU_EXPECT(mock->unit, 0, 193 "mock store requires a direct frame or register address, got %u", 194 (unsigned)addr.base_kind); 195 } 196 } 197 198 static void mock_tls_addr_of(NativeTarget *target, NativeLoc dst, ObjSymId sym, 199 i64 addend) { 200 MockNative *mock = mock_of(target); 201 expect_reg(mock, dst, "tls_addr_of destination"); 202 CU_EXPECT(mock->unit, sym == 17u && addend == 9, 203 "tls address metadata should remain intact"); 204 mock->callback_mask |= SAW_TLS_ADDR; 205 } 206 207 static void mock_bitfield_load(NativeTarget *target, NativeLoc dst, 208 NativeAddr addr, BitFieldAccess access) { 209 MockNative *mock = mock_of(target); 210 expect_reg(mock, dst, "bitfield_load destination"); 211 CU_EXPECT(mock->unit, 212 addr.base_kind == NATIVE_ADDR_BASE_FRAME || 213 addr.base_kind == NATIVE_ADDR_BASE_REG, 214 "bitfield record should remain a frame or register address"); 215 if (addr.base_kind == NATIVE_ADDR_BASE_REG && dst.v.reg == addr.base.reg) 216 ++mock->in_place_reg_address_bitfield_loads; 217 CU_EXPECT(mock->unit, access.bit_offset == 3u && access.bit_width == 11u, 218 "bitfield geometry should remain intact"); 219 mock->callback_mask |= SAW_BITFIELD_LOAD; 220 } 221 222 static void mock_bitfield_store(NativeTarget *target, NativeAddr addr, 223 NativeLoc src, BitFieldAccess access) { 224 MockNative *mock = mock_of(target); 225 expect_reg(mock, src, "bitfield_store source"); 226 CU_EXPECT(mock->unit, addr.base_kind == NATIVE_ADDR_BASE_FRAME, 227 "bitfield store record should remain a frame address"); 228 CU_EXPECT(mock->unit, access.bit_offset == 3u && access.bit_width == 11u, 229 "bitfield store geometry should remain intact"); 230 mock->callback_mask |= SAW_BITFIELD_STORE; 231 } 232 233 static int mock_machine_op_clobbers( 234 NativeTarget *target, const NativeMachineOp *op, 235 u32 mask[NATIVE_REG_CLASS_COUNT]) { 236 MockNative *mock = mock_of(target); 237 mask[0] = mask[1] = mask[2] = 0u; 238 if (!mock->report_bitfield_store_clobber || 239 op->kind != NATIVE_MOP_BITFIELD_STORE) 240 return 0; 241 if (mock->machine_clobber_reg >= 32u) return 0; 242 mask[NATIVE_REG_INT] = 1u << mock->machine_clobber_reg; 243 return 1; 244 } 245 246 static void mock_load_label_addr(NativeTarget *target, NativeLoc dst, 247 MCLabel label) { 248 MockNative *mock = mock_of(target); 249 expect_reg(mock, dst, "load_label_addr destination"); 250 CU_EXPECT(mock->unit, label != MC_LABEL_NONE, 251 "label address should reference an allocated native label"); 252 mock->callback_mask |= SAW_LABEL_ADDR; 253 } 254 255 static void mock_alloca(NativeTarget *target, NativeLoc dst, NativeLoc size, 256 u32 align) { 257 MockNative *mock = mock_of(target); 258 expect_reg(mock, dst, "alloca destination"); 259 expect_reg(mock, size, "alloca size"); 260 CU_EXPECT(mock->unit, dst.v.reg != size.v.reg, 261 "alloca destination must not clobber its size input"); 262 CU_EXPECT(mock->unit, align == 32u, "alloca alignment should remain intact"); 263 mock->callback_mask |= SAW_ALLOCA; 264 } 265 266 static void mock_binop(NativeTarget *target, BinOp op, NativeLoc dst, 267 NativeLoc lhs, NativeLoc rhs) { 268 MockNative *mock = mock_of(target); 269 expect_reg(mock, dst, "binop destination"); 270 expect_reg(mock, lhs, "binop lhs"); 271 expect_reg(mock, rhs, "binop rhs"); 272 CU_EXPECT(mock->unit, op == BO_IADD, "binop opcode should remain intact"); 273 CU_EXPECT(mock->unit, dst.v.reg != rhs.v.reg && lhs.v.reg != rhs.v.reg, 274 "all-stack binop must keep its rhs distinct while allowing the " 275 "dead lhs register to become the destination"); 276 mock->callback_mask |= SAW_BINOP; 277 } 278 279 static void mock_atomic_rmw(NativeTarget *target, KitCgAtomicOp op, 280 NativeLoc dst, NativeAddr addr, NativeLoc val, 281 MemAccess mem, KitCgMemOrder order) { 282 MockNative *mock = mock_of(target); 283 expect_reg(mock, dst, "atomic_rmw destination"); 284 expect_reg(mock, val, "atomic_rmw value"); 285 CU_EXPECT(mock->unit, dst.v.reg != val.v.reg, 286 "atomic_rmw destination must not alias its value input"); 287 CU_EXPECT(mock->unit, addr.base_kind == NATIVE_ADDR_BASE_FRAME, 288 "atomic_rmw address should retain direct frame storage"); 289 CU_EXPECT(mock->unit, 290 op == KIT_CG_ATOMIC_ADD && mem.size == 8u && 291 order == KIT_CG_MO_SEQ_CST, 292 "atomic_rmw metadata should remain intact"); 293 mock->callback_mask |= SAW_ATOMIC_RMW; 294 } 295 296 static void mock_atomic_load(NativeTarget *target, NativeLoc dst, 297 NativeAddr addr, MemAccess mem, 298 KitCgMemOrder order) { 299 MockNative *mock = mock_of(target); 300 expect_reg(mock, dst, "atomic_load destination"); 301 CU_EXPECT(mock->unit, 302 addr.base_kind == NATIVE_ADDR_BASE_REG && 303 addr.index_kind == NATIVE_ADDR_INDEX_NONE && 304 dst.v.reg == addr.base.reg, 305 "atomic load should consume and reuse one collapsed scoped base"); 306 CU_EXPECT(mock->unit, 307 mem.size == 8u && order == KIT_CG_MO_SEQ_CST, 308 "atomic load metadata should remain intact"); 309 ++mock->atomic_loads; 310 } 311 312 static void mock_atomic_store(NativeTarget *target, NativeAddr addr, 313 NativeLoc val, MemAccess mem, 314 KitCgMemOrder order) { 315 MockNative *mock = mock_of(target); 316 expect_reg(mock, val, "atomic_store value"); 317 CU_EXPECT(mock->unit, addr.base_kind == NATIVE_ADDR_BASE_REG, 318 "atomic store pressure address should be materialized"); 319 CU_EXPECT(mock->unit, 320 addr.base_kind != NATIVE_ADDR_BASE_REG || 321 val.v.reg != addr.base.reg, 322 "atomic store reused the overwritten address-base cache fact"); 323 CU_EXPECT(mock->unit, 324 mem.size == 8u && order == KIT_CG_MO_SEQ_CST, 325 "atomic store metadata should remain intact"); 326 ++mock->atomic_stores; 327 } 328 329 static void mock_atomic_cas(NativeTarget *target, NativeLoc prior, NativeLoc ok, 330 NativeAddr addr, NativeLoc expected, 331 NativeLoc desired, MemAccess mem, 332 KitCgMemOrder success, KitCgMemOrder failure) { 333 MockNative *mock = mock_of(target); 334 NativeLoc locs[4] = {prior, ok, expected, desired}; 335 for (u32 i = 0; i < 4u; ++i) 336 expect_reg(mock, locs[i], "atomic_cas scalar operand"); 337 for (u32 i = 0; i < 4u; ++i) 338 for (u32 j = i + 1u; j < 4u; ++j) 339 CU_EXPECT(mock->unit, locs[i].v.reg != locs[j].v.reg, 340 "all-stack atomic_cas must lease four simultaneous registers"); 341 CU_EXPECT(mock->unit, addr.base_kind == NATIVE_ADDR_BASE_FRAME, 342 "atomic_cas address should retain direct frame storage"); 343 CU_EXPECT(mock->unit, 344 mem.size == 8u && success == KIT_CG_MO_ACQ_REL && 345 failure == KIT_CG_MO_ACQUIRE, 346 "atomic_cas metadata should remain intact"); 347 mock->callback_mask |= SAW_ATOMIC_CAS; 348 } 349 350 static void mock_intrinsic(NativeTarget *target, IntrinKind kind, 351 const NativeLoc *dsts, u32 ndst, 352 const NativeLoc *args, u32 narg) { 353 MockNative *mock = mock_of(target); 354 CU_EXPECT(mock->unit, 355 kind == INTRIN_UADD_OVERFLOW && ndst == 2u && narg == 2u, 356 "multi-result intrinsic shape should remain intact"); 357 for (u32 i = 0; i < ndst; ++i) 358 expect_reg(mock, dsts[i], "intrinsic destination"); 359 for (u32 i = 0; i < narg; ++i) 360 expect_reg(mock, args[i], "intrinsic argument"); 361 if (ndst == 2u && narg == 2u) { 362 Reg regs[4] = {dsts[0].v.reg, dsts[1].v.reg, args[0].v.reg, args[1].v.reg}; 363 for (u32 i = 0; i < 4u; ++i) 364 for (u32 j = i + 1u; j < 4u; ++j) 365 CU_EXPECT(mock->unit, regs[i] != regs[j], 366 "simultaneous intrinsic operands must have distinct temps"); 367 } 368 mock->callback_mask |= SAW_INTRINSIC; 369 } 370 371 static NativeLoc mock_reg_loc(KitCgTypeId type, NativeAllocClass cls, Reg reg) { 372 NativeLoc loc; 373 memset(&loc, 0, sizeof loc); 374 loc.kind = NATIVE_LOC_REG; 375 loc.cls = (u8)cls; 376 loc.type = type; 377 loc.v.reg = reg; 378 return loc; 379 } 380 381 /* Deliberately source the post-call copy from the same frame value used by 382 * the argument. The call clobbers the sole staging register, so the emitter 383 * must end the pre-call temp/cache phase and reload this source afterwards. */ 384 static void mock_marshal_call(NativeTarget *target, const NativeCallDesc *desc, 385 NativeCallPhase *plan) { 386 MockNative *mock = mock_of(target); 387 if (mock->plan_two_arg_indirect_call) { 388 NativeLoc staged_callee; 389 CU_EXPECT(mock->unit, 390 desc->nargs == 2u && desc->nresults == 0u && 391 desc->callee.kind == NATIVE_LOC_REG, 392 "indirect callee must be materialized before the argument plan"); 393 if (desc->nargs != 2u || desc->nresults != 0u) 394 return; 395 if (desc->callee.kind == NATIVE_LOC_REG) { 396 staged_callee = 397 mock_reg_loc(desc->callee.type, NATIVE_REG_INT, 12u); 398 mock_move(target, staged_callee, desc->callee); 399 plan->callee = staged_callee; 400 mock->phase_callee_value = mock->reg_values[staged_callee.v.reg]; 401 } else { 402 /* Keep the mock executable against the pre-fix ordering too: the 403 * emitter used to materialize this plan location after its argument 404 * moves. The expectation above remains the focused red assertion. */ 405 plan->callee = desc->callee; 406 if (desc->callee.kind == NATIVE_LOC_FRAME) 407 mock->phase_callee_value = desc->callee.v.frame; 408 } 409 for (u32 i = 0; i < 2u; ++i) { 410 memset(&mock->phase_call_args[i], 0, 411 sizeof mock->phase_call_args[i]); 412 mock->phase_call_args[i].src = desc->args[i]; 413 mock->phase_call_args[i].dst = 414 mock_reg_loc(desc->args[i].type, NATIVE_REG_INT, (Reg)(8u + i)); 415 mock->phase_call_args[i].mem.type = desc->args[i].type; 416 mock->phase_call_args[i].mem.size = 8u; 417 mock->phase_call_args[i].mem.align = 8u; 418 CU_EXPECT(mock->unit, desc->args[i].kind == NATIVE_LOC_FRAME, 419 "planned call argument %u must retain its frame source", i); 420 if (desc->args[i].kind == NATIVE_LOC_FRAME) 421 mock->phase_arg_values[i] = desc->args[i].v.frame; 422 } 423 plan->args = mock->phase_call_args; 424 plan->nargs = 2u; 425 return; 426 } 427 CU_EXPECT(mock->unit, desc->nargs == 1u && desc->nresults == 1u, 428 "call phase test requires one argument and one result"); 429 if (desc->nargs != 1u || desc->nresults != 1u) 430 return; 431 memset(&mock->call_arg, 0, sizeof mock->call_arg); 432 memset(&mock->call_ret, 0, sizeof mock->call_ret); 433 plan->callee = desc->callee; 434 mock->call_arg.src = desc->args[0]; 435 mock->call_arg.dst = mock_reg_loc(desc->args[0].type, NATIVE_REG_INT, 8u); 436 mock->call_arg.mem.type = desc->args[0].type; 437 mock->call_arg.mem.size = 8u; 438 mock->call_arg.mem.align = 8u; 439 plan->args = &mock->call_arg; 440 plan->nargs = 1u; 441 mock->call_ret.src = desc->args[0]; 442 mock->call_ret.dst = desc->results[0]; 443 mock->call_ret.mem.type = desc->results[0].type; 444 mock->call_ret.mem.size = 8u; 445 mock->call_ret.mem.align = 8u; 446 plan->rets = &mock->call_ret; 447 plan->nrets = 1u; 448 } 449 450 static void mock_emit_call(NativeTarget *target, const NativeCallPhase *plan) { 451 MockNative *mock = mock_of(target); 452 if (mock->plan_two_arg_indirect_call) { 453 CU_EXPECT(mock->unit, 454 plan->callee.kind == NATIVE_LOC_REG && plan->nargs == 2u && 455 plan->nrets == 0u, 456 "two-argument indirect call plan changed before emission"); 457 if (plan->callee.kind == NATIVE_LOC_REG) { 458 CU_EXPECT(mock->unit, 459 mock->reg_values[plan->callee.v.reg] == 460 mock->phase_callee_value, 461 "argument staging overwrote the indirect callee"); 462 } 463 CU_EXPECT(mock->unit, 464 mock->reg_values[8u] == mock->phase_arg_values[0], 465 "later call setup overwrote completed ABI argument r8"); 466 CU_EXPECT(mock->unit, 467 mock->reg_values[9u] == mock->phase_arg_values[1], 468 "second ABI argument did not reach r9"); 469 ++mock->calls; 470 return; 471 } 472 CU_EXPECT(mock->unit, plan->nargs == 1u && plan->nrets == 1u, 473 "call plan shape changed before emission"); 474 ++mock->calls; 475 } 476 477 static void mock_marshal_ret(NativeTarget *target, const CGFuncDesc *desc, 478 const NativeLoc *value, 479 NativeCallPhaseRet **out_rets, u32 *out_nrets) { 480 MockNative *mock = mock_of(target); 481 (void)desc; 482 if (!mock->plan_four_part_ret) { 483 *out_rets = NULL; 484 *out_nrets = 0u; 485 return; 486 } 487 CU_EXPECT(mock->unit, value && value->kind == NATIVE_LOC_FRAME, 488 "four-part return test requires a frame-resident source"); 489 if (!value || value->kind != NATIVE_LOC_FRAME) { 490 *out_rets = NULL; 491 *out_nrets = 0u; 492 return; 493 } 494 for (u32 i = 0; i < 4u; ++i) { 495 memset(&mock->ret_parts[i], 0, sizeof mock->ret_parts[i]); 496 mock->ret_parts[i].src = 497 native_loc_stack(value->type, value->v.frame, (i32)(i * 8u)); 498 mock->ret_parts[i].dst = 499 mock_reg_loc(value->type, NATIVE_REG_INT, (Reg)(8u + i)); 500 mock->ret_parts[i].mem.type = value->type; 501 mock->ret_parts[i].mem.size = 8u; 502 mock->ret_parts[i].mem.align = 8u; 503 } 504 *out_rets = mock->ret_parts; 505 *out_nrets = 4u; 506 } 507 508 static void mock_ret(NativeTarget *target) { (void)target; } 509 510 static void mock_init(MockNative *mock, KitUnit *unit, Compiler *c) { 511 memset(mock, 0, sizeof *mock); 512 mock->unit = unit; 513 mock->base.c = c; 514 mock->int_temps[0] = 8u; 515 mock->int_temps[1] = 9u; 516 mock->machine_clobber_reg = 10u; 517 mock->classes[NATIVE_REG_INT].cls = NATIVE_REG_INT; 518 mock->classes[NATIVE_REG_INT].scratch = mock->int_temps; 519 mock->classes[NATIVE_REG_INT].nscratch = 2u; 520 mock->classes[NATIVE_REG_INT].emit_temps = mock->int_temps; 521 mock->classes[NATIVE_REG_INT].nemit_temps = 2u; 522 mock->classes[NATIVE_REG_INT].emit_cache_mask = 523 (1u << mock->int_temps[0]) | (1u << mock->int_temps[1]); 524 mock->classes[NATIVE_REG_FP].cls = NATIVE_REG_FP; 525 mock->regs.classes = mock->classes; 526 mock->regs.nclasses = 2u; 527 mock->base.regs = &mock->regs; 528 mock->base.class_for_type = mock_class_for_type; 529 mock->base.addr_legal = mock_addr_legal; 530 mock->base.func_begin_known_frame = mock_func_begin_known_frame; 531 mock->base.func_end = mock_func_end; 532 mock->base.label_new = mock_label_new; 533 mock->base.label_place = mock_label_place; 534 mock->base.load_imm = mock_load_imm; 535 mock->base.move = mock_move; 536 mock->base.load_const = mock_load_const; 537 mock->base.load_addr = mock_load_addr; 538 mock->base.load = mock_load; 539 mock->base.store = mock_store; 540 mock->base.tls_addr_of = mock_tls_addr_of; 541 mock->base.bitfield_load = mock_bitfield_load; 542 mock->base.bitfield_store = mock_bitfield_store; 543 mock->base.load_label_addr = mock_load_label_addr; 544 mock->base.alloca_ = mock_alloca; 545 mock->base.binop = mock_binop; 546 mock->base.atomic_load = mock_atomic_load; 547 mock->base.atomic_store = mock_atomic_store; 548 mock->base.atomic_rmw = mock_atomic_rmw; 549 mock->base.atomic_cas = mock_atomic_cas; 550 mock->base.intrinsic = mock_intrinsic; 551 mock->base.marshal_call = mock_marshal_call; 552 mock->base.emit_call = mock_emit_call; 553 mock->base.machine_op_clobbers = mock_machine_op_clobbers; 554 mock->base.marshal_ret = mock_marshal_ret; 555 mock->base.ret = mock_ret; 556 } 557 558 static Operand stack_op(FrameSlot slot, KitCgTypeId type) { 559 Operand op; 560 memset(&op, 0, sizeof op); 561 op.kind = OPK_STACK; 562 op.cls = RC_INT; 563 op.type = type; 564 op.v.frame_slot = slot; 565 return op; 566 } 567 568 static Operand frame_addr_op(FrameSlot slot, KitCgTypeId type) { 569 Operand op = stack_op(slot, type); 570 op.kind = OPK_FRAME_ADDR; 571 return op; 572 } 573 574 static Operand imm_op(i64 value, KitCgTypeId type) { 575 Operand op; 576 memset(&op, 0, sizeof op); 577 op.kind = OPK_IMM; 578 op.cls = RC_INT; 579 op.type = type; 580 op.v.imm = value; 581 return op; 582 } 583 584 static Operand reg_op(Reg reg, KitCgTypeId type) { 585 Operand op; 586 memset(&op, 0, sizeof op); 587 op.kind = OPK_REG; 588 op.cls = RC_INT; 589 op.type = type; 590 op.v.reg = reg; 591 return op; 592 } 593 594 static Operand global_op(ObjSymId sym, KitCgTypeId type) { 595 Operand op; 596 memset(&op, 0, sizeof op); 597 op.kind = OPK_GLOBAL; 598 op.cls = RC_INT; 599 op.type = type; 600 op.v.global.sym = sym; 601 return op; 602 } 603 604 static Operand indirect_frame_op(FrameSlot base, KitCgTypeId ptr_type) { 605 Operand op; 606 memset(&op, 0, sizeof op); 607 op.kind = OPK_INDIRECT; 608 op.cls = RC_INT; 609 op.type = ptr_type; 610 op.v.ind.base = base; 611 op.v.ind.index = REG_NONE; 612 op.v.ind.base_kind = OPT_INDIRECT_FRAME; 613 op.v.ind.index_kind = OPT_INDIRECT_REG; 614 op.v.ind.base_type = ptr_type; 615 return op; 616 } 617 618 static Operand indirect_frame_index_op(FrameSlot base, FrameSlot index, 619 KitCgTypeId ptr_type, 620 KitCgTypeId index_type) { 621 Operand op = indirect_frame_op(base, ptr_type); 622 op.v.ind.index = index; 623 op.v.ind.index_kind = OPT_INDIRECT_FRAME; 624 op.v.ind.index_type = index_type; 625 op.v.ind.log2_scale = 3u; 626 return op; 627 } 628 629 static FrameSlot add_slot(Func *f, KitCgTypeId type, u32 size, u32 align) { 630 FrameSlotDesc desc; 631 memset(&desc, 0, sizeof desc); 632 desc.type = type; 633 desc.size = size; 634 desc.align = align; 635 desc.kind = FS_SPILL; 636 return ir_frame_slot_new(f, &desc); 637 } 638 639 static Inst *emit_with_ops(Func *f, u32 block, IROp op, u32 nopnds) { 640 Inst *in = ir_emit(f, block, op); 641 in->nopnds = nopnds; 642 in->opnds = nopnds ? arena_zarray(f->arena, Operand, nopnds) : NULL; 643 return in; 644 } 645 646 static void frame_destinations_are_staged(KitUnit *unit) { 647 KitCompiler *kit_c = NULL; 648 Compiler *c; 649 KitCgTypeId i64_type, bool_type, ptr_type; 650 OptCGFuncDesc desc; 651 Func *f; 652 MockNative mock; 653 u32 block; 654 FrameSlot slots[19]; 655 u64 constant = 0x1122334455667788ull; 656 Inst *in; 657 658 CU_EXPECT(unit, 659 kit_unit_compiler_new( 660 unit, 661 kit_unit_target(KIT_ARCH_X86_64, KIT_OS_LINUX, KIT_OBJ_ELF), 662 &kit_c) == KIT_OK && 663 kit_c != NULL, 664 "compiler allocation failed"); 665 if (!kit_c) 666 return; 667 c = (Compiler *)kit_c; 668 i64_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_I64); 669 bool_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_BOOL); 670 ptr_type = kit_cg_type_ptr( 671 kit_c, kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID), 0); 672 673 memset(&desc, 0, sizeof desc); 674 desc.result_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID); 675 f = ir_func_new(c, &desc); 676 block = ir_block_new(f); 677 f->entry = block; 678 ir_note_emit(f, block); 679 f->opt_rewritten = 1; 680 /* Model x64's deliberately narrow driver-temp bank. The third binop temp 681 * and the extra multi-result temps must be scavenged from dead caller-saved 682 * allocation registers, never represented as MIR values. */ 683 for (u32 i = 0; i < 2u; ++i) { 684 f->emit_temp_regs[RC_INT][i] = (Reg)(8u + i); 685 f->opt_reserved_regs[RC_INT] |= 1u << (8u + i); 686 } 687 f->emit_temp_reg_count[RC_INT] = 2u; 688 f->opt_hard_regs[RC_INT][0] = 10u; 689 f->opt_hard_regs[RC_INT][1] = 11u; 690 f->opt_hard_reg_count[RC_INT] = 2u; 691 f->opt_caller_saved[RC_INT] = (1u << 10) | (1u << 11); 692 693 slots[0] = add_slot(f, i64_type, 8u, 8u); /* load_const dst */ 694 slots[1] = add_slot(f, ptr_type, 8u, 8u); /* tls dst */ 695 slots[2] = add_slot(f, i64_type, 8u, 8u); /* bitfield dst */ 696 slots[3] = add_slot(f, i64_type, 8u, 8u); /* bitfield record */ 697 slots[4] = add_slot(f, ptr_type, 8u, 8u); /* label dst */ 698 slots[5] = add_slot(f, ptr_type, 8u, 8u); /* alloca dst */ 699 slots[6] = add_slot(f, i64_type, 8u, 8u); /* intrinsic value */ 700 slots[7] = add_slot(f, bool_type, 1u, 1u); /* intrinsic overflow */ 701 slots[8] = add_slot(f, i64_type, 8u, 8u); /* intrinsic arg 0 */ 702 slots[9] = add_slot(f, i64_type, 8u, 8u); /* intrinsic arg 1 */ 703 slots[10] = add_slot(f, i64_type, 8u, 8u); /* binop dst */ 704 slots[11] = add_slot(f, i64_type, 8u, 8u); /* binop lhs */ 705 slots[12] = add_slot(f, i64_type, 8u, 8u); /* binop rhs */ 706 slots[13] = add_slot(f, i64_type, 8u, 8u); /* atomic_rmw dst */ 707 slots[14] = add_slot(f, i64_type, 8u, 8u); /* atomic storage */ 708 slots[15] = add_slot(f, i64_type, 8u, 8u); /* atomic_rmw value */ 709 slots[16] = add_slot(f, i64_type, 8u, 8u); /* atomic_cas prior */ 710 slots[17] = add_slot(f, bool_type, 1u, 1u); /* atomic_cas ok */ 711 slots[18] = add_slot(f, i64_type, 8u, 8u); /* atomic_cas expected */ 712 /* Reuse the binop lhs slot as the CAS desired input: the instructions have 713 * disjoint scopes, so this also checks location materialization caching does 714 * not leak between scopes. */ 715 716 in = emit_with_ops(f, block, IR_LOAD_CONST, 1u); 717 in->opnds[0] = stack_op(slots[0], i64_type); 718 in->extra.cbytes.type = i64_type; 719 in->extra.cbytes.bytes = (const u8 *)&constant; 720 in->extra.cbytes.size = 8u; 721 in->extra.cbytes.align = 8u; 722 723 in = emit_with_ops(f, block, IR_TLS_ADDR_OF, 1u); 724 in->opnds[0] = stack_op(slots[1], ptr_type); 725 { 726 IRTlsAux *aux = arena_znew(f->arena, IRTlsAux); 727 aux->sym = 17u; 728 aux->addend = 9; 729 in->extra.aux = aux; 730 } 731 732 in = emit_with_ops(f, block, IR_BITFIELD_LOAD, 2u); 733 in->opnds[0] = stack_op(slots[2], i64_type); 734 in->opnds[1] = frame_addr_op(slots[3], ptr_type); 735 { 736 IRBitFieldAux *aux = arena_znew(f->arena, IRBitFieldAux); 737 aux->access.field_type = i64_type; 738 aux->access.storage.type = i64_type; 739 aux->access.storage.size = 8u; 740 aux->access.storage.align = 8u; 741 aux->access.bit_offset = 3u; 742 aux->access.bit_width = 11u; 743 in->extra.aux = aux; 744 } 745 746 in = emit_with_ops(f, block, IR_LOAD_LABEL_ADDR, 1u); 747 in->opnds[0] = stack_op(slots[4], ptr_type); 748 in->extra.imm = block; 749 750 in = emit_with_ops(f, block, IR_ALLOCA, 2u); 751 in->opnds[0] = stack_op(slots[5], ptr_type); 752 in->opnds[1] = imm_op(96, i64_type); 753 in->extra.imm = 32; 754 755 in = emit_with_ops(f, block, IR_INTRINSIC, 0u); 756 { 757 IRIntrinAux *aux = arena_znew(f->arena, IRIntrinAux); 758 aux->kind = INTRIN_UADD_OVERFLOW; 759 aux->ndst = 2u; 760 aux->narg = 2u; 761 aux->dsts = arena_zarray(f->arena, Operand, aux->ndst); 762 aux->args = arena_zarray(f->arena, Operand, aux->narg); 763 aux->dsts[0] = stack_op(slots[6], i64_type); 764 aux->dsts[1] = stack_op(slots[7], bool_type); 765 aux->args[0] = stack_op(slots[8], i64_type); 766 aux->args[1] = stack_op(slots[9], i64_type); 767 in->extra.aux = aux; 768 } 769 770 in = emit_with_ops(f, block, IR_BINOP, 3u); 771 in->opnds[0] = stack_op(slots[10], i64_type); 772 in->opnds[1] = stack_op(slots[11], i64_type); 773 in->opnds[2] = stack_op(slots[12], i64_type); 774 in->extra.imm = BO_IADD; 775 776 in = emit_with_ops(f, block, IR_ATOMIC_RMW, 3u); 777 in->opnds[0] = stack_op(slots[13], i64_type); 778 in->opnds[1] = frame_addr_op(slots[14], ptr_type); 779 in->opnds[2] = stack_op(slots[15], i64_type); 780 { 781 IRAtomicAux *aux = arena_znew(f->arena, IRAtomicAux); 782 aux->op = KIT_CG_ATOMIC_ADD; 783 aux->mo = KIT_CG_MO_SEQ_CST; 784 aux->mem.type = i64_type; 785 aux->mem.size = 8u; 786 aux->mem.align = 8u; 787 in->extra.aux = aux; 788 } 789 790 in = emit_with_ops(f, block, IR_ATOMIC_CAS, 5u); 791 in->opnds[0] = stack_op(slots[16], i64_type); 792 in->opnds[1] = stack_op(slots[17], bool_type); 793 in->opnds[2] = frame_addr_op(slots[14], ptr_type); 794 in->opnds[3] = stack_op(slots[18], i64_type); 795 in->opnds[4] = stack_op(slots[11], i64_type); 796 { 797 IRCasAux *aux = arena_znew(f->arena, IRCasAux); 798 aux->mem.type = i64_type; 799 aux->mem.size = 8u; 800 aux->mem.align = 8u; 801 aux->success = KIT_CG_MO_ACQ_REL; 802 aux->failure = KIT_CG_MO_ACQUIRE; 803 in->extra.aux = aux; 804 } 805 806 mock_init(&mock, unit, c); 807 opt_emit_native(c, f, &mock.base); 808 809 CU_EXPECT(unit, 810 mock.callback_mask == 811 (SAW_LOAD_CONST | SAW_TLS_ADDR | SAW_BITFIELD_LOAD | 812 SAW_LABEL_ADDR | SAW_ALLOCA | SAW_INTRINSIC | SAW_BINOP | 813 SAW_ATOMIC_RMW | SAW_ATOMIC_CAS), 814 "all frame-destination emitter hooks should be exercised"); 815 { 816 u32 expected = (1u << slots[0]) | (1u << slots[1]) | (1u << slots[2]) | 817 (1u << slots[4]) | (1u << slots[5]) | (1u << slots[6]) | 818 (1u << slots[7]) | (1u << slots[10]) | (1u << slots[13]) | 819 (1u << slots[16]) | (1u << slots[17]); 820 CU_EXPECT(unit, mock.writeback_slots == expected, 821 "each staged destination should be written back exactly once " 822 "(got 0x%x expected 0x%x)", 823 mock.writeback_slots, expected); 824 } 825 CU_EXPECT(unit, mock.loads == 7u, 826 "stack intrinsic/binop/atomic inputs should each materialize once"); 827 828 kit_compiler_free(kit_c); 829 } 830 831 static void cached_frame_value_feeds_indirect_address(KitUnit *unit) { 832 KitCompiler *kit_c = NULL; 833 Compiler *c; 834 KitCgTypeId i64_type, ptr_type; 835 OptCGFuncDesc desc; 836 Func *f; 837 MockNative mock; 838 FrameSlot pointer_slot; 839 Inst *in; 840 u32 block; 841 842 CU_EXPECT(unit, 843 kit_unit_compiler_new( 844 unit, 845 kit_unit_target(KIT_ARCH_X86_64, KIT_OS_LINUX, KIT_OBJ_ELF), 846 &kit_c) == KIT_OK && 847 kit_c != NULL, 848 "compiler allocation failed for indirect frame-cache test"); 849 if (!kit_c) 850 return; 851 c = (Compiler *)kit_c; 852 i64_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_I64); 853 ptr_type = kit_cg_type_ptr( 854 kit_c, kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID), 0); 855 856 memset(&desc, 0, sizeof desc); 857 desc.result_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID); 858 f = ir_func_new(c, &desc); 859 block = ir_block_new(f); 860 f->entry = block; 861 ir_note_emit(f, block); 862 f->opt_rewritten = 1; 863 f->emit_temp_regs[RC_INT][0] = 8u; 864 f->emit_temp_reg_count[RC_INT] = 1u; 865 f->opt_reserved_regs[RC_INT] = 1u << 8u; 866 pointer_slot = add_slot(f, ptr_type, 8u, 8u); 867 868 /* The store makes the frame home authoritative while retaining its clean 869 * register copy. The next address consumes that copy as an indirect base. */ 870 in = emit_with_ops(f, block, IR_LOAD_IMM, 1u); 871 in->opnds[0] = stack_op(pointer_slot, ptr_type); 872 in->extra.imm = 0x1000; 873 874 in = emit_with_ops(f, block, IR_LOAD, 2u); 875 in->opnds[0] = reg_op(10u, i64_type); 876 in->opnds[1] = indirect_frame_op(pointer_slot, ptr_type); 877 in->extra.mem.type = i64_type; 878 in->extra.mem.size = 8u; 879 in->extra.mem.align = 8u; 880 881 mock_init(&mock, unit, c); 882 opt_emit_native(c, f, &mock.base); 883 884 CU_EXPECT(unit, mock.loads == 1u && mock.reg_address_loads == 1u, 885 "cached spill should feed the indirect base without a frame reload " 886 "(loads=%u register-address-loads=%u)", 887 mock.loads, mock.reg_address_loads); 888 CU_EXPECT(unit, mock.frame_loads == 0u, 889 "cached indirect base unexpectedly reloaded from its frame home"); 890 891 kit_compiler_free(kit_c); 892 } 893 894 static void frame_load_reuses_scoped_address_base_under_pressure( 895 KitUnit *unit) { 896 KitCompiler *kit_c = NULL; 897 Compiler *c; 898 KitCgTypeId i64_type, ptr_type; 899 OptCGFuncDesc desc; 900 Func *f; 901 MockNative mock; 902 FrameSlot base_slot, index_slot, result_slot; 903 Inst *in; 904 u32 block; 905 906 CU_EXPECT(unit, 907 kit_unit_compiler_new( 908 unit, 909 kit_unit_target(KIT_ARCH_X86_64, KIT_OS_LINUX, KIT_OBJ_ELF), 910 &kit_c) == KIT_OK && 911 kit_c != NULL, 912 "compiler allocation failed for scoped-base pressure test"); 913 if (!kit_c) return; 914 c = (Compiler *)kit_c; 915 i64_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_I64); 916 ptr_type = kit_cg_type_ptr( 917 kit_c, kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID), 0); 918 919 memset(&desc, 0, sizeof desc); 920 desc.result_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID); 921 f = ir_func_new(c, &desc); 922 block = ir_block_new(f); 923 f->entry = block; 924 ir_note_emit(f, block); 925 f->opt_rewritten = 1; 926 f->emit_temp_regs[RC_INT][0] = 8u; 927 f->emit_temp_regs[RC_INT][1] = 9u; 928 f->emit_temp_reg_count[RC_INT] = 2u; 929 f->opt_reserved_regs[RC_INT] = (1u << 8u) | (1u << 9u); 930 base_slot = add_slot(f, ptr_type, 8u, 8u); 931 index_slot = add_slot(f, i64_type, 8u, 8u); 932 result_slot = add_slot(f, i64_type, 8u, 8u); 933 934 /* Each frame write leaves a clean value in one emitter temp. The indexed 935 * load consumes both cached values as its address while its result also 936 * needs a register before frame writeback: exactly the x64 SQLite pressure 937 * shape that must reuse the now-transient materialized address base. */ 938 in = emit_with_ops(f, block, IR_LOAD_IMM, 1u); 939 in->opnds[0] = stack_op(base_slot, ptr_type); 940 in->extra.imm = 0x1000; 941 942 in = emit_with_ops(f, block, IR_LOAD_IMM, 1u); 943 in->opnds[0] = stack_op(index_slot, i64_type); 944 in->extra.imm = 3; 945 946 in = emit_with_ops(f, block, IR_LOAD, 2u); 947 in->opnds[0] = stack_op(result_slot, i64_type); 948 in->opnds[1] = 949 indirect_frame_index_op(base_slot, index_slot, ptr_type, i64_type); 950 in->extra.mem.type = i64_type; 951 in->extra.mem.size = 8u; 952 in->extra.mem.align = 8u; 953 954 mock_init(&mock, unit, c); 955 opt_emit_native(c, f, &mock.base); 956 957 CU_EXPECT(unit, 958 mock.load_addrs == 1u && mock.reg_address_loads == 1u && 959 mock.in_place_reg_address_loads == 1u, 960 "frame load should materialize once and consume the scoped base " 961 "in place (addr=%u reg-load=%u in-place=%u)", 962 mock.load_addrs, mock.reg_address_loads, 963 mock.in_place_reg_address_loads); 964 CU_EXPECT(unit, (mock.writeback_slots & (1u << result_slot)) != 0u, 965 "in-place frame load result was not written back"); 966 967 kit_compiler_free(kit_c); 968 } 969 970 static void frame_address_reuses_scoped_base_under_pressure(KitUnit *unit) { 971 KitCompiler *kit_c = NULL; 972 Compiler *c; 973 KitCgTypeId i64_type, ptr_type; 974 OptCGFuncDesc desc; 975 Func *f; 976 MockNative mock; 977 FrameSlot base_slot, index_slot, result_slot; 978 Inst *in; 979 u32 block; 980 981 CU_EXPECT(unit, 982 kit_unit_compiler_new( 983 unit, 984 kit_unit_target(KIT_ARCH_X86_64, KIT_OS_LINUX, KIT_OBJ_ELF), 985 &kit_c) == KIT_OK && 986 kit_c != NULL, 987 "compiler allocation failed for scoped address-result test"); 988 if (!kit_c) return; 989 c = (Compiler *)kit_c; 990 i64_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_I64); 991 ptr_type = kit_cg_type_ptr( 992 kit_c, kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID), 0); 993 994 memset(&desc, 0, sizeof desc); 995 desc.result_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID); 996 f = ir_func_new(c, &desc); 997 block = ir_block_new(f); 998 f->entry = block; 999 ir_note_emit(f, block); 1000 f->opt_rewritten = 1; 1001 f->emit_temp_regs[RC_INT][0] = 8u; 1002 f->emit_temp_regs[RC_INT][1] = 9u; 1003 f->emit_temp_reg_count[RC_INT] = 2u; 1004 f->opt_reserved_regs[RC_INT] = (1u << 8u) | (1u << 9u); 1005 base_slot = add_slot(f, ptr_type, 8u, 8u); 1006 index_slot = add_slot(f, i64_type, 8u, 8u); 1007 result_slot = add_slot(f, ptr_type, 8u, 8u); 1008 1009 in = emit_with_ops(f, block, IR_LOAD_IMM, 1u); 1010 in->opnds[0] = stack_op(base_slot, ptr_type); 1011 in->extra.imm = 0x2000; 1012 1013 in = emit_with_ops(f, block, IR_LOAD_IMM, 1u); 1014 in->opnds[0] = stack_op(index_slot, i64_type); 1015 in->extra.imm = 5; 1016 1017 in = emit_with_ops(f, block, IR_ADDR_OF, 2u); 1018 in->opnds[0] = stack_op(result_slot, ptr_type); 1019 in->opnds[1] = 1020 indirect_frame_index_op(base_slot, index_slot, ptr_type, i64_type); 1021 1022 mock_init(&mock, unit, c); 1023 opt_emit_native(c, f, &mock.base); 1024 1025 CU_EXPECT(unit, mock.load_addrs == 1u, 1026 "frame address should collapse into its scoped base exactly once " 1027 "(materializations=%u)", 1028 mock.load_addrs); 1029 CU_EXPECT(unit, (mock.writeback_slots & (1u << result_slot)) != 0u, 1030 "in-place frame address result was not written back"); 1031 1032 kit_compiler_free(kit_c); 1033 } 1034 1035 static void collapsed_address_releases_index_for_store_value(KitUnit *unit) { 1036 KitCompiler *kit_c = NULL; 1037 Compiler *c; 1038 KitCgTypeId i64_type, ptr_type; 1039 OptCGFuncDesc desc; 1040 Func *f; 1041 MockNative mock; 1042 FrameSlot base_slot, index_slot; 1043 Inst *in; 1044 u32 block; 1045 1046 CU_EXPECT(unit, 1047 kit_unit_compiler_new( 1048 unit, 1049 kit_unit_target(KIT_ARCH_X86_64, KIT_OS_LINUX, KIT_OBJ_ELF), 1050 &kit_c) == KIT_OK && 1051 kit_c != NULL, 1052 "compiler allocation failed for collapsed-address store test"); 1053 if (!kit_c) return; 1054 c = (Compiler *)kit_c; 1055 i64_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_I64); 1056 ptr_type = kit_cg_type_ptr( 1057 kit_c, kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID), 0); 1058 1059 memset(&desc, 0, sizeof desc); 1060 desc.result_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID); 1061 f = ir_func_new(c, &desc); 1062 block = ir_block_new(f); 1063 f->entry = block; 1064 ir_note_emit(f, block); 1065 f->opt_rewritten = 1; 1066 f->emit_temp_regs[RC_INT][0] = 8u; 1067 f->emit_temp_regs[RC_INT][1] = 9u; 1068 f->emit_temp_reg_count[RC_INT] = 2u; 1069 f->opt_reserved_regs[RC_INT] = (1u << 8u) | (1u << 9u); 1070 base_slot = add_slot(f, ptr_type, 8u, 8u); 1071 index_slot = add_slot(f, i64_type, 8u, 8u); 1072 1073 in = emit_with_ops(f, block, IR_LOAD_IMM, 1u); 1074 in->opnds[0] = stack_op(base_slot, ptr_type); 1075 in->extra.imm = 0x3000; 1076 1077 in = emit_with_ops(f, block, IR_LOAD_IMM, 1u); 1078 in->opnds[0] = stack_op(index_slot, i64_type); 1079 in->extra.imm = 7; 1080 1081 in = emit_with_ops(f, block, IR_STORE, 2u); 1082 in->opnds[0] = 1083 indirect_frame_index_op(base_slot, index_slot, ptr_type, i64_type); 1084 in->opnds[1] = imm_op(42, i64_type); 1085 in->extra.mem.type = i64_type; 1086 in->extra.mem.size = 8u; 1087 in->extra.mem.align = 8u; 1088 1089 mock_init(&mock, unit, c); 1090 opt_emit_native(c, f, &mock.base); 1091 1092 CU_EXPECT(unit, mock.load_addrs == 1u && mock.reg_address_stores == 1u, 1093 "collapsed store should reuse the dead index lease for its value " 1094 "(addr=%u stores=%u)", 1095 mock.load_addrs, mock.reg_address_stores); 1096 1097 kit_compiler_free(kit_c); 1098 } 1099 1100 static void frame_bitfield_load_reuses_scoped_address_base(KitUnit *unit) { 1101 KitCompiler *kit_c = NULL; 1102 Compiler *c; 1103 KitCgTypeId i64_type, ptr_type; 1104 OptCGFuncDesc desc; 1105 Func *f; 1106 MockNative mock; 1107 FrameSlot base_slot, index_slot, result_slot; 1108 IRBitFieldAux *aux; 1109 Inst *in; 1110 u32 block; 1111 1112 CU_EXPECT(unit, 1113 kit_unit_compiler_new( 1114 unit, 1115 kit_unit_target(KIT_ARCH_X86_64, KIT_OS_LINUX, KIT_OBJ_ELF), 1116 &kit_c) == KIT_OK && 1117 kit_c != NULL, 1118 "compiler allocation failed for bitfield pressure test"); 1119 if (!kit_c) return; 1120 c = (Compiler *)kit_c; 1121 i64_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_I64); 1122 ptr_type = kit_cg_type_ptr( 1123 kit_c, kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID), 0); 1124 1125 memset(&desc, 0, sizeof desc); 1126 desc.result_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID); 1127 f = ir_func_new(c, &desc); 1128 block = ir_block_new(f); 1129 f->entry = block; 1130 ir_note_emit(f, block); 1131 f->opt_rewritten = 1; 1132 f->emit_temp_regs[RC_INT][0] = 8u; 1133 f->emit_temp_regs[RC_INT][1] = 9u; 1134 f->emit_temp_reg_count[RC_INT] = 2u; 1135 f->opt_reserved_regs[RC_INT] = (1u << 8u) | (1u << 9u); 1136 base_slot = add_slot(f, ptr_type, 8u, 8u); 1137 index_slot = add_slot(f, i64_type, 8u, 8u); 1138 result_slot = add_slot(f, i64_type, 8u, 8u); 1139 1140 in = emit_with_ops(f, block, IR_LOAD_IMM, 1u); 1141 in->opnds[0] = stack_op(base_slot, ptr_type); 1142 in->extra.imm = 0x4000; 1143 1144 in = emit_with_ops(f, block, IR_LOAD_IMM, 1u); 1145 in->opnds[0] = stack_op(index_slot, i64_type); 1146 in->extra.imm = 9; 1147 1148 in = emit_with_ops(f, block, IR_BITFIELD_LOAD, 2u); 1149 in->opnds[0] = stack_op(result_slot, i64_type); 1150 in->opnds[1] = 1151 indirect_frame_index_op(base_slot, index_slot, ptr_type, i64_type); 1152 aux = arena_znew(f->arena, IRBitFieldAux); 1153 aux->access.field_type = i64_type; 1154 aux->access.storage.type = i64_type; 1155 aux->access.storage.size = 8u; 1156 aux->access.storage.align = 8u; 1157 aux->access.bit_offset = 3u; 1158 aux->access.bit_width = 11u; 1159 in->extra.aux = aux; 1160 1161 mock_init(&mock, unit, c); 1162 opt_emit_native(c, f, &mock.base); 1163 1164 CU_EXPECT(unit, 1165 mock.load_addrs == 1u && 1166 mock.in_place_reg_address_bitfield_loads == 1u, 1167 "bitfield load should consume its scoped address base in place " 1168 "(addr=%u in-place=%u)", 1169 mock.load_addrs, mock.in_place_reg_address_bitfield_loads); 1170 CU_EXPECT(unit, (mock.writeback_slots & (1u << result_slot)) != 0u, 1171 "in-place bitfield result was not written back"); 1172 1173 kit_compiler_free(kit_c); 1174 } 1175 1176 static void in_place_collapse_drops_overwritten_cache_fact(KitUnit *unit) { 1177 KitCompiler *kit_c = NULL; 1178 Compiler *c; 1179 KitCgTypeId i64_type, ptr_type; 1180 OptCGFuncDesc desc; 1181 Func *f; 1182 MockNative mock; 1183 FrameSlot base_slot, index_slot; 1184 IRAtomicAux *aux; 1185 Inst *in; 1186 u32 block; 1187 1188 CU_EXPECT(unit, 1189 kit_unit_compiler_new( 1190 unit, 1191 kit_unit_target(KIT_ARCH_X86_64, KIT_OS_LINUX, KIT_OBJ_ELF), 1192 &kit_c) == KIT_OK && 1193 kit_c != NULL, 1194 "compiler allocation failed for collapse cache-fact test"); 1195 if (!kit_c) return; 1196 c = (Compiler *)kit_c; 1197 i64_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_I64); 1198 ptr_type = kit_cg_type_ptr( 1199 kit_c, kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID), 0); 1200 1201 memset(&desc, 0, sizeof desc); 1202 desc.result_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID); 1203 f = ir_func_new(c, &desc); 1204 block = ir_block_new(f); 1205 f->entry = block; 1206 ir_note_emit(f, block); 1207 f->opt_rewritten = 1; 1208 f->emit_temp_regs[RC_INT][0] = 8u; 1209 f->emit_temp_regs[RC_INT][1] = 9u; 1210 f->emit_temp_reg_count[RC_INT] = 2u; 1211 f->opt_reserved_regs[RC_INT] = (1u << 8u) | (1u << 9u); 1212 base_slot = add_slot(f, ptr_type, 8u, 8u); 1213 index_slot = add_slot(f, i64_type, 8u, 8u); 1214 1215 in = emit_with_ops(f, block, IR_LOAD_IMM, 1u); 1216 in->opnds[0] = stack_op(base_slot, ptr_type); 1217 in->extra.imm = 0x5000; 1218 1219 in = emit_with_ops(f, block, IR_LOAD_IMM, 1u); 1220 in->opnds[0] = stack_op(index_slot, i64_type); 1221 in->extra.imm = 2; 1222 1223 /* The address consumes both cached spill values and is collapsed in place 1224 * into the cached base register. The atomic value then requests that same 1225 * base spill again without an address-avoidance rider. It must reload the 1226 * authoritative frame home; the register now contains base+index, not the 1227 * cached base value. */ 1228 in = emit_with_ops(f, block, IR_ATOMIC_STORE, 2u); 1229 in->opnds[0] = 1230 indirect_frame_index_op(base_slot, index_slot, ptr_type, i64_type); 1231 in->opnds[1] = stack_op(base_slot, ptr_type); 1232 aux = arena_znew(f->arena, IRAtomicAux); 1233 aux->mem.type = ptr_type; 1234 aux->mem.size = 8u; 1235 aux->mem.align = 8u; 1236 aux->mo = KIT_CG_MO_SEQ_CST; 1237 in->extra.aux = aux; 1238 1239 mock_init(&mock, unit, c); 1240 mock.reject_indexed_addr = 1u; 1241 opt_emit_native(c, f, &mock.base); 1242 1243 CU_EXPECT(unit, 1244 mock.load_addrs == 1u && mock.atomic_stores == 1u && 1245 mock.frame_loads == 1u, 1246 "overwritten base source should rematerialize exactly once " 1247 "(addr=%u atomic=%u reloads=%u)", 1248 mock.load_addrs, mock.atomic_stores, mock.frame_loads); 1249 1250 kit_compiler_free(kit_c); 1251 } 1252 1253 static void frame_atomic_load_reuses_scoped_address_base(KitUnit *unit) { 1254 KitCompiler *kit_c = NULL; 1255 Compiler *c; 1256 KitCgTypeId i64_type, ptr_type; 1257 OptCGFuncDesc desc; 1258 Func *f; 1259 MockNative mock; 1260 u32 block; 1261 FrameSlot base_slot, index_slot, dst_slot; 1262 Inst *in; 1263 IRAtomicAux *aux; 1264 1265 CU_EXPECT(unit, 1266 kit_unit_compiler_new( 1267 unit, 1268 kit_unit_target(KIT_ARCH_X86_64, KIT_OS_LINUX, KIT_OBJ_ELF), 1269 &kit_c) == KIT_OK && 1270 kit_c != NULL, 1271 "compiler allocation failed for atomic-load pressure test"); 1272 if (!kit_c) return; 1273 c = (Compiler *)kit_c; 1274 i64_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_I64); 1275 ptr_type = kit_cg_type_ptr( 1276 kit_c, kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID), 0); 1277 1278 memset(&desc, 0, sizeof desc); 1279 desc.result_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID); 1280 f = ir_func_new(c, &desc); 1281 block = ir_block_new(f); 1282 f->entry = block; 1283 ir_note_emit(f, block); 1284 f->opt_rewritten = 1; 1285 f->emit_temp_regs[RC_INT][0] = 8u; 1286 f->emit_temp_regs[RC_INT][1] = 9u; 1287 f->emit_temp_reg_count[RC_INT] = 2u; 1288 f->opt_reserved_regs[RC_INT] = (1u << 8u) | (1u << 9u); 1289 base_slot = add_slot(f, ptr_type, 8u, 8u); 1290 index_slot = add_slot(f, i64_type, 8u, 8u); 1291 dst_slot = add_slot(f, i64_type, 8u, 8u); 1292 1293 in = emit_with_ops(f, block, IR_LOAD_IMM, 1u); 1294 in->opnds[0] = stack_op(base_slot, ptr_type); 1295 in->extra.imm = 0x5000; 1296 in = emit_with_ops(f, block, IR_LOAD_IMM, 1u); 1297 in->opnds[0] = stack_op(index_slot, i64_type); 1298 in->extra.imm = 2; 1299 1300 in = emit_with_ops(f, block, IR_ATOMIC_LOAD, 2u); 1301 in->opnds[0] = stack_op(dst_slot, i64_type); 1302 in->opnds[1] = 1303 indirect_frame_index_op(base_slot, index_slot, ptr_type, i64_type); 1304 aux = arena_znew(f->arena, IRAtomicAux); 1305 aux->mem.type = i64_type; 1306 aux->mem.size = 8u; 1307 aux->mem.align = 8u; 1308 aux->mo = KIT_CG_MO_SEQ_CST; 1309 in->extra.aux = aux; 1310 1311 mock_init(&mock, unit, c); 1312 opt_emit_native(c, f, &mock.base); 1313 1314 CU_EXPECT(unit, mock.load_addrs == 1u && mock.atomic_loads == 1u, 1315 "atomic load should collapse once and reach its hook once " 1316 "(addr=%u atomic=%u)", 1317 mock.load_addrs, mock.atomic_loads); 1318 kit_compiler_free(kit_c); 1319 } 1320 1321 static void atomic_store_collapses_address_for_value_temp(KitUnit *unit) { 1322 KitCompiler *kit_c = NULL; 1323 Compiler *c; 1324 KitCgTypeId i64_type, ptr_type; 1325 OptCGFuncDesc desc; 1326 Func *f; 1327 MockNative mock; 1328 u32 block; 1329 FrameSlot base_slot, index_slot; 1330 Inst *in; 1331 IRAtomicAux *aux; 1332 1333 CU_EXPECT(unit, 1334 kit_unit_compiler_new( 1335 unit, 1336 kit_unit_target(KIT_ARCH_X86_64, KIT_OS_LINUX, KIT_OBJ_ELF), 1337 &kit_c) == KIT_OK && 1338 kit_c != NULL, 1339 "compiler allocation failed for atomic-store pressure test"); 1340 if (!kit_c) return; 1341 c = (Compiler *)kit_c; 1342 i64_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_I64); 1343 ptr_type = kit_cg_type_ptr( 1344 kit_c, kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID), 0); 1345 1346 memset(&desc, 0, sizeof desc); 1347 desc.result_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID); 1348 f = ir_func_new(c, &desc); 1349 block = ir_block_new(f); 1350 f->entry = block; 1351 ir_note_emit(f, block); 1352 f->opt_rewritten = 1; 1353 f->emit_temp_regs[RC_INT][0] = 8u; 1354 f->emit_temp_regs[RC_INT][1] = 9u; 1355 f->emit_temp_reg_count[RC_INT] = 2u; 1356 f->opt_reserved_regs[RC_INT] = (1u << 8u) | (1u << 9u); 1357 base_slot = add_slot(f, ptr_type, 8u, 8u); 1358 index_slot = add_slot(f, i64_type, 8u, 8u); 1359 1360 in = emit_with_ops(f, block, IR_LOAD_IMM, 1u); 1361 in->opnds[0] = stack_op(base_slot, ptr_type); 1362 in->extra.imm = 0x5000; 1363 in = emit_with_ops(f, block, IR_LOAD_IMM, 1u); 1364 in->opnds[0] = stack_op(index_slot, i64_type); 1365 in->extra.imm = 2; 1366 1367 in = emit_with_ops(f, block, IR_ATOMIC_STORE, 2u); 1368 in->opnds[0] = 1369 indirect_frame_index_op(base_slot, index_slot, ptr_type, i64_type); 1370 in->opnds[1] = imm_op(42, i64_type); 1371 aux = arena_znew(f->arena, IRAtomicAux); 1372 aux->mem.type = i64_type; 1373 aux->mem.size = 8u; 1374 aux->mem.align = 8u; 1375 aux->mo = KIT_CG_MO_SEQ_CST; 1376 in->extra.aux = aux; 1377 1378 mock_init(&mock, unit, c); 1379 opt_emit_native(c, f, &mock.base); 1380 1381 CU_EXPECT(unit, mock.load_addrs == 1u && mock.atomic_stores == 1u, 1382 "atomic store should collapse once and reach its hook once " 1383 "(addr=%u atomic=%u)", 1384 mock.load_addrs, mock.atomic_stores); 1385 kit_compiler_free(kit_c); 1386 } 1387 1388 static void allocator_cache_waits_for_hard_liveness(KitUnit *unit) { 1389 KitCompiler *kit_c = NULL; 1390 Compiler *c; 1391 KitCgTypeId i64_type, ptr_type; 1392 OptCGFuncDesc desc; 1393 Func *f; 1394 MockNative mock; 1395 FrameSlot clobbered_slot, preserved_slot, record_slot; 1396 Inst *in, *clobber; 1397 NativeMachineOp mop; 1398 u32 clobber_mask[NATIVE_REG_CLASS_COUNT]; 1399 u32 block; 1400 1401 CU_EXPECT(unit, 1402 kit_unit_compiler_new( 1403 unit, 1404 kit_unit_target(KIT_ARCH_X86_64, KIT_OS_LINUX, KIT_OBJ_ELF), 1405 &kit_c) == KIT_OK && 1406 kit_c != NULL, 1407 "compiler allocation failed for allocator-cache clobber test"); 1408 if (!kit_c) 1409 return; 1410 c = (Compiler *)kit_c; 1411 i64_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_I64); 1412 ptr_type = kit_cg_type_ptr( 1413 kit_c, kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID), 0); 1414 1415 memset(&desc, 0, sizeof desc); 1416 desc.result_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID); 1417 f = ir_func_new(c, &desc); 1418 block = ir_block_new(f); 1419 f->entry = block; 1420 ir_note_emit(f, block); 1421 f->opt_rewritten = 1; 1422 f->emit_temp_regs[RC_INT][0] = 8u; 1423 f->emit_temp_regs[RC_INT][1] = 9u; 1424 f->emit_temp_reg_count[RC_INT] = 2u; 1425 f->opt_reserved_regs[RC_INT] = (1u << 8u) | (1u << 9u); 1426 f->opt_hard_regs[RC_INT][0] = 10u; 1427 f->opt_hard_regs[RC_INT][1] = 11u; 1428 f->opt_hard_regs[RC_INT][2] = 12u; 1429 f->opt_hard_reg_count[RC_INT] = 3u; 1430 f->opt_caller_saved[RC_INT] = 1431 (1u << 10u) | (1u << 11u) | (1u << 12u); 1432 clobbered_slot = add_slot(f, i64_type, 8u, 8u); 1433 preserved_slot = add_slot(f, i64_type, 8u, 8u); 1434 record_slot = add_slot(f, i64_type, 8u, 8u); 1435 1436 /* Even though r10 is dead after this writeback, lazy native emission has not 1437 * needed hard-liveness yet. It must leave the authoritative frame copy alone 1438 * rather than retain and later lease an ordinary allocation register. */ 1439 in = emit_with_ops(f, block, IR_COPY, 2u); 1440 in->opnds[0] = stack_op(clobbered_slot, i64_type); 1441 in->opnds[1] = reg_op(10u, i64_type); 1442 1443 clobber = emit_with_ops(f, block, IR_BITFIELD_STORE, 2u); 1444 clobber->opnds[0] = frame_addr_op(record_slot, ptr_type); 1445 clobber->opnds[1] = imm_op(7, i64_type); 1446 { 1447 IRBitFieldAux *aux = arena_znew(f->arena, IRBitFieldAux); 1448 aux->access.field_type = i64_type; 1449 aux->access.storage.type = i64_type; 1450 aux->access.storage.size = 8u; 1451 aux->access.storage.align = 8u; 1452 aux->access.bit_offset = 3u; 1453 aux->access.bit_width = 11u; 1454 clobber->extra.aux = aux; 1455 } 1456 1457 in = emit_with_ops(f, block, IR_COPY, 2u); 1458 in->opnds[0] = reg_op(12u, i64_type); 1459 in->opnds[1] = stack_op(clobbered_slot, i64_type); 1460 1461 /* Establish a new, independently dead r10 value. The ordinary label hook 1462 * preserves it, but without hard-liveness the emitter must still reload the 1463 * frame rather than turn r10 into a persistent cache register. */ 1464 in = emit_with_ops(f, block, IR_LOAD_IMM, 1u); 1465 in->opnds[0] = reg_op(10u, i64_type); 1466 in->extra.imm = 99; 1467 1468 in = emit_with_ops(f, block, IR_COPY, 2u); 1469 in->opnds[0] = stack_op(preserved_slot, i64_type); 1470 in->opnds[1] = reg_op(10u, i64_type); 1471 1472 in = emit_with_ops(f, block, IR_LOAD_LABEL_ADDR, 1u); 1473 in->opnds[0] = reg_op(11u, ptr_type); 1474 in->extra.imm = block; 1475 1476 in = emit_with_ops(f, block, IR_COPY, 2u); 1477 in->opnds[0] = reg_op(12u, i64_type); 1478 in->opnds[1] = stack_op(preserved_slot, i64_type); 1479 1480 mock_init(&mock, unit, c); 1481 mock.report_bitfield_store_clobber = 1u; 1482 /* The O0 direct-target value bank is deliberately empty. In particular, it 1483 * must not make the machinized allocation registers cache-eligible before 1484 * the lazy hard-liveness table has actually been built. */ 1485 mock.int_phys[0].reg = 10u; 1486 mock.int_phys[0].cls = NATIVE_REG_INT; 1487 mock.int_phys[0].flags = NATIVE_REG_CALLER_SAVED; 1488 mock.classes[NATIVE_REG_INT].phys = mock.int_phys; 1489 mock.classes[NATIVE_REG_INT].nphys = 1u; 1490 1491 memset(&mop, 0, sizeof mop); 1492 mop.kind = NATIVE_MOP_BITFIELD_STORE; 1493 CU_EXPECT(unit, 1494 mock.base.machine_op_clobbers(&mock.base, &mop, clobber_mask) && 1495 clobber_mask[NATIVE_REG_INT] == (1u << 10u), 1496 "mock bitfield-store effect must report its allocator clobber"); 1497 f->inst_clobbers_cap = f->next_inst_id; 1498 f->inst_clobbers = 1499 arena_zarray(f->arena, OptInstClobberMask, f->inst_clobbers_cap); 1500 CU_EXPECT(unit, 1501 clobber->id != INST_ID_NONE && 1502 clobber->id < f->inst_clobbers_cap, 1503 "bitfield store must have a valid clobber-table instruction id"); 1504 if (clobber->id != INST_ID_NONE && clobber->id < f->inst_clobbers_cap) 1505 for (u32 cls = 0; cls < OPT_REG_CLASSES; ++cls) 1506 f->inst_clobbers[clobber->id][cls] = clobber_mask[cls]; 1507 1508 opt_emit_native(c, f, &mock.base); 1509 1510 CU_EXPECT(unit, 1511 (mock.callback_mask & 1512 (SAW_BITFIELD_STORE | SAW_LABEL_ADDR)) == 1513 (SAW_BITFIELD_STORE | SAW_LABEL_ADDR), 1514 "clobbering and ordinary preservation hooks must both execute"); 1515 CU_EXPECT(unit, mock.frame_loads == 2u && mock.loads == 2u, 1516 "allocation registers must not be retained before lazy " 1517 "hard-liveness is available (loads=%u frame-loads=%u)", 1518 mock.loads, mock.frame_loads); 1519 1520 kit_compiler_free(kit_c); 1521 } 1522 1523 static void reserved_cache_obeys_machine_clobbers(KitUnit *unit) { 1524 KitCompiler *kit_c = NULL; 1525 Compiler *c; 1526 KitCgTypeId i64_type, ptr_type; 1527 OptCGFuncDesc desc; 1528 Func *f; 1529 MockNative mock; 1530 FrameSlot clobbered_slot, preserved_slot, record_slot; 1531 Inst *in, *clobber; 1532 NativeMachineOp mop; 1533 u32 clobber_mask[NATIVE_REG_CLASS_COUNT]; 1534 u32 block; 1535 1536 CU_EXPECT(unit, 1537 kit_unit_compiler_new( 1538 unit, 1539 kit_unit_target(KIT_ARCH_X86_64, KIT_OS_LINUX, KIT_OBJ_ELF), 1540 &kit_c) == KIT_OK && 1541 kit_c != NULL, 1542 "compiler allocation failed for reserved-cache clobber test"); 1543 if (!kit_c) return; 1544 c = (Compiler *)kit_c; 1545 i64_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_I64); 1546 ptr_type = kit_cg_type_ptr( 1547 kit_c, kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID), 0); 1548 1549 memset(&desc, 0, sizeof desc); 1550 desc.result_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID); 1551 f = ir_func_new(c, &desc); 1552 block = ir_block_new(f); 1553 f->entry = block; 1554 ir_note_emit(f, block); 1555 f->opt_rewritten = 1; 1556 f->emit_temp_regs[RC_INT][0] = 8u; 1557 f->emit_temp_regs[RC_INT][1] = 9u; 1558 f->emit_temp_reg_count[RC_INT] = 2u; 1559 f->opt_reserved_regs[RC_INT] = (1u << 8u) | (1u << 9u); 1560 clobbered_slot = add_slot(f, i64_type, 8u, 8u); 1561 preserved_slot = add_slot(f, i64_type, 8u, 8u); 1562 record_slot = add_slot(f, i64_type, 8u, 8u); 1563 1564 /* The stack destination is staged through r8, then retained because r8 is 1565 * in the target's explicit emit_cache_mask. The following machine hook says 1566 * it overwrites r8. This happens on the lazy fast path, before the scavenger 1567 * has requested hard-liveness, and must still invalidate the cache fact. */ 1568 in = emit_with_ops(f, block, IR_LOAD_IMM, 1u); 1569 in->opnds[0] = stack_op(clobbered_slot, i64_type); 1570 in->extra.imm = 41; 1571 1572 clobber = emit_with_ops(f, block, IR_BITFIELD_STORE, 2u); 1573 clobber->opnds[0] = frame_addr_op(record_slot, ptr_type); 1574 clobber->opnds[1] = imm_op(7, i64_type); 1575 { 1576 IRBitFieldAux *aux = arena_znew(f->arena, IRBitFieldAux); 1577 aux->access.field_type = i64_type; 1578 aux->access.storage.type = i64_type; 1579 aux->access.storage.size = 8u; 1580 aux->access.storage.align = 8u; 1581 aux->access.bit_offset = 3u; 1582 aux->access.bit_width = 11u; 1583 clobber->extra.aux = aux; 1584 } 1585 1586 in = emit_with_ops(f, block, IR_COPY, 2u); 1587 in->opnds[0] = reg_op(10u, i64_type); 1588 in->opnds[1] = stack_op(clobbered_slot, i64_type); 1589 1590 /* Establish a second r8 cache fact and cross a hook with no r8 effect. The 1591 * reload should forward from r8, proving the focused invalidation above did 1592 * not degenerate into clearing every cache entry on every instruction. */ 1593 in = emit_with_ops(f, block, IR_LOAD_IMM, 1u); 1594 in->opnds[0] = stack_op(preserved_slot, i64_type); 1595 in->extra.imm = 99; 1596 1597 in = emit_with_ops(f, block, IR_LOAD_LABEL_ADDR, 1u); 1598 in->opnds[0] = reg_op(11u, ptr_type); 1599 in->extra.imm = block; 1600 1601 in = emit_with_ops(f, block, IR_COPY, 2u); 1602 in->opnds[0] = reg_op(12u, i64_type); 1603 in->opnds[1] = stack_op(preserved_slot, i64_type); 1604 1605 mock_init(&mock, unit, c); 1606 mock.report_bitfield_store_clobber = 1u; 1607 mock.machine_clobber_reg = 8u; 1608 memset(&mop, 0, sizeof mop); 1609 mop.kind = NATIVE_MOP_BITFIELD_STORE; 1610 CU_EXPECT(unit, 1611 mock.base.machine_op_clobbers(&mock.base, &mop, clobber_mask) && 1612 clobber_mask[NATIVE_REG_INT] == (1u << 8u), 1613 "mock bitfield-store effect must report reserved cache r8"); 1614 f->inst_clobbers_cap = f->next_inst_id; 1615 f->inst_clobbers = 1616 arena_zarray(f->arena, OptInstClobberMask, f->inst_clobbers_cap); 1617 CU_EXPECT(unit, 1618 clobber->id != INST_ID_NONE && 1619 clobber->id < f->inst_clobbers_cap, 1620 "bitfield store must have a valid clobber-table instruction id"); 1621 if (clobber->id != INST_ID_NONE && clobber->id < f->inst_clobbers_cap) 1622 for (u32 cls = 0; cls < OPT_REG_CLASSES; ++cls) 1623 f->inst_clobbers[clobber->id][cls] = clobber_mask[cls]; 1624 1625 opt_emit_native(c, f, &mock.base); 1626 1627 CU_EXPECT(unit, 1628 (mock.callback_mask & 1629 (SAW_BITFIELD_STORE | SAW_LABEL_ADDR)) == 1630 (SAW_BITFIELD_STORE | SAW_LABEL_ADDR), 1631 "clobbering and preserving hooks must both execute"); 1632 CU_EXPECT(unit, mock.frame_loads == 1u && mock.loads == 1u, 1633 "reserved r8 clobber must reload only the affected frame value " 1634 "(loads=%u frame-loads=%u)", 1635 mock.loads, mock.frame_loads); 1636 1637 kit_compiler_free(kit_c); 1638 } 1639 1640 static void call_ends_pre_call_temp_phase(KitUnit *unit) { 1641 KitCompiler *kit_c = NULL; 1642 Compiler *c; 1643 KitCgTypeId i64_type, ptr_type; 1644 OptCGFuncDesc desc; 1645 Func *f; 1646 MockNative mock; 1647 FrameSlot arg_slot, result_slot; 1648 IRCallAux *aux; 1649 Inst *call; 1650 u32 block; 1651 1652 CU_EXPECT(unit, 1653 kit_unit_compiler_new( 1654 unit, 1655 kit_unit_target(KIT_ARCH_X86_64, KIT_OS_LINUX, KIT_OBJ_ELF), 1656 &kit_c) == KIT_OK && 1657 kit_c != NULL, 1658 "compiler allocation failed for call phase test"); 1659 if (!kit_c) 1660 return; 1661 c = (Compiler *)kit_c; 1662 i64_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_I64); 1663 ptr_type = kit_cg_type_ptr( 1664 kit_c, kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID), 0); 1665 1666 memset(&desc, 0, sizeof desc); 1667 desc.result_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID); 1668 f = ir_func_new(c, &desc); 1669 block = ir_block_new(f); 1670 f->entry = block; 1671 ir_note_emit(f, block); 1672 f->opt_rewritten = 1; 1673 f->emit_temp_regs[RC_INT][0] = 8u; 1674 f->emit_temp_regs[RC_INT][1] = 9u; 1675 f->emit_temp_reg_count[RC_INT] = 2u; 1676 f->opt_reserved_regs[RC_INT] = (1u << 8u) | (1u << 9u); 1677 f->opt_caller_saved[RC_INT] = (1u << 8u) | (1u << 9u); 1678 arg_slot = add_slot(f, i64_type, 8u, 8u); 1679 result_slot = add_slot(f, i64_type, 8u, 8u); 1680 1681 call = emit_with_ops(f, block, IR_CALL, 0u); 1682 aux = arena_znew(f->arena, IRCallAux); 1683 aux->desc.fn_type = i64_type; 1684 aux->desc.callee = global_op(17u, ptr_type); 1685 aux->desc.nargs = 1u; 1686 aux->desc.args = arena_zarray(f->arena, CGABIValue, 1u); 1687 aux->desc.args[0].type = i64_type; 1688 aux->desc.args[0].storage = stack_op(arg_slot, i64_type); 1689 aux->desc.ret.type = i64_type; 1690 aux->desc.ret.storage = stack_op(result_slot, i64_type); 1691 call->extra.aux = aux; 1692 1693 mock_init(&mock, unit, c); 1694 /* r9 materializes the frame argument into fixed ABI register r8. Both are 1695 * caller-clobbered, so neither pre-call fact may survive into the ret copy. 1696 */ 1697 opt_emit_native(c, f, &mock.base); 1698 1699 CU_EXPECT(unit, mock.calls == 1u, "mock call was not emitted exactly once"); 1700 CU_EXPECT(unit, mock.loads == 2u && mock.frame_loads == 2u, 1701 "post-call return copy reused a pre-call materialization " 1702 "(loads=%u frame-loads=%u)", 1703 mock.loads, mock.frame_loads); 1704 CU_EXPECT(unit, (mock.writeback_slots & (1u << result_slot)) != 0u, 1705 "post-call result was not written to its frame destination"); 1706 1707 kit_compiler_free(kit_c); 1708 } 1709 1710 static void return_parts_keep_prior_abi_destinations_live(KitUnit *unit) { 1711 KitCompiler *kit_c = NULL; 1712 Compiler *c; 1713 KitCgTypeId i64_type; 1714 OptCGFuncDesc desc; 1715 Func *f; 1716 MockNative mock; 1717 FrameSlot value_slot; 1718 IRRetAux *aux; 1719 Inst *ret; 1720 u32 block; 1721 1722 CU_EXPECT(unit, 1723 kit_unit_compiler_new( 1724 unit, 1725 kit_unit_target(KIT_ARCH_X86_64, KIT_OS_LINUX, KIT_OBJ_ELF), 1726 &kit_c) == KIT_OK && 1727 kit_c != NULL, 1728 "compiler allocation failed for multi-part return phase test"); 1729 if (!kit_c) 1730 return; 1731 c = (Compiler *)kit_c; 1732 i64_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_I64); 1733 1734 memset(&desc, 0, sizeof desc); 1735 desc.result_type = i64_type; 1736 f = ir_func_new(c, &desc); 1737 block = ir_block_new(f); 1738 f->entry = block; 1739 ir_note_emit(f, block); 1740 f->opt_rewritten = 1; 1741 for (u32 i = 0; i < 4u; ++i) { 1742 f->emit_temp_regs[RC_INT][i] = (Reg)(8u + i); 1743 f->opt_reserved_regs[RC_INT] |= 1u << (8u + i); 1744 } 1745 f->emit_temp_reg_count[RC_INT] = 4u; 1746 value_slot = add_slot(f, i64_type, 32u, 8u); 1747 1748 ret = emit_with_ops(f, block, IR_RET, 0u); 1749 aux = arena_znew(f->arena, IRRetAux); 1750 aux->present = 1u; 1751 aux->val.type = i64_type; 1752 aux->val.storage = stack_op(value_slot, i64_type); 1753 ret->extra.aux = aux; 1754 1755 mock_init(&mock, unit, c); 1756 mock.plan_four_part_ret = 1u; 1757 mock.int_temps[2] = 10u; 1758 mock.int_temps[3] = 11u; 1759 mock.classes[NATIVE_REG_INT].nscratch = 4u; 1760 mock.classes[NATIVE_REG_INT].nemit_temps = 4u; 1761 mock.classes[NATIVE_REG_INT].emit_cache_mask = 1762 (1u << 8u) | (1u << 9u) | (1u << 10u) | (1u << 11u); 1763 opt_emit_native(c, f, &mock.base); 1764 1765 CU_EXPECT(unit, mock.nframe_load_dsts == 4u, 1766 "four return parts should each load their frame source " 1767 "(loads=%u)", 1768 mock.nframe_load_dsts); 1769 if (mock.nframe_load_dsts == 4u) 1770 for (u32 i = 0; i < 4u; ++i) 1771 CU_EXPECT(unit, mock.frame_load_dsts[i] == (Reg)(8u + i), 1772 "return part %u should load directly into r%u", i, 1773 (unsigned)(8u + i)); 1774 CU_EXPECT(unit, mock.moves == 0u, 1775 "frame return parts should not route through emitter temps"); 1776 1777 kit_compiler_free(kit_c); 1778 } 1779 1780 static void zero_spill_uses_architectural_zero_register(KitUnit *unit) { 1781 KitCompiler *kit_c = NULL; 1782 Compiler *c; 1783 KitCgTypeId i64_type; 1784 OptCGFuncDesc desc; 1785 Func *f; 1786 MockNative mock; 1787 FrameSlot value_slot; 1788 Inst *load; 1789 u32 block; 1790 1791 CU_EXPECT(unit, 1792 kit_unit_compiler_new( 1793 unit, 1794 kit_unit_target(KIT_ARCH_ARM_64, KIT_OS_LINUX, KIT_OBJ_ELF), 1795 &kit_c) == KIT_OK && 1796 kit_c != NULL, 1797 "compiler allocation failed for zero-spill test"); 1798 if (!kit_c) 1799 return; 1800 c = (Compiler *)kit_c; 1801 i64_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_I64); 1802 1803 memset(&desc, 0, sizeof desc); 1804 desc.result_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID); 1805 f = ir_func_new(c, &desc); 1806 block = ir_block_new(f); 1807 f->entry = block; 1808 ir_note_emit(f, block); 1809 f->opt_rewritten = 1; 1810 f->emit_temp_regs[RC_INT][0] = 8u; 1811 f->emit_temp_reg_count[RC_INT] = 1u; 1812 value_slot = add_slot(f, i64_type, 8u, 8u); 1813 1814 load = emit_with_ops(f, block, IR_LOAD_IMM, 1u); 1815 load->opnds[0] = stack_op(value_slot, i64_type); 1816 load->extra.imm = 0; 1817 1818 mock_init(&mock, unit, c); 1819 mock.base.has_store_zero_reg = 1u; 1820 mock.base.store_zero_reg = 31u; 1821 opt_emit_native(c, f, &mock.base); 1822 1823 CU_EXPECT(unit, mock.load_imms == 0u, 1824 "zero spill should not materialize an emitter temporary"); 1825 CU_EXPECT(unit, mock.moves == 0u, 1826 "zero spill should not move through an emitter temporary"); 1827 CU_EXPECT(unit, (mock.writeback_slots & (1u << value_slot)) != 0u, 1828 "zero spill was not stored to its authoritative frame home"); 1829 1830 kit_compiler_free(kit_c); 1831 } 1832 1833 static void call_setup_owns_callee_and_completed_arguments(KitUnit *unit) { 1834 KitCompiler *kit_c = NULL; 1835 Compiler *c; 1836 KitCgTypeId i64_type, ptr_type; 1837 OptCGFuncDesc desc; 1838 Func *f; 1839 MockNative mock; 1840 FrameSlot callee_slot, arg_slots[2]; 1841 IRCallAux *aux; 1842 Inst *call; 1843 u32 block; 1844 1845 CU_EXPECT(unit, 1846 kit_unit_compiler_new( 1847 unit, 1848 kit_unit_target(KIT_ARCH_X86_64, KIT_OS_LINUX, KIT_OBJ_ELF), 1849 &kit_c) == KIT_OK && 1850 kit_c != NULL, 1851 "compiler allocation failed for call setup ownership test"); 1852 if (!kit_c) 1853 return; 1854 c = (Compiler *)kit_c; 1855 i64_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_I64); 1856 ptr_type = kit_cg_type_ptr( 1857 kit_c, kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID), 0); 1858 1859 memset(&desc, 0, sizeof desc); 1860 desc.result_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID); 1861 f = ir_func_new(c, &desc); 1862 block = ir_block_new(f); 1863 f->entry = block; 1864 ir_note_emit(f, block); 1865 f->opt_rewritten = 1; 1866 for (u32 i = 0; i < 4u; ++i) { 1867 f->emit_temp_regs[RC_INT][i] = (Reg)(8u + i); 1868 f->opt_reserved_regs[RC_INT] |= 1u << (8u + i); 1869 } 1870 f->emit_temp_reg_count[RC_INT] = 4u; 1871 callee_slot = add_slot(f, ptr_type, 8u, 8u); 1872 arg_slots[0] = add_slot(f, i64_type, 8u, 8u); 1873 arg_slots[1] = add_slot(f, i64_type, 8u, 8u); 1874 1875 call = emit_with_ops(f, block, IR_CALL, 0u); 1876 aux = arena_znew(f->arena, IRCallAux); 1877 aux->desc.fn_type = i64_type; 1878 aux->desc.callee = stack_op(callee_slot, ptr_type); 1879 aux->desc.nargs = 2u; 1880 aux->desc.args = arena_zarray(f->arena, CGABIValue, 2u); 1881 for (u32 i = 0; i < 2u; ++i) { 1882 aux->desc.args[i].type = i64_type; 1883 aux->desc.args[i].storage = stack_op(arg_slots[i], i64_type); 1884 } 1885 call->extra.aux = aux; 1886 1887 mock_init(&mock, unit, c); 1888 mock.plan_two_arg_indirect_call = 1u; 1889 mock.int_temps[2] = 10u; 1890 mock.int_temps[3] = 11u; 1891 mock.classes[NATIVE_REG_INT].nscratch = 4u; 1892 mock.classes[NATIVE_REG_INT].nemit_temps = 4u; 1893 mock.classes[NATIVE_REG_INT].emit_cache_mask = 1894 (1u << 8u) | (1u << 9u) | (1u << 10u) | (1u << 11u); 1895 opt_emit_native(c, f, &mock.base); 1896 1897 CU_EXPECT(unit, mock.calls == 1u, 1898 "two-argument indirect call was not emitted exactly once"); 1899 CU_EXPECT(unit, mock.nframe_load_dsts == 3u && mock.frame_loads == 3u, 1900 "callee and two arguments should each load once " 1901 "(recorded=%u frame-loads=%u)", 1902 mock.nframe_load_dsts, mock.frame_loads); 1903 1904 kit_compiler_free(kit_c); 1905 } 1906 1907 int main(void) { 1908 KitUnit unit; 1909 kit_unit_init(&unit); 1910 frame_destinations_are_staged(&unit); 1911 cached_frame_value_feeds_indirect_address(&unit); 1912 frame_load_reuses_scoped_address_base_under_pressure(&unit); 1913 frame_address_reuses_scoped_base_under_pressure(&unit); 1914 collapsed_address_releases_index_for_store_value(&unit); 1915 frame_bitfield_load_reuses_scoped_address_base(&unit); 1916 in_place_collapse_drops_overwritten_cache_fact(&unit); 1917 frame_atomic_load_reuses_scoped_address_base(&unit); 1918 atomic_store_collapses_address_for_value_temp(&unit); 1919 allocator_cache_waits_for_hard_liveness(&unit); 1920 reserved_cache_obeys_machine_clobbers(&unit); 1921 call_ends_pre_call_temp_phase(&unit); 1922 return_parts_keep_prior_abi_destinations_live(&unit); 1923 zero_spill_uses_architectural_zero_register(&unit); 1924 call_setup_owns_callee_and_completed_arguments(&unit); 1925 if (unit.fails) { 1926 fprintf(stderr, "native-emit-frame-dst: %d/%d failed\n", unit.fails, 1927 unit.checks); 1928 return 1; 1929 } 1930 printf("native-emit-frame-dst: %d checks, 0 failures\n", unit.checks); 1931 return 0; 1932 }