atomic.c (15405B)
1 #include "arch/arch.h" 2 #include "cg/internal.h" 3 4 MemAccess api_mem_for_atomic(KitCg* g, KitCgTypeId val_ty) { 5 MemAccess ma; 6 u32 size = 7 api_require_scalar_mem_type_resolved(g, "atomic memory access", val_ty); 8 if (size > CG_MAX_ATOMIC_SIZE) { 9 compiler_panic(g->c, g->cur_loc, 10 "KitCg: atomic memory access size exceeds 8 bytes"); 11 } 12 memset(&ma, 0, sizeof ma); 13 ma.type = val_ty; 14 ma.size = size; 15 ma.align = val_ty ? abi_cg_alignof(g->c->abi, val_ty) : 0; 16 ma.flags = MF_ATOMIC; 17 ma.alias.kind = (u8)ALIAS_UNKNOWN; 18 return ma; 19 } 20 21 /* Native (lock-free) atomic ceiling for the target, read from the arch backend 22 * descriptor (ArchImpl.atomic_lock_free_max). Most targets — aa64, x64, rv64, 23 * wasm32 — lower 8-byte (i64-width) atomics lock-free. rv32 reports 4: it has 24 * no native 64-bit atomic instructions (lr.d/sc.d/amo*.d are RV64-only), so 25 * 8-byte atomics there must go through the libatomic spinlock shim. (wasm32 has 26 * 4-byte pointers but still reports 8 — this is a per-arch capability, not a 27 * pointer-width test.) 28 * 29 * NOTE: this predicate is the single source of truth shared with the C 30 * front-end's __atomic_always_lock_free / __atomic_is_lock_free builtins (they 31 * route through kit_cg_atomic_is_lock_free below). Keeping it here guarantees 32 * that when kit cc compiles rt/lib/atomic/atomic_freestanding.c FOR rv32, the 33 * shim's IS_LOCK_FREE_8 test (__atomic_always_lock_free(8, p)) evaluates false, 34 * so the shim takes the spinlock path instead of recursing into an illegal 35 * native 8-byte atomic. */ 36 static u32 cg_atomic_lock_free_max(KitCompiler* c) { 37 const ArchImpl* a = arch_for_compiler(c); 38 return a ? a->atomic_lock_free_max : CG_MAX_ATOMIC_SIZE; 39 } 40 41 int kit_cg_atomic_is_legal(KitCompiler* c, KitCgMemAccess access, 42 KitCgMemOrder order) { 43 KitCgTypeId ty = resolve_type(c, access.type); 44 (void)order; 45 if (!ty) return 0; 46 if (cg_type_is_aggregate(c, ty) || cg_type_is_void(c, ty)) return 0; 47 /* Still legal up to 8 bytes everywhere: the libcall path makes 8-byte atomics 48 * available even when they are not lock-free. */ 49 return abi_cg_sizeof(c->abi, ty) <= CG_MAX_ATOMIC_SIZE; 50 } 51 52 int kit_cg_atomic_is_lock_free(KitCompiler* c, KitCgMemAccess access) { 53 KitCgTypeId ty = resolve_type(c, access.type); 54 if (!ty) return 0; 55 if (cg_type_is_aggregate(c, ty) || cg_type_is_void(c, ty)) return 0; 56 /* Lock-free up to the native atomic width, NOT the pointer width: wasm32 has 57 * 4-byte pointers but lowers 8-byte (i64) atomics lock-free, while rv32 does 58 * not have native 64-bit atomics. */ 59 return abi_cg_sizeof(c->abi, ty) <= cg_atomic_lock_free_max(c); 60 } 61 62 /* True when an atomic access of `val_ty` must be lowered to a libatomic 63 * (__atomic_*_8) libcall instead of a native instruction sequence. Today this 64 * is exactly the 8-byte-on-a-4-byte-target case (rv32). */ 65 static int cg_atomic_needs_libcall(KitCg* g, KitCgTypeId val_ty) { 66 return abi_cg_sizeof(g->c->abi, val_ty) == 8 && 67 cg_atomic_lock_free_max(g->c) < 8u; 68 } 69 70 /* Map a KitCgAtomicOp to the libatomic __atomic_fetch_<op>_8 / __atomic_*_8 71 * entry point. XCHG maps to __atomic_exchange_8. */ 72 static const char* cg_atomic_rmw_libcall_8(KitCgAtomicOp op) { 73 switch (op) { 74 case KIT_CG_ATOMIC_XCHG: 75 return "__atomic_exchange_8"; 76 case KIT_CG_ATOMIC_ADD: 77 return "__atomic_fetch_add_8"; 78 case KIT_CG_ATOMIC_SUB: 79 return "__atomic_fetch_sub_8"; 80 case KIT_CG_ATOMIC_AND: 81 return "__atomic_fetch_and_8"; 82 case KIT_CG_ATOMIC_OR: 83 return "__atomic_fetch_or_8"; 84 case KIT_CG_ATOMIC_XOR: 85 return "__atomic_fetch_xor_8"; 86 case KIT_CG_ATOMIC_NAND: 87 return "__atomic_fetch_nand_8"; 88 } 89 return NULL; 90 } 91 92 /* Declare a runtime function symbol with an arbitrary (<=5) param list. Mirrors 93 * api_runtime_helper (wide.c) but without its 3-param ceiling, which the 94 * 5-argument __atomic_compare_exchange_8 needs. */ 95 static KitCgSym cg_atomic_runtime_sym(KitCg* g, const char* name, 96 KitCgTypeId ret, 97 const KitCgTypeId* params, u32 nparams) { 98 KitCgFuncParam ps[5]; 99 KitCgFuncResult result; 100 KitCgFuncSig sig; 101 KitCgDecl decl; 102 if (nparams > 5) return KIT_CG_SYM_NONE; 103 memset(ps, 0, sizeof ps); 104 for (u32 i = 0; i < nparams; ++i) ps[i].type = params[i]; 105 memset(&sig, 0, sizeof sig); 106 memset(&result, 0, sizeof result); 107 /* A function result is always a valid type id post-cutover; a void helper 108 * (e.g. __atomic_store_8) is the void builtin, never KIT_CG_TYPE_NONE, which 109 * kit_cg_type_func rejects. Normalize a NONE/0 ret to the void builtin. */ 110 result.type = ret ? ret : builtin_id(KIT_CG_BUILTIN_VOID); 111 sig.result = result; 112 sig.params = ps; 113 sig.nparams = nparams; 114 sig.call_conv = KIT_CG_CC_TARGET_C; 115 memset(&decl, 0, sizeof decl); 116 decl.kind = KIT_CG_DECL_FUNC; 117 decl.linkage_name = kit_cg_c_linkage_name( 118 (KitCompiler*)g->c, 119 pool_intern_slice(g->c->global, slice_from_cstr(name))); 120 decl.display_name = decl.linkage_name; 121 decl.type = kit_cg_type_func((KitCompiler*)g->c, sig); 122 decl.sym.bind = KIT_SB_GLOBAL; 123 decl.sym.visibility = KIT_CG_VIS_DEFAULT; 124 return kit_cg_decl(g, decl); 125 } 126 127 /* Emit a runtime call: push args[0..nparams) then call. The single (optional) 128 * result is left on the value stack, matching api_runtime_call_values. */ 129 static void cg_atomic_runtime_call(KitCg* g, const char* name, KitCgTypeId ret, 130 const KitCgTypeId* params, u32 nparams, 131 ApiSValue* args) { 132 KitCgCallAttrs attrs; 133 KitCgSym sym = cg_atomic_runtime_sym(g, name, ret, params, nparams); 134 memset(&attrs, 0, sizeof attrs); 135 for (u32 i = 0; i < nparams; ++i) api_push(g, args[i]); 136 api_call_symbol_common(g, sym, nparams, attrs); 137 } 138 139 void kit_cg_atomic_load(KitCg* g, KitCgMemAccess access, KitCgMemOrder order) { 140 ApiSValue ptr; 141 KitCgTypeId pty, val_ty; 142 Operand addr, dst; 143 CGLocal rr; 144 if (!g) return; 145 api_local_const_memory_boundary(g); 146 ptr = api_pop(g); 147 pty = api_sv_type(&ptr); 148 val_ty = resolve_type(g->c, access.type); 149 if (!val_ty) val_ty = api_atomic_pointee(g, pty, "KitCg: atomic_load"); 150 if (api_unevaluated(g)) { 151 api_release(g, &ptr); 152 api_push(g, api_uneval_value(g, val_ty)); 153 api_const_set_top(g, api_const_unknown(val_ty)); 154 return; 155 } 156 api_require_pointer_value(g, "atomic_load pointer", pty); 157 if (cg_atomic_needs_libcall(g, val_ty)) { 158 /* u64 __atomic_load_8(const void* ptr, int memorder) */ 159 KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32); 160 KitCgTypeId ps[2]; 161 ApiSValue args[2]; 162 ps[0] = pty; 163 ps[1] = i32; 164 args[0] = ptr; 165 args[1] = api_make_sv(api_op_imm((i64)order, i32), i32); 166 cg_atomic_runtime_call(g, "__atomic_load_8", val_ty, ps, 2, args); 167 return; 168 } 169 addr = api_force_local(g, &ptr, pty); 170 rr = api_alloc_temp_local(g, val_ty); 171 dst = api_op_local(rr, val_ty); 172 g->target->atomic_load(g->target, dst, addr, api_mem_for_atomic(g, val_ty), 173 order); 174 api_release(g, &ptr); 175 api_push(g, api_make_sv(dst, val_ty)); 176 } 177 178 void kit_cg_atomic_store(KitCg* g, KitCgMemAccess access, KitCgMemOrder order) { 179 ApiSValue val, ptr; 180 KitCgTypeId pty, val_ty; 181 Operand addr, src; 182 if (!g) return; 183 api_local_const_memory_boundary(g); 184 val = api_pop(g); 185 ptr = api_pop(g); 186 pty = api_sv_type(&ptr); 187 val_ty = resolve_type(g->c, access.type); 188 if (!val_ty) val_ty = api_atomic_pointee(g, pty, "KitCg: atomic_store"); 189 if (api_unevaluated(g)) { 190 api_release(g, &val); 191 api_release(g, &ptr); 192 return; 193 } 194 api_require_pointer_value(g, "atomic_store pointer", pty); 195 api_validate_memory_value_resolved(g, "atomic_store", val_ty, 196 api_sv_type(&val)); 197 if (cg_atomic_needs_libcall(g, val_ty)) { 198 /* void __atomic_store_8(void* ptr, u64 val, int memorder) */ 199 KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32); 200 KitCgTypeId ps[3]; 201 ApiSValue args[3]; 202 ps[0] = pty; 203 ps[1] = val_ty; 204 ps[2] = i32; 205 args[0] = ptr; 206 args[1] = val; 207 args[2] = api_make_sv(api_op_imm((i64)order, i32), i32); 208 cg_atomic_runtime_call(g, "__atomic_store_8", (KitCgTypeId)0, ps, 3, args); 209 return; 210 } 211 addr = api_force_local(g, &ptr, pty); 212 src = api_sv_op_is_local_or_imm(&val) ? val.op 213 : api_force_local(g, &val, val_ty); 214 g->target->atomic_store(g->target, addr, src, api_mem_for_atomic(g, val_ty), 215 order); 216 api_release(g, &val); 217 api_release(g, &ptr); 218 } 219 220 void kit_cg_atomic_rmw(KitCg* g, KitCgMemAccess access, KitCgAtomicOp op, 221 KitCgMemOrder order) { 222 ApiSValue val, ptr; 223 KitCgTypeId pty, val_ty; 224 Operand addr, vop, dst; 225 CGLocal rr; 226 if (!g) return; 227 api_local_const_memory_boundary(g); 228 val = api_pop(g); 229 ptr = api_pop(g); 230 pty = api_sv_type(&ptr); 231 val_ty = resolve_type(g->c, access.type); 232 if (!val_ty) val_ty = api_atomic_pointee(g, pty, "KitCg: atomic_rmw"); 233 if (api_unevaluated(g)) { 234 api_release(g, &val); 235 api_release(g, &ptr); 236 api_push(g, api_uneval_value(g, val_ty)); 237 api_const_set_top(g, api_const_unknown(val_ty)); 238 return; 239 } 240 api_require_pointer_value(g, "atomic_rmw pointer", pty); 241 api_validate_memory_value_resolved(g, "atomic_rmw", val_ty, 242 api_sv_type(&val)); 243 if (cg_atomic_needs_libcall(g, val_ty)) { 244 /* u64 __atomic_{exchange,fetch_*}_8(void* ptr, u64 val, int memorder). 245 * All return the prior value, matching native atomic_rmw semantics. */ 246 const char* name = cg_atomic_rmw_libcall_8(op); 247 KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32); 248 KitCgTypeId ps[3]; 249 ApiSValue args[3]; 250 if (!name) { 251 compiler_panic(g->c, g->cur_loc, 252 "KitCg: unsupported 8-byte atomic rmw op"); 253 return; 254 } 255 ps[0] = pty; 256 ps[1] = val_ty; 257 ps[2] = i32; 258 args[0] = ptr; 259 args[1] = val; 260 args[2] = api_make_sv(api_op_imm((i64)order, i32), i32); 261 cg_atomic_runtime_call(g, name, val_ty, ps, 3, args); 262 return; 263 } 264 addr = api_force_local(g, &ptr, pty); 265 vop = api_sv_op_is_local_or_imm(&val) ? val.op 266 : api_force_local(g, &val, val_ty); 267 rr = api_alloc_temp_local(g, val_ty); 268 dst = api_op_local(rr, val_ty); 269 g->target->atomic_rmw(g->target, op, dst, addr, vop, 270 api_mem_for_atomic(g, val_ty), order); 271 api_release(g, &val); 272 api_release(g, &ptr); 273 api_push(g, api_make_sv(dst, val_ty)); 274 } 275 276 void kit_cg_atomic_cmpxchg(KitCg* g, KitCgMemAccess access, 277 KitCgMemOrder success, KitCgMemOrder failure, 278 int weak) { 279 ApiSValue desired, expected, ptr; 280 KitCgTypeId pty, val_ty, bool_ty; 281 Operand addr, exp_op, des_op, prior, ok; 282 CGLocal pr, kr; 283 if (!g) return; 284 api_local_const_memory_boundary(g); 285 (void)weak; 286 desired = api_pop(g); 287 expected = api_pop(g); 288 ptr = api_pop(g); 289 pty = api_sv_type(&ptr); 290 val_ty = resolve_type(g->c, access.type); 291 if (!val_ty) val_ty = api_atomic_pointee(g, pty, "KitCg: atomic_cmpxchg"); 292 if (api_unevaluated(g)) { 293 bool_ty = builtin_id(KIT_CG_BUILTIN_BOOL); 294 api_release(g, &desired); 295 api_release(g, &expected); 296 api_release(g, &ptr); 297 api_push(g, api_uneval_value(g, val_ty)); 298 api_const_set_top(g, api_const_unknown(val_ty)); 299 api_push(g, api_uneval_value(g, bool_ty)); 300 api_const_set_top(g, api_const_unknown(bool_ty)); 301 return; 302 } 303 api_require_pointer_value(g, "atomic_cmpxchg pointer", pty); 304 api_validate_memory_value_resolved(g, "atomic_cmpxchg expected", val_ty, 305 api_sv_type(&expected)); 306 api_validate_memory_value_resolved(g, "atomic_cmpxchg desired", val_ty, 307 api_sv_type(&desired)); 308 if (cg_atomic_needs_libcall(g, val_ty)) { 309 /* bool __atomic_compare_exchange_8(void* ptr, void* expected, u64 desired, 310 * int succ, int fail). 311 * libatomic takes `expected` by pointer and updates *expected with the 312 * observed value on failure. Our ABI is value-in / value-out, so spill the 313 * expected value to a stack slot, pass its address, then reload the slot to 314 * obtain `prior`. */ 315 KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32); 316 KitCgTypeId ptr_to_val = cg_type_ptr_to(g->c, val_ty); 317 KitCgTypeId ps[5]; 318 ApiSValue args[5]; 319 Operand exp_slot, exp_addr, exp_src; 320 CGLocal er, ar, pr2; 321 bool_ty = builtin_id(KIT_CG_BUILTIN_BOOL); 322 /* Materialize the expected value into an addressable stack slot. */ 323 er = api_alloc_temp_local(g, val_ty); 324 exp_slot = api_op_local(er, val_ty); 325 exp_src = api_sv_op_is_local_or_imm(&expected) 326 ? expected.op 327 : api_force_local(g, &expected, val_ty); 328 g->target->store(g->target, exp_slot, exp_src, 329 api_mem_for_lvalue(g, &exp_slot, val_ty)); 330 ar = api_alloc_temp_local(g, ptr_to_val); 331 exp_addr = api_op_local(ar, ptr_to_val); 332 g->target->addr_of(g->target, exp_addr, exp_slot); 333 ps[0] = pty; 334 ps[1] = ptr_to_val; 335 ps[2] = val_ty; 336 ps[3] = i32; 337 ps[4] = i32; 338 args[0] = ptr; 339 args[1] = api_make_sv(exp_addr, ptr_to_val); 340 args[2] = desired; 341 args[3] = api_make_sv(api_op_imm((i64)success, i32), i32); 342 args[4] = api_make_sv(api_op_imm((i64)failure, i32), i32); 343 cg_atomic_runtime_call(g, "__atomic_compare_exchange_8", bool_ty, ps, 5, 344 args); 345 { 346 ApiSValue ok_sv = api_pop(g); /* the returned bool */ 347 ok = ok_sv.op; 348 } 349 /* Reload the (possibly updated) expected slot as `prior`. */ 350 pr2 = api_alloc_temp_local(g, val_ty); 351 prior = api_op_local(pr2, val_ty); 352 g->target->load(g->target, prior, exp_slot, 353 api_mem_for_lvalue(g, &exp_slot, val_ty)); 354 /* `ptr` and `desired` were pushed as call args and are consumed by the 355 * call; only `expected` (spilled to a slot, not pushed) is still owned. */ 356 api_release(g, &expected); 357 api_push(g, api_make_sv(prior, val_ty)); 358 api_push(g, api_make_sv(ok, bool_ty)); 359 return; 360 } 361 addr = api_force_local(g, &ptr, pty); 362 exp_op = api_sv_op_is_local_or_imm(&expected) 363 ? expected.op 364 : api_force_local(g, &expected, val_ty); 365 des_op = api_sv_op_is_local_or_imm(&desired) 366 ? desired.op 367 : api_force_local(g, &desired, val_ty); 368 bool_ty = builtin_id(KIT_CG_BUILTIN_BOOL); 369 pr = api_alloc_temp_local(g, val_ty); 370 kr = api_alloc_temp_local(g, bool_ty); 371 prior = api_op_local(pr, val_ty); 372 ok = api_op_local(kr, bool_ty); 373 g->target->atomic_cas(g->target, prior, ok, addr, exp_op, des_op, 374 api_mem_for_atomic(g, val_ty), success, failure); 375 api_release(g, &desired); 376 api_release(g, &expected); 377 api_release(g, &ptr); 378 api_push(g, api_make_sv(prior, val_ty)); 379 api_push(g, api_make_sv(ok, bool_ty)); 380 } 381 382 void kit_cg_atomic_fence(KitCg* g, KitCgMemOrder order) { 383 if (!g) return; 384 if (api_unevaluated(g)) return; 385 api_local_const_memory_boundary(g); 386 g->target->fence(g->target, order); 387 } 388 389 /* ============================================================ 390 * Inline asm (stub) 391 * ============================================================ */