atomic_common.inc (11597B)
1 //===-- atomic_common.inc - shared body for atomic_*.c ------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 // Includer must define BEFORE this include: 8 // typedef ... Lock; 9 // static inline void lock(Lock *l); 10 // static inline void unlock(Lock *l); 11 // static Lock locks[SPINLOCK_COUNT]; 12 // 13 // Uses GCC-style __atomic_* builtins (the family kit provides) rather than 14 // Clang-specific __c11_atomic_*. 15 // 16 // kit does not treat the unsuffixed generic entry points 17 // (__atomic_load / __atomic_store / __atomic_exchange / 18 // __atomic_compare_exchange / __atomic_is_lock_free) as compiler builtins, 19 // so we define them under their final library names directly — no 20 // `#pragma redefine_extname` rename is required. 21 //===----------------------------------------------------------------------===// 22 23 static inline Lock *lock_for_pointer(void *ptr) { 24 intptr_t hash = (intptr_t)ptr; 25 hash >>= 4; 26 intptr_t low = hash & SPINLOCK_MASK; 27 hash >>= 16; 28 hash ^= low; 29 return locks + (hash & SPINLOCK_MASK); 30 } 31 32 static inline int bytes_equal(const void *a, const void *b, int size) { 33 const unsigned char *p = (const unsigned char *)a; 34 const unsigned char *q = (const unsigned char *)b; 35 for (int i = 0; i < size; ++i) { 36 unsigned char pa = p[i]; 37 unsigned char qb = q[i]; 38 if (pa != qb) return 0; 39 } 40 return 1; 41 } 42 43 #define ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(size, p) \ 44 (__atomic_always_lock_free(size, p) || \ 45 (__atomic_always_lock_free(size, 0) && ((uintptr_t)p % size) == 0)) 46 #define IS_LOCK_FREE_1(p) ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(1, p) 47 #define IS_LOCK_FREE_2(p) ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(2, p) 48 #define IS_LOCK_FREE_4(p) ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(4, p) 49 #define IS_LOCK_FREE_8(p) ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(8, p) 50 #define IS_LOCK_FREE_16(p) ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(16, p) 51 52 #define TRY_LOCK_FREE_CASE(n, type, ptr) \ 53 case n: \ 54 if (IS_LOCK_FREE_##n(ptr)) { LOCK_FREE_ACTION(type); } \ 55 break; 56 57 #if HAS_INT128 58 #define TRY_LOCK_FREE_CASE_16(p) TRY_LOCK_FREE_CASE(16, __uint128_t, p) 59 #define OPTIMISED_CASES \ 60 OPTIMISED_CASE(1, IS_LOCK_FREE_1, uint8_t) \ 61 OPTIMISED_CASE(2, IS_LOCK_FREE_2, uint16_t) \ 62 OPTIMISED_CASE(4, IS_LOCK_FREE_4, uint32_t) \ 63 OPTIMISED_CASE(8, IS_LOCK_FREE_8, uint64_t) \ 64 OPTIMISED_CASE(16, IS_LOCK_FREE_16, __uint128_t) 65 #else 66 #define TRY_LOCK_FREE_CASE_16(p) /* no __uint128_t */ 67 #define OPTIMISED_CASES \ 68 OPTIMISED_CASE(1, IS_LOCK_FREE_1, uint8_t) \ 69 OPTIMISED_CASE(2, IS_LOCK_FREE_2, uint16_t) \ 70 OPTIMISED_CASE(4, IS_LOCK_FREE_4, uint32_t) \ 71 OPTIMISED_CASE(8, IS_LOCK_FREE_8, uint64_t) 72 #endif 73 74 #define LOCK_FREE_CASES(ptr) \ 75 do { \ 76 switch (size) { \ 77 TRY_LOCK_FREE_CASE(1, uint8_t, ptr) \ 78 TRY_LOCK_FREE_CASE(2, uint16_t, ptr) \ 79 TRY_LOCK_FREE_CASE(4, uint32_t, ptr) \ 80 TRY_LOCK_FREE_CASE(8, uint64_t, ptr) \ 81 TRY_LOCK_FREE_CASE_16(ptr) \ 82 default: break; \ 83 } \ 84 } while (0) 85 86 bool __atomic_is_lock_free(size_t size, void *ptr) { 87 #define LOCK_FREE_ACTION(type) return true; 88 LOCK_FREE_CASES(ptr); 89 #undef LOCK_FREE_ACTION 90 return false; 91 } 92 93 #pragma clang diagnostic push 94 #pragma clang diagnostic ignored "-Watomic-alignment" 95 96 void __atomic_load(int size, void *src, void *dest, int model) { 97 #define LOCK_FREE_ACTION(type) \ 98 *((type *)dest) = __atomic_load_n((type *)src, model); \ 99 return; 100 LOCK_FREE_CASES(src); 101 #undef LOCK_FREE_ACTION 102 Lock *l = lock_for_pointer(src); 103 lock(l); 104 __builtin_memcpy(dest, src, size); 105 unlock(l); 106 } 107 108 void __atomic_store(int size, void *dest, void *src, int model) { 109 #define LOCK_FREE_ACTION(type) \ 110 __atomic_store_n((type *)dest, *(type *)src, model); \ 111 return; 112 LOCK_FREE_CASES(dest); 113 #undef LOCK_FREE_ACTION 114 Lock *l = lock_for_pointer(dest); 115 lock(l); 116 __builtin_memcpy(dest, src, size); 117 unlock(l); 118 } 119 120 int __atomic_compare_exchange(int size, void *ptr, void *expected, 121 void *desired, int success, int failure) { 122 (void)success; 123 (void)failure; 124 Lock *l = lock_for_pointer(ptr); 125 lock(l); 126 if (bytes_equal(ptr, expected, size)) { 127 __builtin_memcpy(ptr, desired, size); 128 unlock(l); 129 return 1; 130 } 131 __builtin_memcpy(expected, ptr, size); 132 unlock(l); 133 return 0; 134 } 135 136 void __atomic_exchange(int size, void *ptr, void *val, void *old, int model) { 137 #define LOCK_FREE_ACTION(type) \ 138 *(type *)old = __atomic_exchange_n((type *)ptr, *(type *)val, model); \ 139 return; 140 LOCK_FREE_CASES(ptr); 141 #undef LOCK_FREE_ACTION 142 Lock *l = lock_for_pointer(ptr); 143 lock(l); 144 __builtin_memcpy(old, ptr, size); 145 __builtin_memcpy(ptr, val, size); 146 unlock(l); 147 } 148 149 #define OPTIMISED_CASE(n, lockfree, type) \ 150 type __atomic_load_##n(type *src, int model) { \ 151 if (lockfree(src)) return __atomic_load_n(src, model); \ 152 Lock *l = lock_for_pointer(src); \ 153 lock(l); \ 154 type val = *src; \ 155 unlock(l); \ 156 return val; \ 157 } 158 OPTIMISED_CASES 159 #undef OPTIMISED_CASE 160 161 #define OPTIMISED_CASE(n, lockfree, type) \ 162 void __atomic_store_##n(type *dest, type val, int model) { \ 163 if (lockfree(dest)) { __atomic_store_n(dest, val, model); return; } \ 164 Lock *l = lock_for_pointer(dest); \ 165 lock(l); \ 166 *dest = val; \ 167 unlock(l); \ 168 } 169 OPTIMISED_CASES 170 #undef OPTIMISED_CASE 171 172 #define OPTIMISED_CASE(n, lockfree, type) \ 173 type __atomic_exchange_##n(type *dest, type val, int model) { \ 174 if (lockfree(dest)) return __atomic_exchange_n(dest, val, model); \ 175 Lock *l = lock_for_pointer(dest); \ 176 lock(l); \ 177 type tmp = *dest; \ 178 *dest = val; \ 179 unlock(l); \ 180 return tmp; \ 181 } 182 OPTIMISED_CASES 183 #undef OPTIMISED_CASE 184 185 #define OPTIMISED_CASE(n, lockfree, type) \ 186 bool __atomic_compare_exchange_##n(type *ptr, type *expected, type desired, \ 187 int success, int failure) { \ 188 (void)success; \ 189 (void)failure; \ 190 Lock *l = lock_for_pointer(ptr); \ 191 lock(l); \ 192 if (*ptr == *expected) { \ 193 *ptr = desired; \ 194 unlock(l); \ 195 return true; \ 196 } \ 197 *expected = *ptr; \ 198 unlock(l); \ 199 return false; \ 200 } 201 OPTIMISED_CASES 202 #undef OPTIMISED_CASE 203 204 #define ATOMIC_RMW(n, lockfree, type, opname, op) \ 205 type __atomic_fetch_##opname##_##n(type *ptr, type val, int model) { \ 206 if (lockfree(ptr)) return __atomic_fetch_##opname(ptr, val, model); \ 207 Lock *l = lock_for_pointer(ptr); \ 208 lock(l); \ 209 type tmp = *ptr; \ 210 *ptr = tmp op val; \ 211 unlock(l); \ 212 return tmp; \ 213 } 214 215 #define ATOMIC_RMW_NAND(n, lockfree, type) \ 216 type __atomic_fetch_nand_##n(type *ptr, type val, int model) { \ 217 if (lockfree(ptr)) return __atomic_fetch_nand(ptr, val, model); \ 218 Lock *l = lock_for_pointer(ptr); \ 219 lock(l); \ 220 type tmp = *ptr; \ 221 *ptr = ~(tmp & val); \ 222 unlock(l); \ 223 return tmp; \ 224 } 225 226 #define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, add, +) 227 OPTIMISED_CASES 228 #undef OPTIMISED_CASE 229 #define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, sub, -) 230 OPTIMISED_CASES 231 #undef OPTIMISED_CASE 232 #define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, and, &) 233 OPTIMISED_CASES 234 #undef OPTIMISED_CASE 235 #define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, or, |) 236 OPTIMISED_CASES 237 #undef OPTIMISED_CASE 238 #define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, xor, ^) 239 OPTIMISED_CASES 240 #undef OPTIMISED_CASE 241 #define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW_NAND(n, lockfree, type) 242 OPTIMISED_CASES 243 #undef OPTIMISED_CASE 244 245 #pragma clang diagnostic pop