1 #ifndef JEMALLOC_INTERNAL_ATOMIC_GCC_SYNC_H
2 #define JEMALLOC_INTERNAL_ATOMIC_GCC_SYNC_H
4 #define ATOMIC_INIT(...) {__VA_ARGS__}
7 atomic_memory_order_relaxed,
8 atomic_memory_order_acquire,
9 atomic_memory_order_release,
10 atomic_memory_order_acq_rel,
11 atomic_memory_order_seq_cst
12 } atomic_memory_order_t;
15 atomic_fence(atomic_memory_order_t mo) {
16 /* Easy cases first: no barrier, and full barrier. */
17 if (mo == atomic_memory_order_relaxed) {
18 asm volatile("" ::: "memory");
21 if (mo == atomic_memory_order_seq_cst) {
22 asm volatile("" ::: "memory");
24 asm volatile("" ::: "memory");
27 asm volatile("" ::: "memory");
28 # if defined(__i386__) || defined(__x86_64__)
29 /* This is implicit on x86. */
30 # elif defined(__ppc__)
31 asm volatile("lwsync");
32 # elif defined(__sparc__) && defined(__arch64__)
33 if (mo == atomic_memory_order_acquire) {
34 asm volatile("membar #LoadLoad | #LoadStore");
35 } else if (mo == atomic_memory_order_release) {
36 asm volatile("membar #LoadStore | #StoreStore");
38 asm volatile("membar #LoadLoad | #LoadStore | #StoreStore");
43 asm volatile("" ::: "memory");
47 * A correct implementation of seq_cst loads and stores on weakly ordered
48 * architectures could do either of the following:
49 * 1. store() is weak-fence -> store -> strong fence, load() is load ->
51 * 2. store() is strong-fence -> store, load() is strong-fence -> load ->
53 * The tricky thing is, load() and store() above can be the load or store
54 * portions of a gcc __sync builtin, so we have to follow GCC's lead, which
55 * means going with strategy 2.
56 * On strongly ordered architectures, the natural strategy is to stick a strong
57 * fence after seq_cst stores, and have naked loads. So we want the strong
58 * fences in different places on different architectures.
59 * atomic_pre_sc_load_fence and atomic_post_sc_store_fence allow us to
64 atomic_pre_sc_load_fence() {
65 # if defined(__i386__) || defined(__x86_64__) || \
66 (defined(__sparc__) && defined(__arch64__))
67 atomic_fence(atomic_memory_order_relaxed);
69 atomic_fence(atomic_memory_order_seq_cst);
74 atomic_post_sc_store_fence() {
75 # if defined(__i386__) || defined(__x86_64__) || \
76 (defined(__sparc__) && defined(__arch64__))
77 atomic_fence(atomic_memory_order_seq_cst);
79 atomic_fence(atomic_memory_order_relaxed);
84 #define JEMALLOC_GENERATE_ATOMICS(type, short_type, \
85 /* unused */ lg_size) \
88 } atomic_##short_type##_t; \
91 atomic_load_##short_type(const atomic_##short_type##_t *a, \
92 atomic_memory_order_t mo) { \
93 if (mo == atomic_memory_order_seq_cst) { \
94 atomic_pre_sc_load_fence(); \
96 type result = a->repr; \
97 if (mo != atomic_memory_order_relaxed) { \
98 atomic_fence(atomic_memory_order_acquire); \
104 atomic_store_##short_type(atomic_##short_type##_t *a, \
105 type val, atomic_memory_order_t mo) { \
106 if (mo != atomic_memory_order_relaxed) { \
107 atomic_fence(atomic_memory_order_release); \
110 if (mo == atomic_memory_order_seq_cst) { \
111 atomic_post_sc_store_fence(); \
116 atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \
117 atomic_memory_order_t mo) { \
119 * Because of FreeBSD, we care about gcc 4.2, which doesn't have\
120 * an atomic exchange builtin. We fake it with a CAS loop. \
123 type old = a->repr; \
124 if (__sync_bool_compare_and_swap(&a->repr, old, val)) { \
131 atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \
132 type *expected, type desired, atomic_memory_order_t success_mo, \
133 atomic_memory_order_t failure_mo) { \
134 type prev = __sync_val_compare_and_swap(&a->repr, *expected, \
136 if (prev == *expected) { \
144 atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \
145 type *expected, type desired, atomic_memory_order_t success_mo, \
146 atomic_memory_order_t failure_mo) { \
147 type prev = __sync_val_compare_and_swap(&a->repr, *expected, \
149 if (prev == *expected) { \
157 #define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, \
158 /* unused */ lg_size) \
159 JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size) \
162 atomic_fetch_add_##short_type(atomic_##short_type##_t *a, type val, \
163 atomic_memory_order_t mo) { \
164 return __sync_fetch_and_add(&a->repr, val); \
168 atomic_fetch_sub_##short_type(atomic_##short_type##_t *a, type val, \
169 atomic_memory_order_t mo) { \
170 return __sync_fetch_and_sub(&a->repr, val); \
174 atomic_fetch_and_##short_type(atomic_##short_type##_t *a, type val, \
175 atomic_memory_order_t mo) { \
176 return __sync_fetch_and_and(&a->repr, val); \
180 atomic_fetch_or_##short_type(atomic_##short_type##_t *a, type val, \
181 atomic_memory_order_t mo) { \
182 return __sync_fetch_and_or(&a->repr, val); \
186 atomic_fetch_xor_##short_type(atomic_##short_type##_t *a, type val, \
187 atomic_memory_order_t mo) { \
188 return __sync_fetch_and_xor(&a->repr, val); \
191 #endif /* JEMALLOC_INTERNAL_ATOMIC_GCC_SYNC_H */