1 //===-- tsan_interceptors_mac.cc ------------------------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file is a part of ThreadSanitizer (TSan), a race detector.
12 // Mac-specific interceptors.
13 //===----------------------------------------------------------------------===//
15 #include "sanitizer_common/sanitizer_platform.h"
18 #include "interception/interception.h"
19 #include "tsan_interceptors.h"
20 #include "tsan_interface.h"
21 #include "tsan_interface_ann.h"
23 #include <libkern/OSAtomic.h>
26 typedef long long_t; // NOLINT
30 // The non-barrier versions of OSAtomic* functions are semantically mo_relaxed,
31 // but the two variants (e.g. OSAtomicAdd32 and OSAtomicAdd32Barrier) are
32 // actually aliases of each other, and we cannot have different interceptors for
33 // them, because they're actually the same function. Thus, we have to stay
34 // conservative and treat the non-barrier versions as mo_acq_rel.
35 static const morder kMacOrderBarrier = mo_acq_rel;
36 static const morder kMacOrderNonBarrier = mo_acq_rel;
38 #define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
39 TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
40 SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \
41 return tsan_atomic_f((volatile tsan_t *)ptr, x, mo); \
44 #define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
45 TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
46 SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \
47 return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x; \
50 #define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
51 TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \
52 SCOPED_TSAN_INTERCEPTOR(f, ptr); \
53 return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1; \
56 #define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \
58 TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \
59 SCOPED_TSAN_INTERCEPTOR(f, ptr); \
60 return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1; \
63 #define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m) \
64 m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \
65 kMacOrderNonBarrier) \
66 m(int32_t, int32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \
68 m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f, \
69 kMacOrderNonBarrier) \
70 m(int64_t, int64_t, a64, f##64##Barrier, __tsan_atomic64_##tsan_atomic_f, \
73 #define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig) \
74 m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \
75 kMacOrderNonBarrier) \
76 m(int32_t, uint32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \
78 m_orig(int32_t, uint32_t, a32, f##32##Orig, __tsan_atomic32_##tsan_atomic_f, \
79 kMacOrderNonBarrier) \
80 m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier, \
81 __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier)
83 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd, fetch_add,
84 OSATOMIC_INTERCEPTOR_PLUS_X)
85 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicIncrement, fetch_add,
86 OSATOMIC_INTERCEPTOR_PLUS_1)
87 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicDecrement, fetch_sub,
88 OSATOMIC_INTERCEPTOR_MINUS_1)
89 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicOr, fetch_or, OSATOMIC_INTERCEPTOR_PLUS_X,
91 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd, fetch_and,
92 OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
93 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor,
94 OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
96 #define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t) \
97 TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \
98 SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \
99 return tsan_atomic_f##_compare_exchange_strong( \
100 (tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \
101 kMacOrderNonBarrier, kMacOrderNonBarrier); \
104 TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \
106 SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \
107 return tsan_atomic_f##_compare_exchange_strong( \
108 (tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \
109 kMacOrderBarrier, kMacOrderNonBarrier); \
112 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int)
113 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong, __tsan_atomic64, a64,
115 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapPtr, __tsan_atomic64, a64,
117 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32,
119 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64,
122 #define OSATOMIC_INTERCEPTOR_BITOP(f, op, m, mo) \
123 TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \
124 SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \
125 char *byte_ptr = ((char *)ptr) + (n >> 3); \
126 char bit_index = n & 7; \
128 char orig_byte = op((a8 *)byte_ptr, mask, mo); \
129 return orig_byte & mask; \
132 #define OSATOMIC_INTERCEPTORS_BITOP(f, op, m) \
133 OSATOMIC_INTERCEPTOR_BITOP(f, op, m, kMacOrderNonBarrier) \
134 OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, m, kMacOrderBarrier)
136 OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet, __tsan_atomic8_fetch_or,
138 OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear, __tsan_atomic8_fetch_and,
139 ~(0x80u >> bit_index))
141 TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead *list, void *item,
143 SCOPED_TSAN_INTERCEPTOR(OSAtomicEnqueue, list, item, offset);
144 __tsan_release(item);
145 REAL(OSAtomicEnqueue)(list, item, offset);
148 TSAN_INTERCEPTOR(void *, OSAtomicDequeue, OSQueueHead *list, size_t offset) {
149 SCOPED_TSAN_INTERCEPTOR(OSAtomicDequeue, list, offset);
150 void *item = REAL(OSAtomicDequeue)(list, offset);
151 if (item) __tsan_acquire(item);
155 // OSAtomicFifoEnqueue and OSAtomicFifoDequeue are only on OS X.
158 TSAN_INTERCEPTOR(void, OSAtomicFifoEnqueue, OSFifoQueueHead *list, void *item,
160 SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoEnqueue, list, item, offset);
161 __tsan_release(item);
162 REAL(OSAtomicFifoEnqueue)(list, item, offset);
165 TSAN_INTERCEPTOR(void *, OSAtomicFifoDequeue, OSFifoQueueHead *list,
167 SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoDequeue, list, offset);
168 void *item = REAL(OSAtomicFifoDequeue)(list, offset);
169 if (item) __tsan_acquire(item);
175 TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) {
176 CHECK(!cur_thread()->is_dead);
177 if (!cur_thread()->is_inited) {
178 return REAL(OSSpinLockLock)(lock);
180 SCOPED_TSAN_INTERCEPTOR(OSSpinLockLock, lock);
181 REAL(OSSpinLockLock)(lock);
182 Acquire(thr, pc, (uptr)lock);
185 TSAN_INTERCEPTOR(bool, OSSpinLockTry, volatile OSSpinLock *lock) {
186 CHECK(!cur_thread()->is_dead);
187 if (!cur_thread()->is_inited) {
188 return REAL(OSSpinLockTry)(lock);
190 SCOPED_TSAN_INTERCEPTOR(OSSpinLockTry, lock);
191 bool result = REAL(OSSpinLockTry)(lock);
193 Acquire(thr, pc, (uptr)lock);
197 TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) {
198 CHECK(!cur_thread()->is_dead);
199 if (!cur_thread()->is_inited) {
200 return REAL(OSSpinLockUnlock)(lock);
202 SCOPED_TSAN_INTERCEPTOR(OSSpinLockUnlock, lock);
203 Release(thr, pc, (uptr)lock);
204 REAL(OSSpinLockUnlock)(lock);
207 TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) {
208 CHECK(!cur_thread()->is_dead);
209 if (!cur_thread()->is_inited) {
210 return REAL(os_lock_lock)(lock);
212 SCOPED_TSAN_INTERCEPTOR(os_lock_lock, lock);
213 REAL(os_lock_lock)(lock);
214 Acquire(thr, pc, (uptr)lock);
217 TSAN_INTERCEPTOR(bool, os_lock_trylock, void *lock) {
218 CHECK(!cur_thread()->is_dead);
219 if (!cur_thread()->is_inited) {
220 return REAL(os_lock_trylock)(lock);
222 SCOPED_TSAN_INTERCEPTOR(os_lock_trylock, lock);
223 bool result = REAL(os_lock_trylock)(lock);
225 Acquire(thr, pc, (uptr)lock);
229 TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) {
230 CHECK(!cur_thread()->is_dead);
231 if (!cur_thread()->is_inited) {
232 return REAL(os_lock_unlock)(lock);
234 SCOPED_TSAN_INTERCEPTOR(os_lock_unlock, lock);
235 Release(thr, pc, (uptr)lock);
236 REAL(os_lock_unlock)(lock);
239 TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler,
240 xpc_connection_t connection, xpc_handler_t handler) {
241 SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection,
243 Release(thr, pc, (uptr)connection);
244 xpc_handler_t new_handler = ^(xpc_object_t object) {
246 SCOPED_INTERCEPTOR_RAW(xpc_connection_set_event_handler);
247 Acquire(thr, pc, (uptr)connection);
251 REAL(xpc_connection_set_event_handler)(connection, new_handler);
254 TSAN_INTERCEPTOR(void, xpc_connection_send_barrier, xpc_connection_t connection,
255 dispatch_block_t barrier) {
256 SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_barrier, connection, barrier);
257 Release(thr, pc, (uptr)connection);
258 dispatch_block_t new_barrier = ^() {
260 SCOPED_INTERCEPTOR_RAW(xpc_connection_send_barrier);
261 Acquire(thr, pc, (uptr)connection);
265 REAL(xpc_connection_send_barrier)(connection, new_barrier);
268 TSAN_INTERCEPTOR(void, xpc_connection_send_message_with_reply,
269 xpc_connection_t connection, xpc_object_t message,
270 dispatch_queue_t replyq, xpc_handler_t handler) {
271 SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_message_with_reply, connection,
272 message, replyq, handler);
273 Release(thr, pc, (uptr)connection);
274 xpc_handler_t new_handler = ^(xpc_object_t object) {
276 SCOPED_INTERCEPTOR_RAW(xpc_connection_send_message_with_reply);
277 Acquire(thr, pc, (uptr)connection);
281 REAL(xpc_connection_send_message_with_reply)
282 (connection, message, replyq, new_handler);
285 // On macOS, libc++ is always linked dynamically, so intercepting works the
287 #define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
290 struct fake_shared_weak_count {
291 volatile a64 shared_owners;
292 volatile a64 shared_weak_owners;
293 virtual void _unused_0x0() = 0;
294 virtual void _unused_0x8() = 0;
295 virtual void on_zero_shared() = 0;
296 virtual void _unused_0x18() = 0;
297 virtual void on_zero_shared_weak() = 0;
301 // This adds a libc++ interceptor for:
302 // void __shared_weak_count::__release_shared() _NOEXCEPT;
303 // Shared and weak pointers in C++ maintain reference counts via atomics in
304 // libc++.dylib, which are TSan-invisible, and this leads to false positives in
305 // destructor code. This interceptor re-implements the whole function so that
306 // the mo_acq_rel semantics of the atomic decrement are visible.
308 // Unfortunately, this interceptor cannot simply Acquire/Release some sync
309 // object and call the original function, because it would have a race between
310 // the sync and the destruction of the object. Calling both under a lock will
311 // not work because the destructor can invoke this interceptor again (and even
312 // in a different thread, so recursive locks don't help).
313 STDCXX_INTERCEPTOR(void, _ZNSt3__119__shared_weak_count16__release_sharedEv,
314 fake_shared_weak_count *o) {
315 if (!flags()->shared_ptr_interceptor)
316 return REAL(_ZNSt3__119__shared_weak_count16__release_sharedEv)(o);
318 SCOPED_TSAN_INTERCEPTOR(_ZNSt3__119__shared_weak_count16__release_sharedEv,
320 if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) {
321 Acquire(thr, pc, (uptr)&o->shared_owners);
323 if (__tsan_atomic64_fetch_add(&o->shared_weak_owners, -1, mo_release) ==
325 Acquire(thr, pc, (uptr)&o->shared_weak_owners);
326 o->on_zero_shared_weak();
331 } // namespace __tsan
333 #endif // SANITIZER_MAC