1 //===-- tsan_interceptors_mac.cc ------------------------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file is a part of ThreadSanitizer (TSan), a race detector.
12 // Mac-specific interceptors.
13 //===----------------------------------------------------------------------===//
15 #include "sanitizer_common/sanitizer_platform.h"
18 #include "interception/interception.h"
19 #include "tsan_interceptors.h"
20 #include "tsan_interface.h"
21 #include "tsan_interface_ann.h"
23 #include <libkern/OSAtomic.h>
25 #if defined(__has_include) && __has_include(<xpc/xpc.h>)
27 #endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
29 typedef long long_t; // NOLINT
33 // The non-barrier versions of OSAtomic* functions are semantically mo_relaxed,
34 // but the two variants (e.g. OSAtomicAdd32 and OSAtomicAdd32Barrier) are
35 // actually aliases of each other, and we cannot have different interceptors for
36 // them, because they're actually the same function. Thus, we have to stay
37 // conservative and treat the non-barrier versions as mo_acq_rel.
38 static const morder kMacOrderBarrier = mo_acq_rel;
39 static const morder kMacOrderNonBarrier = mo_acq_rel;
41 #define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
42 TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
43 SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \
44 return tsan_atomic_f((volatile tsan_t *)ptr, x, mo); \
47 #define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
48 TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
49 SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \
50 return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x; \
53 #define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
54 TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \
55 SCOPED_TSAN_INTERCEPTOR(f, ptr); \
56 return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1; \
59 #define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \
61 TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \
62 SCOPED_TSAN_INTERCEPTOR(f, ptr); \
63 return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1; \
66 #define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m) \
67 m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \
68 kMacOrderNonBarrier) \
69 m(int32_t, int32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \
71 m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f, \
72 kMacOrderNonBarrier) \
73 m(int64_t, int64_t, a64, f##64##Barrier, __tsan_atomic64_##tsan_atomic_f, \
76 #define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig) \
77 m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \
78 kMacOrderNonBarrier) \
79 m(int32_t, uint32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \
81 m_orig(int32_t, uint32_t, a32, f##32##Orig, __tsan_atomic32_##tsan_atomic_f, \
82 kMacOrderNonBarrier) \
83 m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier, \
84 __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier)
86 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd, fetch_add,
87 OSATOMIC_INTERCEPTOR_PLUS_X)
88 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicIncrement, fetch_add,
89 OSATOMIC_INTERCEPTOR_PLUS_1)
90 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicDecrement, fetch_sub,
91 OSATOMIC_INTERCEPTOR_MINUS_1)
92 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicOr, fetch_or, OSATOMIC_INTERCEPTOR_PLUS_X,
94 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd, fetch_and,
95 OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
96 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor,
97 OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
99 #define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t) \
100 TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \
101 SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \
102 return tsan_atomic_f##_compare_exchange_strong( \
103 (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \
104 kMacOrderNonBarrier, kMacOrderNonBarrier); \
107 TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \
109 SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \
110 return tsan_atomic_f##_compare_exchange_strong( \
111 (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \
112 kMacOrderBarrier, kMacOrderNonBarrier); \
115 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int)
116 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong, __tsan_atomic64, a64,
118 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapPtr, __tsan_atomic64, a64,
120 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32,
122 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64,
125 #define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \
126 TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \
127 SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \
128 volatile char *byte_ptr = ((volatile char *)ptr) + (n >> 3); \
129 char bit = 0x80u >> (n & 7); \
130 char mask = clear ? ~bit : bit; \
131 char orig_byte = op((volatile a8 *)byte_ptr, mask, mo); \
132 return orig_byte & bit; \
135 #define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear) \
136 OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \
137 OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier)
139 OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet, __tsan_atomic8_fetch_or, false)
140 OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear, __tsan_atomic8_fetch_and,
143 TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead *list, void *item,
145 SCOPED_TSAN_INTERCEPTOR(OSAtomicEnqueue, list, item, offset);
146 __tsan_release(item);
147 REAL(OSAtomicEnqueue)(list, item, offset);
150 TSAN_INTERCEPTOR(void *, OSAtomicDequeue, OSQueueHead *list, size_t offset) {
151 SCOPED_TSAN_INTERCEPTOR(OSAtomicDequeue, list, offset);
152 void *item = REAL(OSAtomicDequeue)(list, offset);
153 if (item) __tsan_acquire(item);
157 // OSAtomicFifoEnqueue and OSAtomicFifoDequeue are only on OS X.
160 TSAN_INTERCEPTOR(void, OSAtomicFifoEnqueue, OSFifoQueueHead *list, void *item,
162 SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoEnqueue, list, item, offset);
163 __tsan_release(item);
164 REAL(OSAtomicFifoEnqueue)(list, item, offset);
167 TSAN_INTERCEPTOR(void *, OSAtomicFifoDequeue, OSFifoQueueHead *list,
169 SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoDequeue, list, offset);
170 void *item = REAL(OSAtomicFifoDequeue)(list, offset);
171 if (item) __tsan_acquire(item);
177 TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) {
178 CHECK(!cur_thread()->is_dead);
179 if (!cur_thread()->is_inited) {
180 return REAL(OSSpinLockLock)(lock);
182 SCOPED_TSAN_INTERCEPTOR(OSSpinLockLock, lock);
183 REAL(OSSpinLockLock)(lock);
184 Acquire(thr, pc, (uptr)lock);
187 TSAN_INTERCEPTOR(bool, OSSpinLockTry, volatile OSSpinLock *lock) {
188 CHECK(!cur_thread()->is_dead);
189 if (!cur_thread()->is_inited) {
190 return REAL(OSSpinLockTry)(lock);
192 SCOPED_TSAN_INTERCEPTOR(OSSpinLockTry, lock);
193 bool result = REAL(OSSpinLockTry)(lock);
195 Acquire(thr, pc, (uptr)lock);
199 TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) {
200 CHECK(!cur_thread()->is_dead);
201 if (!cur_thread()->is_inited) {
202 return REAL(OSSpinLockUnlock)(lock);
204 SCOPED_TSAN_INTERCEPTOR(OSSpinLockUnlock, lock);
205 Release(thr, pc, (uptr)lock);
206 REAL(OSSpinLockUnlock)(lock);
209 TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) {
210 CHECK(!cur_thread()->is_dead);
211 if (!cur_thread()->is_inited) {
212 return REAL(os_lock_lock)(lock);
214 SCOPED_TSAN_INTERCEPTOR(os_lock_lock, lock);
215 REAL(os_lock_lock)(lock);
216 Acquire(thr, pc, (uptr)lock);
219 TSAN_INTERCEPTOR(bool, os_lock_trylock, void *lock) {
220 CHECK(!cur_thread()->is_dead);
221 if (!cur_thread()->is_inited) {
222 return REAL(os_lock_trylock)(lock);
224 SCOPED_TSAN_INTERCEPTOR(os_lock_trylock, lock);
225 bool result = REAL(os_lock_trylock)(lock);
227 Acquire(thr, pc, (uptr)lock);
231 TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) {
232 CHECK(!cur_thread()->is_dead);
233 if (!cur_thread()->is_inited) {
234 return REAL(os_lock_unlock)(lock);
236 SCOPED_TSAN_INTERCEPTOR(os_lock_unlock, lock);
237 Release(thr, pc, (uptr)lock);
238 REAL(os_lock_unlock)(lock);
241 #if defined(__has_include) && __has_include(<xpc/xpc.h>)
243 TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler,
244 xpc_connection_t connection, xpc_handler_t handler) {
245 SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection,
247 Release(thr, pc, (uptr)connection);
248 xpc_handler_t new_handler = ^(xpc_object_t object) {
250 SCOPED_INTERCEPTOR_RAW(xpc_connection_set_event_handler);
251 Acquire(thr, pc, (uptr)connection);
255 REAL(xpc_connection_set_event_handler)(connection, new_handler);
258 TSAN_INTERCEPTOR(void, xpc_connection_send_barrier, xpc_connection_t connection,
259 dispatch_block_t barrier) {
260 SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_barrier, connection, barrier);
261 Release(thr, pc, (uptr)connection);
262 dispatch_block_t new_barrier = ^() {
264 SCOPED_INTERCEPTOR_RAW(xpc_connection_send_barrier);
265 Acquire(thr, pc, (uptr)connection);
269 REAL(xpc_connection_send_barrier)(connection, new_barrier);
272 TSAN_INTERCEPTOR(void, xpc_connection_send_message_with_reply,
273 xpc_connection_t connection, xpc_object_t message,
274 dispatch_queue_t replyq, xpc_handler_t handler) {
275 SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_message_with_reply, connection,
276 message, replyq, handler);
277 Release(thr, pc, (uptr)connection);
278 xpc_handler_t new_handler = ^(xpc_object_t object) {
280 SCOPED_INTERCEPTOR_RAW(xpc_connection_send_message_with_reply);
281 Acquire(thr, pc, (uptr)connection);
285 REAL(xpc_connection_send_message_with_reply)
286 (connection, message, replyq, new_handler);
289 TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) {
290 SCOPED_TSAN_INTERCEPTOR(xpc_connection_cancel, connection);
291 Release(thr, pc, (uptr)connection);
292 REAL(xpc_connection_cancel)(connection);
295 #endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
297 // On macOS, libc++ is always linked dynamically, so intercepting works the
299 #define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
302 struct fake_shared_weak_count {
303 volatile a64 shared_owners;
304 volatile a64 shared_weak_owners;
305 virtual void _unused_0x0() = 0;
306 virtual void _unused_0x8() = 0;
307 virtual void on_zero_shared() = 0;
308 virtual void _unused_0x18() = 0;
309 virtual void on_zero_shared_weak() = 0;
313 // The following code adds libc++ interceptors for:
314 // void __shared_weak_count::__release_shared() _NOEXCEPT;
315 // bool __shared_count::__release_shared() _NOEXCEPT;
316 // Shared and weak pointers in C++ maintain reference counts via atomics in
317 // libc++.dylib, which are TSan-invisible, and this leads to false positives in
318 // destructor code. These interceptors re-implements the whole functions so that
319 // the mo_acq_rel semantics of the atomic decrement are visible.
321 // Unfortunately, the interceptors cannot simply Acquire/Release some sync
322 // object and call the original function, because it would have a race between
323 // the sync and the destruction of the object. Calling both under a lock will
324 // not work because the destructor can invoke this interceptor again (and even
325 // in a different thread, so recursive locks don't help).
327 STDCXX_INTERCEPTOR(void, _ZNSt3__119__shared_weak_count16__release_sharedEv,
328 fake_shared_weak_count *o) {
329 if (!flags()->shared_ptr_interceptor)
330 return REAL(_ZNSt3__119__shared_weak_count16__release_sharedEv)(o);
332 SCOPED_TSAN_INTERCEPTOR(_ZNSt3__119__shared_weak_count16__release_sharedEv,
334 if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) {
335 Acquire(thr, pc, (uptr)&o->shared_owners);
337 if (__tsan_atomic64_fetch_add(&o->shared_weak_owners, -1, mo_release) ==
339 Acquire(thr, pc, (uptr)&o->shared_weak_owners);
340 o->on_zero_shared_weak();
345 STDCXX_INTERCEPTOR(bool, _ZNSt3__114__shared_count16__release_sharedEv,
346 fake_shared_weak_count *o) {
347 if (!flags()->shared_ptr_interceptor)
348 return REAL(_ZNSt3__114__shared_count16__release_sharedEv)(o);
350 SCOPED_TSAN_INTERCEPTOR(_ZNSt3__114__shared_count16__release_sharedEv, o);
351 if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) {
352 Acquire(thr, pc, (uptr)&o->shared_owners);
360 struct call_once_callback_args {
361 void (*orig_func)(void *arg);
366 void call_once_callback_wrapper(void *arg) {
367 call_once_callback_args *new_args = (call_once_callback_args *)arg;
368 new_args->orig_func(new_args->orig_arg);
369 __tsan_release(new_args->flag);
373 // This adds a libc++ interceptor for:
374 // void __call_once(volatile unsigned long&, void*, void(*)(void*));
375 // C++11 call_once is implemented via an internal function __call_once which is
376 // inside libc++.dylib, and the atomic release store inside it is thus
377 // TSan-invisible. To avoid false positives, this interceptor wraps the callback
378 // function and performs an explicit Release after the user code has run.
379 STDCXX_INTERCEPTOR(void, _ZNSt3__111__call_onceERVmPvPFvS2_E, void *flag,
380 void *arg, void (*func)(void *arg)) {
381 call_once_callback_args new_args = {func, arg, flag};
382 REAL(_ZNSt3__111__call_onceERVmPvPFvS2_E)(flag, &new_args,
383 call_once_callback_wrapper);
386 } // namespace __tsan
388 #endif // SANITIZER_MAC