contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cc

   1 //===-- tsan_interceptors_mac.cc ------------------------------------------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // This file is a part of ThreadSanitizer (TSan), a race detector.
  11 //
  12 // Mac-specific interceptors.
  13 //===----------------------------------------------------------------------===//
  14
  15 #include "sanitizer_common/sanitizer_platform.h"
  16 #if SANITIZER_MAC
  17
  18 #include "interception/interception.h"
  19 #include "tsan_interceptors.h"
  20 #include "tsan_interface.h"
  21 #include "tsan_interface_ann.h"
  22
  23 #include <libkern/OSAtomic.h>
  24
  25 #if defined(__has_include) && __has_include(<xpc/xpc.h>)
  26 #include <xpc/xpc.h>
  27 #endif  // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
  28
  29 typedef long long_t;  // NOLINT
  30
  31 namespace __tsan {
  32
  33 // The non-barrier versions of OSAtomic* functions are semantically mo_relaxed,
  34 // but the two variants (e.g. OSAtomicAdd32 and OSAtomicAdd32Barrier) are
  35 // actually aliases of each other, and we cannot have different interceptors for
  36 // them, because they're actually the same function.  Thus, we have to stay
  37 // conservative and treat the non-barrier versions as mo_acq_rel.
  38 static const morder kMacOrderBarrier = mo_acq_rel;
  39 static const morder kMacOrderNonBarrier = mo_acq_rel;
  40
  41 #define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
  42   TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) {                 \
  43     SCOPED_TSAN_INTERCEPTOR(f, x, ptr);                                 \
  44     return tsan_atomic_f((volatile tsan_t *)ptr, x, mo);                \
  45   }
  46
  47 #define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
  48   TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) {                        \
  49     SCOPED_TSAN_INTERCEPTOR(f, x, ptr);                                        \
  50     return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x;                   \
  51   }
  52
  53 #define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
  54   TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) {                             \
  55     SCOPED_TSAN_INTERCEPTOR(f, ptr);                                           \
  56     return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1;                   \
  57   }
  58
  59 #define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \
  60                                      mo)                                    \
  61   TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) {                          \
  62     SCOPED_TSAN_INTERCEPTOR(f, ptr);                                        \
  63     return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1;                \
  64   }
  65
  66 #define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m)                  \
  67   m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f,             \
  68     kMacOrderNonBarrier)                                                       \
  69   m(int32_t, int32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f,    \
  70     kMacOrderBarrier)                                                          \
  71   m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f,             \
  72     kMacOrderNonBarrier)                                                       \
  73   m(int64_t, int64_t, a64, f##64##Barrier, __tsan_atomic64_##tsan_atomic_f,    \
  74     kMacOrderBarrier)
  75
  76 #define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig)             \
  77   m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f,            \
  78     kMacOrderNonBarrier)                                                       \
  79   m(int32_t, uint32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f,   \
  80     kMacOrderBarrier)                                                          \
  81   m_orig(int32_t, uint32_t, a32, f##32##Orig, __tsan_atomic32_##tsan_atomic_f, \
  82     kMacOrderNonBarrier)                                                       \
  83   m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier,                           \
  84     __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier)
  85
  86 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd, fetch_add,
  87                                  OSATOMIC_INTERCEPTOR_PLUS_X)
  88 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicIncrement, fetch_add,
  89                                  OSATOMIC_INTERCEPTOR_PLUS_1)
  90 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicDecrement, fetch_sub,
  91                                  OSATOMIC_INTERCEPTOR_MINUS_1)
  92 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicOr, fetch_or, OSATOMIC_INTERCEPTOR_PLUS_X,
  93                               OSATOMIC_INTERCEPTOR)
  94 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd, fetch_and,
  95                               OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
  96 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor,
  97                               OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
  98
  99 #define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t)              \
 100   TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) {    \
 101     SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr);                  \
 102     return tsan_atomic_f##_compare_exchange_strong(                         \
 103         (tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value,             \
 104         kMacOrderNonBarrier, kMacOrderNonBarrier);                          \
 105   }                                                                         \
 106                                                                             \
 107   TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value,              \
 108                    t volatile *ptr) {                                       \
 109     SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr);         \
 110     return tsan_atomic_f##_compare_exchange_strong(                         \
 111         (tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value,             \
 112         kMacOrderBarrier, kMacOrderNonBarrier);                             \
 113   }
 114
 115 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int)
 116 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong, __tsan_atomic64, a64,
 117                           long_t)
 118 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapPtr, __tsan_atomic64, a64,
 119                           void *)
 120 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32,
 121                           int32_t)
 122 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64,
 123                           int64_t)
 124
 125 #define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo)          \
 126   TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \
 127     SCOPED_TSAN_INTERCEPTOR(f, n, ptr);                       \
 128     char *byte_ptr = ((char *)ptr) + (n >> 3);                \
 129     char bit = 0x80u >> (n & 7);                              \
 130     char mask = clear ? ~bit : bit;                           \
 131     char orig_byte = op((a8 *)byte_ptr, mask, mo);            \
 132     return orig_byte & bit;                                   \
 133   }
 134
 135 #define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear)               \
 136   OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \
 137   OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier)
 138
 139 OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet, __tsan_atomic8_fetch_or, false)
 140 OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear, __tsan_atomic8_fetch_and,
 141                             true)
 142
 143 TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead *list, void *item,
 144                  size_t offset) {
 145   SCOPED_TSAN_INTERCEPTOR(OSAtomicEnqueue, list, item, offset);
 146   __tsan_release(item);
 147   REAL(OSAtomicEnqueue)(list, item, offset);
 148 }
 149
 150 TSAN_INTERCEPTOR(void *, OSAtomicDequeue, OSQueueHead *list, size_t offset) {
 151   SCOPED_TSAN_INTERCEPTOR(OSAtomicDequeue, list, offset);
 152   void *item = REAL(OSAtomicDequeue)(list, offset);
 153   if (item) __tsan_acquire(item);
 154   return item;
 155 }
 156
 157 // OSAtomicFifoEnqueue and OSAtomicFifoDequeue are only on OS X.
 158 #if !SANITIZER_IOS
 159
 160 TSAN_INTERCEPTOR(void, OSAtomicFifoEnqueue, OSFifoQueueHead *list, void *item,
 161                  size_t offset) {
 162   SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoEnqueue, list, item, offset);
 163   __tsan_release(item);
 164   REAL(OSAtomicFifoEnqueue)(list, item, offset);
 165 }
 166
 167 TSAN_INTERCEPTOR(void *, OSAtomicFifoDequeue, OSFifoQueueHead *list,
 168                  size_t offset) {
 169   SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoDequeue, list, offset);
 170   void *item = REAL(OSAtomicFifoDequeue)(list, offset);
 171   if (item) __tsan_acquire(item);
 172   return item;
 173 }
 174
 175 #endif
 176
 177 TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) {
 178   CHECK(!cur_thread()->is_dead);
 179   if (!cur_thread()->is_inited) {
 180     return REAL(OSSpinLockLock)(lock);
 181   }
 182   SCOPED_TSAN_INTERCEPTOR(OSSpinLockLock, lock);
 183   REAL(OSSpinLockLock)(lock);
 184   Acquire(thr, pc, (uptr)lock);
 185 }
 186
 187 TSAN_INTERCEPTOR(bool, OSSpinLockTry, volatile OSSpinLock *lock) {
 188   CHECK(!cur_thread()->is_dead);
 189   if (!cur_thread()->is_inited) {
 190     return REAL(OSSpinLockTry)(lock);
 191   }
 192   SCOPED_TSAN_INTERCEPTOR(OSSpinLockTry, lock);
 193   bool result = REAL(OSSpinLockTry)(lock);
 194   if (result)
 195     Acquire(thr, pc, (uptr)lock);
 196   return result;
 197 }
 198
 199 TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) {
 200   CHECK(!cur_thread()->is_dead);
 201   if (!cur_thread()->is_inited) {
 202     return REAL(OSSpinLockUnlock)(lock);
 203   }
 204   SCOPED_TSAN_INTERCEPTOR(OSSpinLockUnlock, lock);
 205   Release(thr, pc, (uptr)lock);
 206   REAL(OSSpinLockUnlock)(lock);
 207 }
 208
 209 TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) {
 210   CHECK(!cur_thread()->is_dead);
 211   if (!cur_thread()->is_inited) {
 212     return REAL(os_lock_lock)(lock);
 213   }
 214   SCOPED_TSAN_INTERCEPTOR(os_lock_lock, lock);
 215   REAL(os_lock_lock)(lock);
 216   Acquire(thr, pc, (uptr)lock);
 217 }
 218
 219 TSAN_INTERCEPTOR(bool, os_lock_trylock, void *lock) {
 220   CHECK(!cur_thread()->is_dead);
 221   if (!cur_thread()->is_inited) {
 222     return REAL(os_lock_trylock)(lock);
 223   }
 224   SCOPED_TSAN_INTERCEPTOR(os_lock_trylock, lock);
 225   bool result = REAL(os_lock_trylock)(lock);
 226   if (result)
 227     Acquire(thr, pc, (uptr)lock);
 228   return result;
 229 }
 230
 231 TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) {
 232   CHECK(!cur_thread()->is_dead);
 233   if (!cur_thread()->is_inited) {
 234     return REAL(os_lock_unlock)(lock);
 235   }
 236   SCOPED_TSAN_INTERCEPTOR(os_lock_unlock, lock);
 237   Release(thr, pc, (uptr)lock);
 238   REAL(os_lock_unlock)(lock);
 239 }
 240
 241 #if defined(__has_include) && __has_include(<xpc/xpc.h>)
 242
 243 TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler,
 244                  xpc_connection_t connection, xpc_handler_t handler) {
 245   SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection,
 246                           handler);
 247   Release(thr, pc, (uptr)connection);
 248   xpc_handler_t new_handler = ^(xpc_object_t object) {
 249     {
 250       SCOPED_INTERCEPTOR_RAW(xpc_connection_set_event_handler);
 251       Acquire(thr, pc, (uptr)connection);
 252     }
 253     handler(object);
 254   };
 255   REAL(xpc_connection_set_event_handler)(connection, new_handler);
 256 }
 257
 258 TSAN_INTERCEPTOR(void, xpc_connection_send_barrier, xpc_connection_t connection,
 259                  dispatch_block_t barrier) {
 260   SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_barrier, connection, barrier);
 261   Release(thr, pc, (uptr)connection);
 262   dispatch_block_t new_barrier = ^() {
 263     {
 264       SCOPED_INTERCEPTOR_RAW(xpc_connection_send_barrier);
 265       Acquire(thr, pc, (uptr)connection);
 266     }
 267     barrier();
 268   };
 269   REAL(xpc_connection_send_barrier)(connection, new_barrier);
 270 }
 271
 272 TSAN_INTERCEPTOR(void, xpc_connection_send_message_with_reply,
 273                  xpc_connection_t connection, xpc_object_t message,
 274                  dispatch_queue_t replyq, xpc_handler_t handler) {
 275   SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_message_with_reply, connection,
 276                           message, replyq, handler);
 277   Release(thr, pc, (uptr)connection);
 278   xpc_handler_t new_handler = ^(xpc_object_t object) {
 279     {
 280       SCOPED_INTERCEPTOR_RAW(xpc_connection_send_message_with_reply);
 281       Acquire(thr, pc, (uptr)connection);
 282     }
 283     handler(object);
 284   };
 285   REAL(xpc_connection_send_message_with_reply)
 286   (connection, message, replyq, new_handler);
 287 }
 288
 289 TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) {
 290   SCOPED_TSAN_INTERCEPTOR(xpc_connection_cancel, connection);
 291   Release(thr, pc, (uptr)connection);
 292   REAL(xpc_connection_cancel)(connection);
 293 }
 294
 295 #endif  // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
 296
 297 // On macOS, libc++ is always linked dynamically, so intercepting works the
 298 // usual way.
 299 #define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
 300
 301 namespace {
 302 struct fake_shared_weak_count {
 303   volatile a64 shared_owners;
 304   volatile a64 shared_weak_owners;
 305   virtual void _unused_0x0() = 0;
 306   virtual void _unused_0x8() = 0;
 307   virtual void on_zero_shared() = 0;
 308   virtual void _unused_0x18() = 0;
 309   virtual void on_zero_shared_weak() = 0;
 310 };
 311 }  // namespace
 312
 313 // The following code adds libc++ interceptors for:
 314 //     void __shared_weak_count::__release_shared() _NOEXCEPT;
 315 //     bool __shared_count::__release_shared() _NOEXCEPT;
 316 // Shared and weak pointers in C++ maintain reference counts via atomics in
 317 // libc++.dylib, which are TSan-invisible, and this leads to false positives in
 318 // destructor code. These interceptors re-implements the whole functions so that
 319 // the mo_acq_rel semantics of the atomic decrement are visible.
 320 //
 321 // Unfortunately, the interceptors cannot simply Acquire/Release some sync
 322 // object and call the original function, because it would have a race between
 323 // the sync and the destruction of the object.  Calling both under a lock will
 324 // not work because the destructor can invoke this interceptor again (and even
 325 // in a different thread, so recursive locks don't help).
 326
 327 STDCXX_INTERCEPTOR(void, _ZNSt3__119__shared_weak_count16__release_sharedEv,
 328                    fake_shared_weak_count *o) {
 329   if (!flags()->shared_ptr_interceptor)
 330     return REAL(_ZNSt3__119__shared_weak_count16__release_sharedEv)(o);
 331
 332   SCOPED_TSAN_INTERCEPTOR(_ZNSt3__119__shared_weak_count16__release_sharedEv,
 333                           o);
 334   if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) {
 335     Acquire(thr, pc, (uptr)&o->shared_owners);
 336     o->on_zero_shared();
 337     if (__tsan_atomic64_fetch_add(&o->shared_weak_owners, -1, mo_release) ==
 338         0) {
 339       Acquire(thr, pc, (uptr)&o->shared_weak_owners);
 340       o->on_zero_shared_weak();
 341     }
 342   }
 343 }
 344
 345 STDCXX_INTERCEPTOR(bool, _ZNSt3__114__shared_count16__release_sharedEv,
 346                    fake_shared_weak_count *o) {
 347   if (!flags()->shared_ptr_interceptor)
 348     return REAL(_ZNSt3__114__shared_count16__release_sharedEv)(o);
 349
 350   SCOPED_TSAN_INTERCEPTOR(_ZNSt3__114__shared_count16__release_sharedEv, o);
 351   if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) {
 352     Acquire(thr, pc, (uptr)&o->shared_owners);
 353     o->on_zero_shared();
 354     return true;
 355   }
 356   return false;
 357 }
 358
 359 namespace {
 360 struct call_once_callback_args {
 361   void (*orig_func)(void *arg);
 362   void *orig_arg;
 363   void *flag;
 364 };
 365
 366 void call_once_callback_wrapper(void *arg) {
 367   call_once_callback_args *new_args = (call_once_callback_args *)arg;
 368   new_args->orig_func(new_args->orig_arg);
 369   __tsan_release(new_args->flag);
 370 }
 371 }  // namespace
 372
 373 // This adds a libc++ interceptor for:
 374 //     void __call_once(volatile unsigned long&, void*, void(*)(void*));
 375 // C++11 call_once is implemented via an internal function __call_once which is
 376 // inside libc++.dylib, and the atomic release store inside it is thus
 377 // TSan-invisible. To avoid false positives, this interceptor wraps the callback
 378 // function and performs an explicit Release after the user code has run.
 379 STDCXX_INTERCEPTOR(void, _ZNSt3__111__call_onceERVmPvPFvS2_E, void *flag,
 380                    void *arg, void (*func)(void *arg)) {
 381   call_once_callback_args new_args = {func, arg, flag};
 382   REAL(_ZNSt3__111__call_onceERVmPvPFvS2_E)(flag, &new_args,
 383                                             call_once_callback_wrapper);
 384 }
 385
 386 }  // namespace __tsan
 387
 388 #endif  // SANITIZER_MAC