contrib/compiler-rt/lib/tsan/rtl/tsan_libdispatch_mac.cc

   1 //===-- tsan_libdispatch_mac.cc -------------------------------------------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // This file is a part of ThreadSanitizer (TSan), a race detector.
  11 //
  12 // Mac-specific libdispatch (GCD) support.
  13 //===----------------------------------------------------------------------===//
  14
  15 #include "sanitizer_common/sanitizer_platform.h"
  16 #if SANITIZER_MAC
  17
  18 #include "sanitizer_common/sanitizer_common.h"
  19 #include "interception/interception.h"
  20 #include "tsan_interceptors.h"
  21 #include "tsan_platform.h"
  22 #include "tsan_rtl.h"
  23
  24 #include <Block.h>
  25 #include <dispatch/dispatch.h>
  26 #include <pthread.h>
  27
  28 // DISPATCH_NOESCAPE is not defined prior to XCode 8.
  29 #ifndef DISPATCH_NOESCAPE
  30 #define DISPATCH_NOESCAPE
  31 #endif
  32
  33 typedef long long_t;  // NOLINT
  34
  35 namespace __tsan {
  36
  37 typedef struct {
  38   dispatch_queue_t queue;
  39   void *orig_context;
  40   dispatch_function_t orig_work;
  41   bool free_context_in_callback;
  42   bool submitted_synchronously;
  43   bool is_barrier_block;
  44   uptr non_queue_sync_object;
  45 } tsan_block_context_t;
  46
  47 // The offsets of different fields of the dispatch_queue_t structure, exported
  48 // by libdispatch.dylib.
  49 extern "C" struct dispatch_queue_offsets_s {
  50   const uint16_t dqo_version;
  51   const uint16_t dqo_label;
  52   const uint16_t dqo_label_size;
  53   const uint16_t dqo_flags;
  54   const uint16_t dqo_flags_size;
  55   const uint16_t dqo_serialnum;
  56   const uint16_t dqo_serialnum_size;
  57   const uint16_t dqo_width;
  58   const uint16_t dqo_width_size;
  59   const uint16_t dqo_running;
  60   const uint16_t dqo_running_size;
  61   const uint16_t dqo_suspend_cnt;
  62   const uint16_t dqo_suspend_cnt_size;
  63   const uint16_t dqo_target_queue;
  64   const uint16_t dqo_target_queue_size;
  65   const uint16_t dqo_priority;
  66   const uint16_t dqo_priority_size;
  67 } dispatch_queue_offsets;
  68
  69 static bool IsQueueSerial(dispatch_queue_t q) {
  70   CHECK_EQ(dispatch_queue_offsets.dqo_width_size, 2);
  71   uptr width = *(uint16_t *)(((uptr)q) + dispatch_queue_offsets.dqo_width);
  72   CHECK_NE(width, 0);
  73   return width == 1;
  74 }
  75
  76 static dispatch_queue_t GetTargetQueueFromQueue(dispatch_queue_t q) {
  77   CHECK_EQ(dispatch_queue_offsets.dqo_target_queue_size, 8);
  78   dispatch_queue_t tq = *(
  79       dispatch_queue_t *)(((uptr)q) + dispatch_queue_offsets.dqo_target_queue);
  80   return tq;
  81 }
  82
  83 static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) {
  84   dispatch_queue_t tq = GetTargetQueueFromQueue((dispatch_queue_t)source);
  85   CHECK_NE(tq, 0);
  86   return tq;
  87 }
  88
  89 static tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc,
  90                                           dispatch_queue_t queue,
  91                                           void *orig_context,
  92                                           dispatch_function_t orig_work) {
  93   tsan_block_context_t *new_context =
  94       (tsan_block_context_t *)user_alloc_internal(thr, pc,
  95                                                   sizeof(tsan_block_context_t));
  96   new_context->queue = queue;
  97   new_context->orig_context = orig_context;
  98   new_context->orig_work = orig_work;
  99   new_context->free_context_in_callback = true;
 100   new_context->submitted_synchronously = false;
 101   new_context->is_barrier_block = false;
 102   new_context->non_queue_sync_object = 0;
 103   return new_context;
 104 }
 105
 106 #define GET_QUEUE_SYNC_VARS(context, q)                                  \
 107   bool is_queue_serial = q && IsQueueSerial(q);                          \
 108   uptr sync_ptr = (uptr)q ?: context->non_queue_sync_object;             \
 109   uptr serial_sync = (uptr)sync_ptr;                                     \
 110   uptr concurrent_sync = sync_ptr ? ((uptr)sync_ptr) + sizeof(uptr) : 0; \
 111   bool serial_task = context->is_barrier_block || is_queue_serial
 112
 113 static void dispatch_sync_pre_execute(ThreadState *thr, uptr pc,
 114                                       tsan_block_context_t *context) {
 115   uptr submit_sync = (uptr)context;
 116   Acquire(thr, pc, submit_sync);
 117
 118   dispatch_queue_t q = context->queue;
 119   do {
 120     GET_QUEUE_SYNC_VARS(context, q);
 121     if (serial_sync) Acquire(thr, pc, serial_sync);
 122     if (serial_task && concurrent_sync) Acquire(thr, pc, concurrent_sync);
 123
 124     if (q) q = GetTargetQueueFromQueue(q);
 125   } while (q);
 126 }
 127
 128 static void dispatch_sync_post_execute(ThreadState *thr, uptr pc,
 129                                        tsan_block_context_t *context) {
 130   uptr submit_sync = (uptr)context;
 131   if (context->submitted_synchronously) Release(thr, pc, submit_sync);
 132
 133   dispatch_queue_t q = context->queue;
 134   do {
 135     GET_QUEUE_SYNC_VARS(context, q);
 136     if (serial_task && serial_sync) Release(thr, pc, serial_sync);
 137     if (!serial_task && concurrent_sync) Release(thr, pc, concurrent_sync);
 138
 139     if (q) q = GetTargetQueueFromQueue(q);
 140   } while (q);
 141 }
 142
 143 static void dispatch_callback_wrap(void *param) {
 144   SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap);
 145   tsan_block_context_t *context = (tsan_block_context_t *)param;
 146
 147   dispatch_sync_pre_execute(thr, pc, context);
 148
 149   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 150   context->orig_work(context->orig_context);
 151   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 152
 153   dispatch_sync_post_execute(thr, pc, context);
 154
 155   if (context->free_context_in_callback) user_free(thr, pc, context);
 156 }
 157
 158 static void invoke_block(void *param) {
 159   dispatch_block_t block = (dispatch_block_t)param;
 160   block();
 161 }
 162
 163 static void invoke_and_release_block(void *param) {
 164   dispatch_block_t block = (dispatch_block_t)param;
 165   block();
 166   Block_release(block);
 167 }
 168
 169 #define DISPATCH_INTERCEPT_B(name, barrier)                                  \
 170   TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \
 171     SCOPED_TSAN_INTERCEPTOR(name, q, block);                                 \
 172     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
 173     dispatch_block_t heap_block = Block_copy(block);                         \
 174     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                             \
 175     tsan_block_context_t *new_context =                                      \
 176         AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);     \
 177     new_context->is_barrier_block = barrier;                                 \
 178     Release(thr, pc, (uptr)new_context);                                     \
 179     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
 180     REAL(name##_f)(q, new_context, dispatch_callback_wrap);                  \
 181     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                             \
 182   }
 183
 184 #define DISPATCH_INTERCEPT_SYNC_B(name, barrier)                             \
 185   TSAN_INTERCEPTOR(void, name, dispatch_queue_t q,                           \
 186                    DISPATCH_NOESCAPE dispatch_block_t block) {               \
 187     SCOPED_TSAN_INTERCEPTOR(name, q, block);                                 \
 188     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
 189     dispatch_block_t heap_block = Block_copy(block);                         \
 190     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                             \
 191     tsan_block_context_t new_context = {                                     \
 192         q, heap_block, &invoke_and_release_block, false, true, barrier, 0};  \
 193     Release(thr, pc, (uptr)&new_context);                                    \
 194     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
 195     REAL(name##_f)(q, &new_context, dispatch_callback_wrap);                 \
 196     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                             \
 197     Acquire(thr, pc, (uptr)&new_context);                                    \
 198   }
 199
 200 #define DISPATCH_INTERCEPT_F(name, barrier)                       \
 201   TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \
 202                    dispatch_function_t work) {                    \
 203     SCOPED_TSAN_INTERCEPTOR(name, q, context, work);              \
 204     tsan_block_context_t *new_context =                           \
 205         AllocContext(thr, pc, q, context, work);                  \
 206     new_context->is_barrier_block = barrier;                      \
 207     Release(thr, pc, (uptr)new_context);                          \
 208     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                \
 209     REAL(name)(q, new_context, dispatch_callback_wrap);           \
 210     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                  \
 211   }
 212
 213 #define DISPATCH_INTERCEPT_SYNC_F(name, barrier)                              \
 214   TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context,             \
 215                    dispatch_function_t work) {                                \
 216     SCOPED_TSAN_INTERCEPTOR(name, q, context, work);                          \
 217     tsan_block_context_t new_context = {                                      \
 218         q, context, work, false, true, barrier, 0};                           \
 219     Release(thr, pc, (uptr)&new_context);                                     \
 220     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                            \
 221     REAL(name)(q, &new_context, dispatch_callback_wrap);                      \
 222     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                              \
 223     Acquire(thr, pc, (uptr)&new_context);                                     \
 224   }
 225
 226 // We wrap dispatch_async, dispatch_sync and friends where we allocate a new
 227 // context, which is used to synchronize (we release the context before
 228 // submitting, and the callback acquires it before executing the original
 229 // callback).
 230 DISPATCH_INTERCEPT_B(dispatch_async, false)
 231 DISPATCH_INTERCEPT_B(dispatch_barrier_async, true)
 232 DISPATCH_INTERCEPT_F(dispatch_async_f, false)
 233 DISPATCH_INTERCEPT_F(dispatch_barrier_async_f, true)
 234 DISPATCH_INTERCEPT_SYNC_B(dispatch_sync, false)
 235 DISPATCH_INTERCEPT_SYNC_B(dispatch_barrier_sync, true)
 236 DISPATCH_INTERCEPT_SYNC_F(dispatch_sync_f, false)
 237 DISPATCH_INTERCEPT_SYNC_F(dispatch_barrier_sync_f, true)
 238
 239 TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when,
 240                  dispatch_queue_t queue, dispatch_block_t block) {
 241   SCOPED_TSAN_INTERCEPTOR(dispatch_after, when, queue, block);
 242   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 243   dispatch_block_t heap_block = Block_copy(block);
 244   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 245   tsan_block_context_t *new_context =
 246       AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block);
 247   Release(thr, pc, (uptr)new_context);
 248   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 249   REAL(dispatch_after_f)(when, queue, new_context, dispatch_callback_wrap);
 250   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 251 }
 252
 253 TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when,
 254                  dispatch_queue_t queue, void *context,
 255                  dispatch_function_t work) {
 256   SCOPED_TSAN_INTERCEPTOR(dispatch_after_f, when, queue, context, work);
 257   WRAP(dispatch_after)(when, queue, ^(void) {
 258     work(context);
 259   });
 260 }
 261
 262 // GCD's dispatch_once implementation has a fast path that contains a racy read
 263 // and it's inlined into user's code. Furthermore, this fast path doesn't
 264 // establish a proper happens-before relations between the initialization and
 265 // code following the call to dispatch_once. We could deal with this in
 266 // instrumented code, but there's not much we can do about it in system
 267 // libraries. Let's disable the fast path (by never storing the value ~0 to
 268 // predicate), so the interceptor is always called, and let's add proper release
 269 // and acquire semantics. Since TSan does not see its own atomic stores, the
 270 // race on predicate won't be reported - the only accesses to it that TSan sees
 271 // are the loads on the fast path. Loads don't race. Secondly, dispatch_once is
 272 // both a macro and a real function, we want to intercept the function, so we
 273 // need to undefine the macro.
 274 #undef dispatch_once
 275 TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate,
 276                  DISPATCH_NOESCAPE dispatch_block_t block) {
 277   SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block);
 278   atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate);
 279   u32 v = atomic_load(a, memory_order_acquire);
 280   if (v == 0 &&
 281       atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) {
 282     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 283     block();
 284     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 285     Release(thr, pc, (uptr)a);
 286     atomic_store(a, 2, memory_order_release);
 287   } else {
 288     while (v != 2) {
 289       internal_sched_yield();
 290       v = atomic_load(a, memory_order_acquire);
 291     }
 292     Acquire(thr, pc, (uptr)a);
 293   }
 294 }
 295
 296 #undef dispatch_once_f
 297 TSAN_INTERCEPTOR(void, dispatch_once_f, dispatch_once_t *predicate,
 298                  void *context, dispatch_function_t function) {
 299   SCOPED_INTERCEPTOR_RAW(dispatch_once_f, predicate, context, function);
 300   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 301   WRAP(dispatch_once)(predicate, ^(void) {
 302     function(context);
 303   });
 304   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 305 }
 306
 307 TSAN_INTERCEPTOR(long_t, dispatch_semaphore_signal,
 308                  dispatch_semaphore_t dsema) {
 309   SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal, dsema);
 310   Release(thr, pc, (uptr)dsema);
 311   return REAL(dispatch_semaphore_signal)(dsema);
 312 }
 313
 314 TSAN_INTERCEPTOR(long_t, dispatch_semaphore_wait, dispatch_semaphore_t dsema,
 315                  dispatch_time_t timeout) {
 316   SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait, dsema, timeout);
 317   long_t result = REAL(dispatch_semaphore_wait)(dsema, timeout);
 318   if (result == 0) Acquire(thr, pc, (uptr)dsema);
 319   return result;
 320 }
 321
 322 TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group,
 323                  dispatch_time_t timeout) {
 324   SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait, group, timeout);
 325   long_t result = REAL(dispatch_group_wait)(group, timeout);
 326   if (result == 0) Acquire(thr, pc, (uptr)group);
 327   return result;
 328 }
 329
 330 TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) {
 331   SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group);
 332   // Acquired in the group noticifaction callback in dispatch_group_notify[_f].
 333   Release(thr, pc, (uptr)group);
 334   REAL(dispatch_group_leave)(group);
 335 }
 336
 337 TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group,
 338                  dispatch_queue_t queue, dispatch_block_t block) {
 339   SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block);
 340   dispatch_retain(group);
 341   dispatch_group_enter(group);
 342   __block dispatch_block_t block_copy = (dispatch_block_t)_Block_copy(block);
 343   WRAP(dispatch_async)(queue, ^(void) {
 344     block_copy();
 345     _Block_release(block_copy);
 346     WRAP(dispatch_group_leave)(group);
 347     dispatch_release(group);
 348   });
 349 }
 350
 351 TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group,
 352                  dispatch_queue_t queue, void *context,
 353                  dispatch_function_t work) {
 354   SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f, group, queue, context, work);
 355   dispatch_retain(group);
 356   dispatch_group_enter(group);
 357   WRAP(dispatch_async)(queue, ^(void) {
 358     work(context);
 359     WRAP(dispatch_group_leave)(group);
 360     dispatch_release(group);
 361   });
 362 }
 363
 364 TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group,
 365                  dispatch_queue_t q, dispatch_block_t block) {
 366   SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block);
 367
 368   // To make sure the group is still available in the callback (otherwise
 369   // it can be already destroyed).  Will be released in the callback.
 370   dispatch_retain(group);
 371
 372   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 373   dispatch_block_t heap_block = Block_copy(^(void) {
 374     {
 375       SCOPED_INTERCEPTOR_RAW(dispatch_read_callback);
 376       // Released when leaving the group (dispatch_group_leave).
 377       Acquire(thr, pc, (uptr)group);
 378     }
 379     dispatch_release(group);
 380     block();
 381   });
 382   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 383   tsan_block_context_t *new_context =
 384       AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
 385   new_context->is_barrier_block = true;
 386   Release(thr, pc, (uptr)new_context);
 387   REAL(dispatch_group_notify_f)(group, q, new_context, dispatch_callback_wrap);
 388 }
 389
 390 TSAN_INTERCEPTOR(void, dispatch_group_notify_f, dispatch_group_t group,
 391                  dispatch_queue_t q, void *context, dispatch_function_t work) {
 392   WRAP(dispatch_group_notify)(group, q, ^(void) { work(context); });
 393 }
 394
 395 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler,
 396                  dispatch_source_t source, dispatch_block_t handler) {
 397   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler, source, handler);
 398   if (handler == nullptr)
 399     return REAL(dispatch_source_set_event_handler)(source, nullptr);
 400   dispatch_queue_t q = GetTargetQueueFromSource(source);
 401   __block tsan_block_context_t new_context = {
 402       q, handler, &invoke_block, false, false, false, 0 };
 403   dispatch_block_t new_handler = Block_copy(^(void) {
 404     new_context.orig_context = handler;  // To explicitly capture "handler".
 405     dispatch_callback_wrap(&new_context);
 406   });
 407   uptr submit_sync = (uptr)&new_context;
 408   Release(thr, pc, submit_sync);
 409   REAL(dispatch_source_set_event_handler)(source, new_handler);
 410   Block_release(new_handler);
 411 }
 412
 413 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler_f,
 414                  dispatch_source_t source, dispatch_function_t handler) {
 415   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler_f, source, handler);
 416   if (handler == nullptr)
 417     return REAL(dispatch_source_set_event_handler)(source, nullptr);
 418   dispatch_block_t block = ^(void) {
 419     handler(dispatch_get_context(source));
 420   };
 421   WRAP(dispatch_source_set_event_handler)(source, block);
 422 }
 423
 424 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler,
 425                  dispatch_source_t source, dispatch_block_t handler) {
 426   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler, source, handler);
 427   if (handler == nullptr)
 428     return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
 429   dispatch_queue_t q = GetTargetQueueFromSource(source);
 430   __block tsan_block_context_t new_context = {
 431       q, handler, &invoke_block, false, false, false, 0};
 432   dispatch_block_t new_handler = Block_copy(^(void) {
 433     new_context.orig_context = handler;  // To explicitly capture "handler".
 434     dispatch_callback_wrap(&new_context);
 435   });
 436   uptr submit_sync = (uptr)&new_context;
 437   Release(thr, pc, submit_sync);
 438   REAL(dispatch_source_set_cancel_handler)(source, new_handler);
 439   Block_release(new_handler);
 440 }
 441
 442 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler_f,
 443                  dispatch_source_t source, dispatch_function_t handler) {
 444   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler_f, source,
 445                           handler);
 446   if (handler == nullptr)
 447     return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
 448   dispatch_block_t block = ^(void) {
 449     handler(dispatch_get_context(source));
 450   };
 451   WRAP(dispatch_source_set_cancel_handler)(source, block);
 452 }
 453
 454 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler,
 455                  dispatch_source_t source, dispatch_block_t handler) {
 456   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler, source,
 457                           handler);
 458   if (handler == nullptr)
 459     return REAL(dispatch_source_set_registration_handler)(source, nullptr);
 460   dispatch_queue_t q = GetTargetQueueFromSource(source);
 461   __block tsan_block_context_t new_context = {
 462       q, handler, &invoke_block, false, false, false, 0};
 463   dispatch_block_t new_handler = Block_copy(^(void) {
 464     new_context.orig_context = handler;  // To explicitly capture "handler".
 465     dispatch_callback_wrap(&new_context);
 466   });
 467   uptr submit_sync = (uptr)&new_context;
 468   Release(thr, pc, submit_sync);
 469   REAL(dispatch_source_set_registration_handler)(source, new_handler);
 470   Block_release(new_handler);
 471 }
 472
 473 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f,
 474                  dispatch_source_t source, dispatch_function_t handler) {
 475   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler_f, source,
 476                           handler);
 477   if (handler == nullptr)
 478     return REAL(dispatch_source_set_registration_handler)(source, nullptr);
 479   dispatch_block_t block = ^(void) {
 480     handler(dispatch_get_context(source));
 481   };
 482   WRAP(dispatch_source_set_registration_handler)(source, block);
 483 }
 484
 485 TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations,
 486                  dispatch_queue_t queue,
 487                  DISPATCH_NOESCAPE void (^block)(size_t)) {
 488   SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block);
 489
 490   void *parent_to_child_sync = nullptr;
 491   uptr parent_to_child_sync_uptr = (uptr)&parent_to_child_sync;
 492   void *child_to_parent_sync = nullptr;
 493   uptr child_to_parent_sync_uptr = (uptr)&child_to_parent_sync;
 494
 495   Release(thr, pc, parent_to_child_sync_uptr);
 496   void (^new_block)(size_t) = ^(size_t iteration) {
 497     SCOPED_INTERCEPTOR_RAW(dispatch_apply);
 498     Acquire(thr, pc, parent_to_child_sync_uptr);
 499     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 500     block(iteration);
 501     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 502     Release(thr, pc, child_to_parent_sync_uptr);
 503   };
 504   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 505   REAL(dispatch_apply)(iterations, queue, new_block);
 506   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 507   Acquire(thr, pc, child_to_parent_sync_uptr);
 508 }
 509
 510 TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations,
 511                  dispatch_queue_t queue, void *context,
 512                  void (*work)(void *, size_t)) {
 513   SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work);
 514   void (^new_block)(size_t) = ^(size_t iteration) {
 515     work(context, iteration);
 516   };
 517   WRAP(dispatch_apply)(iterations, queue, new_block);
 518 }
 519
 520 DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
 521 DECLARE_REAL_AND_INTERCEPTOR(int, munmap, void *addr, long_t sz)
 522
 523 TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer,
 524                  size_t size, dispatch_queue_t q, dispatch_block_t destructor) {
 525   SCOPED_TSAN_INTERCEPTOR(dispatch_data_create, buffer, size, q, destructor);
 526   if ((q == nullptr) || (destructor == DISPATCH_DATA_DESTRUCTOR_DEFAULT))
 527     return REAL(dispatch_data_create)(buffer, size, q, destructor);
 528
 529   if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE)
 530     destructor = ^(void) { WRAP(free)((void *)(uintptr_t)buffer); };
 531   else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP)
 532     destructor = ^(void) { WRAP(munmap)((void *)(uintptr_t)buffer, size); };
 533
 534   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 535   dispatch_block_t heap_block = Block_copy(destructor);
 536   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 537   tsan_block_context_t *new_context =
 538       AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
 539   uptr submit_sync = (uptr)new_context;
 540   Release(thr, pc, submit_sync);
 541   return REAL(dispatch_data_create)(buffer, size, q, ^(void) {
 542     dispatch_callback_wrap(new_context);
 543   });
 544 }
 545
 546 typedef void (^fd_handler_t)(dispatch_data_t data, int error);
 547 typedef void (^cleanup_handler_t)(int error);
 548
 549 TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length,
 550                  dispatch_queue_t q, fd_handler_t h) {
 551   SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h);
 552   __block tsan_block_context_t new_context = {
 553       q, nullptr, &invoke_block, false, false, false, 0};
 554   fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
 555     new_context.orig_context = ^(void) {
 556       h(data, error);
 557     };
 558     dispatch_callback_wrap(&new_context);
 559   });
 560   uptr submit_sync = (uptr)&new_context;
 561   Release(thr, pc, submit_sync);
 562   REAL(dispatch_read)(fd, length, q, new_h);
 563   Block_release(new_h);
 564 }
 565
 566 TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data,
 567                  dispatch_queue_t q, fd_handler_t h) {
 568   SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h);
 569   __block tsan_block_context_t new_context = {
 570       q, nullptr, &invoke_block, false, false, false, 0};
 571   fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
 572     new_context.orig_context = ^(void) {
 573       h(data, error);
 574     };
 575     dispatch_callback_wrap(&new_context);
 576   });
 577   uptr submit_sync = (uptr)&new_context;
 578   Release(thr, pc, submit_sync);
 579   REAL(dispatch_write)(fd, data, q, new_h);
 580   Block_release(new_h);
 581 }
 582
 583 TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset,
 584                  size_t length, dispatch_queue_t q, dispatch_io_handler_t h) {
 585   SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h);
 586   __block tsan_block_context_t new_context = {
 587       q, nullptr, &invoke_block, false, false, false, 0};
 588   dispatch_io_handler_t new_h =
 589       Block_copy(^(bool done, dispatch_data_t data, int error) {
 590         new_context.orig_context = ^(void) {
 591           h(done, data, error);
 592         };
 593         dispatch_callback_wrap(&new_context);
 594       });
 595   uptr submit_sync = (uptr)&new_context;
 596   Release(thr, pc, submit_sync);
 597   REAL(dispatch_io_read)(channel, offset, length, q, new_h);
 598   Block_release(new_h);
 599 }
 600
 601 TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset,
 602                  dispatch_data_t data, dispatch_queue_t q,
 603                  dispatch_io_handler_t h) {
 604   SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h);
 605   __block tsan_block_context_t new_context = {
 606       q, nullptr, &invoke_block, false, false, false, 0};
 607   dispatch_io_handler_t new_h =
 608       Block_copy(^(bool done, dispatch_data_t data, int error) {
 609         new_context.orig_context = ^(void) {
 610           h(done, data, error);
 611         };
 612         dispatch_callback_wrap(&new_context);
 613       });
 614   uptr submit_sync = (uptr)&new_context;
 615   Release(thr, pc, submit_sync);
 616   REAL(dispatch_io_write)(channel, offset, data, q, new_h);
 617   Block_release(new_h);
 618 }
 619
 620 TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel,
 621                  dispatch_block_t barrier) {
 622   SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier);
 623   __block tsan_block_context_t new_context = {
 624       nullptr, nullptr, &invoke_block, false, false, false, 0};
 625   new_context.non_queue_sync_object = (uptr)channel;
 626   new_context.is_barrier_block = true;
 627   dispatch_block_t new_block = Block_copy(^(void) {
 628     new_context.orig_context = ^(void) {
 629       barrier();
 630     };
 631     dispatch_callback_wrap(&new_context);
 632   });
 633   uptr submit_sync = (uptr)&new_context;
 634   Release(thr, pc, submit_sync);
 635   REAL(dispatch_io_barrier)(channel, new_block);
 636   Block_release(new_block);
 637 }
 638
 639 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type,
 640                  dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) {
 641   SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h);
 642   __block dispatch_io_t new_channel = nullptr;
 643   __block tsan_block_context_t new_context = {
 644       q, nullptr, &invoke_block, false, false, false, 0};
 645   cleanup_handler_t new_h = Block_copy(^(int error) {
 646     {
 647       SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
 648       Acquire(thr, pc, (uptr)new_channel);  // Release() in dispatch_io_close.
 649     }
 650     new_context.orig_context = ^(void) {
 651       h(error);
 652     };
 653     dispatch_callback_wrap(&new_context);
 654   });
 655   uptr submit_sync = (uptr)&new_context;
 656   Release(thr, pc, submit_sync);
 657   new_channel = REAL(dispatch_io_create)(type, fd, q, new_h);
 658   Block_release(new_h);
 659   return new_channel;
 660 }
 661
 662 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path,
 663                  dispatch_io_type_t type, const char *path, int oflag,
 664                  mode_t mode, dispatch_queue_t q, cleanup_handler_t h) {
 665   SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode,
 666                           q, h);
 667   __block dispatch_io_t new_channel = nullptr;
 668   __block tsan_block_context_t new_context = {
 669       q, nullptr, &invoke_block, false, false, false, 0};
 670   cleanup_handler_t new_h = Block_copy(^(int error) {
 671     {
 672       SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
 673       Acquire(thr, pc, (uptr)new_channel);  // Release() in dispatch_io_close.
 674     }
 675     new_context.orig_context = ^(void) {
 676       h(error);
 677     };
 678     dispatch_callback_wrap(&new_context);
 679   });
 680   uptr submit_sync = (uptr)&new_context;
 681   Release(thr, pc, submit_sync);
 682   new_channel =
 683       REAL(dispatch_io_create_with_path)(type, path, oflag, mode, q, new_h);
 684   Block_release(new_h);
 685   return new_channel;
 686 }
 687
 688 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io,
 689                  dispatch_io_type_t type, dispatch_io_t io, dispatch_queue_t q,
 690                  cleanup_handler_t h) {
 691   SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h);
 692   __block dispatch_io_t new_channel = nullptr;
 693   __block tsan_block_context_t new_context = {
 694       q, nullptr, &invoke_block, false, false, false, 0};
 695   cleanup_handler_t new_h = Block_copy(^(int error) {
 696     {
 697       SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
 698       Acquire(thr, pc, (uptr)new_channel);  // Release() in dispatch_io_close.
 699     }
 700     new_context.orig_context = ^(void) {
 701       h(error);
 702     };
 703     dispatch_callback_wrap(&new_context);
 704   });
 705   uptr submit_sync = (uptr)&new_context;
 706   Release(thr, pc, submit_sync);
 707   new_channel = REAL(dispatch_io_create_with_io)(type, io, q, new_h);
 708   Block_release(new_h);
 709   return new_channel;
 710 }
 711
 712 TSAN_INTERCEPTOR(void, dispatch_io_close, dispatch_io_t channel,
 713                  dispatch_io_close_flags_t flags) {
 714   SCOPED_TSAN_INTERCEPTOR(dispatch_io_close, channel, flags);
 715   Release(thr, pc, (uptr)channel);  // Acquire() in dispatch_io_create[_*].
 716   return REAL(dispatch_io_close)(channel, flags);
 717 }
 718
 719 // Resuming a suspended queue needs to synchronize with all subsequent
 720 // executions of blocks in that queue.
 721 TSAN_INTERCEPTOR(void, dispatch_resume, dispatch_object_t o) {
 722   SCOPED_TSAN_INTERCEPTOR(dispatch_resume, o);
 723   Release(thr, pc, (uptr)o);  // Synchronizes with the Acquire() on serial_sync
 724                               // in dispatch_sync_pre_execute
 725   return REAL(dispatch_resume)(o);
 726 }
 727
 728 }  // namespace __tsan
 729
 730 #endif  // SANITIZER_MAC