contrib/compiler-rt/lib/tsan/rtl/tsan_libdispatch_mac.cc

   1 //===-- tsan_libdispatch_mac.cc -------------------------------------------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // This file is a part of ThreadSanitizer (TSan), a race detector.
  11 //
  12 // Mac-specific libdispatch (GCD) support.
  13 //===----------------------------------------------------------------------===//
  14
  15 #include "sanitizer_common/sanitizer_platform.h"
  16 #if SANITIZER_MAC
  17
  18 #include "sanitizer_common/sanitizer_common.h"
  19 #include "interception/interception.h"
  20 #include "tsan_interceptors.h"
  21 #include "tsan_platform.h"
  22 #include "tsan_rtl.h"
  23
  24 #include <Block.h>
  25 #include <dispatch/dispatch.h>
  26 #include <pthread.h>
  27
  28 typedef long long_t;  // NOLINT
  29
  30 namespace __tsan {
  31
  32 typedef struct {
  33   dispatch_queue_t queue;
  34   void *orig_context;
  35   dispatch_function_t orig_work;
  36   bool free_context_in_callback;
  37   bool submitted_synchronously;
  38   bool is_barrier_block;
  39   uptr non_queue_sync_object;
  40 } tsan_block_context_t;
  41
  42 // The offsets of different fields of the dispatch_queue_t structure, exported
  43 // by libdispatch.dylib.
  44 extern "C" struct dispatch_queue_offsets_s {
  45   const uint16_t dqo_version;
  46   const uint16_t dqo_label;
  47   const uint16_t dqo_label_size;
  48   const uint16_t dqo_flags;
  49   const uint16_t dqo_flags_size;
  50   const uint16_t dqo_serialnum;
  51   const uint16_t dqo_serialnum_size;
  52   const uint16_t dqo_width;
  53   const uint16_t dqo_width_size;
  54   const uint16_t dqo_running;
  55   const uint16_t dqo_running_size;
  56   const uint16_t dqo_suspend_cnt;
  57   const uint16_t dqo_suspend_cnt_size;
  58   const uint16_t dqo_target_queue;
  59   const uint16_t dqo_target_queue_size;
  60   const uint16_t dqo_priority;
  61   const uint16_t dqo_priority_size;
  62 } dispatch_queue_offsets;
  63
  64 static bool IsQueueSerial(dispatch_queue_t q) {
  65   CHECK_EQ(dispatch_queue_offsets.dqo_width_size, 2);
  66   uptr width = *(uint16_t *)(((uptr)q) + dispatch_queue_offsets.dqo_width);
  67   CHECK_NE(width, 0);
  68   return width == 1;
  69 }
  70
  71 static dispatch_queue_t GetTargetQueueFromQueue(dispatch_queue_t q) {
  72   CHECK_EQ(dispatch_queue_offsets.dqo_target_queue_size, 8);
  73   dispatch_queue_t tq = *(
  74       dispatch_queue_t *)(((uptr)q) + dispatch_queue_offsets.dqo_target_queue);
  75   return tq;
  76 }
  77
  78 static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) {
  79   dispatch_queue_t tq = GetTargetQueueFromQueue((dispatch_queue_t)source);
  80   CHECK_NE(tq, 0);
  81   return tq;
  82 }
  83
  84 static tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc,
  85                                           dispatch_queue_t queue,
  86                                           void *orig_context,
  87                                           dispatch_function_t orig_work) {
  88   tsan_block_context_t *new_context =
  89       (tsan_block_context_t *)user_alloc(thr, pc, sizeof(tsan_block_context_t));
  90   new_context->queue = queue;
  91   new_context->orig_context = orig_context;
  92   new_context->orig_work = orig_work;
  93   new_context->free_context_in_callback = true;
  94   new_context->submitted_synchronously = false;
  95   new_context->is_barrier_block = false;
  96   return new_context;
  97 }
  98
  99 #define GET_QUEUE_SYNC_VARS(context, q)                      \
 100   bool is_queue_serial = q && IsQueueSerial(q);              \
 101   uptr sync_ptr = (uptr)q ?: context->non_queue_sync_object; \
 102   uptr serial_sync = (uptr)sync_ptr;                         \
 103   uptr concurrent_sync = ((uptr)sync_ptr) + sizeof(uptr);    \
 104   bool serial_task = context->is_barrier_block || is_queue_serial
 105
 106 static void dispatch_sync_pre_execute(ThreadState *thr, uptr pc,
 107                                       tsan_block_context_t *context) {
 108   uptr submit_sync = (uptr)context;
 109   Acquire(thr, pc, submit_sync);
 110
 111   dispatch_queue_t q = context->queue;
 112   do {
 113     GET_QUEUE_SYNC_VARS(context, q);
 114     Acquire(thr, pc, serial_sync);
 115     if (serial_task) Acquire(thr, pc, concurrent_sync);
 116
 117     if (q) q = GetTargetQueueFromQueue(q);
 118   } while (q);
 119 }
 120
 121 static void dispatch_sync_post_execute(ThreadState *thr, uptr pc,
 122                                        tsan_block_context_t *context) {
 123   uptr submit_sync = (uptr)context;
 124   if (context->submitted_synchronously) Release(thr, pc, submit_sync);
 125
 126   dispatch_queue_t q = context->queue;
 127   do {
 128     GET_QUEUE_SYNC_VARS(context, q);
 129     Release(thr, pc, serial_task ? serial_sync : concurrent_sync);
 130
 131     if (q) q = GetTargetQueueFromQueue(q);
 132   } while (q);
 133 }
 134
 135 static void dispatch_callback_wrap(void *param) {
 136   SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap);
 137   tsan_block_context_t *context = (tsan_block_context_t *)param;
 138
 139   dispatch_sync_pre_execute(thr, pc, context);
 140
 141   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 142   context->orig_work(context->orig_context);
 143   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 144
 145   dispatch_sync_post_execute(thr, pc, context);
 146
 147   if (context->free_context_in_callback) user_free(thr, pc, context);
 148 }
 149
 150 static void invoke_block(void *param) {
 151   dispatch_block_t block = (dispatch_block_t)param;
 152   block();
 153 }
 154
 155 static void invoke_and_release_block(void *param) {
 156   dispatch_block_t block = (dispatch_block_t)param;
 157   block();
 158   Block_release(block);
 159 }
 160
 161 #define DISPATCH_INTERCEPT_B(name, barrier)                                  \
 162   TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \
 163     SCOPED_TSAN_INTERCEPTOR(name, q, block);                                 \
 164     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
 165     dispatch_block_t heap_block = Block_copy(block);                         \
 166     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                             \
 167     tsan_block_context_t *new_context =                                      \
 168         AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);     \
 169     new_context->is_barrier_block = barrier;                                 \
 170     Release(thr, pc, (uptr)new_context);                                     \
 171     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
 172     REAL(name##_f)(q, new_context, dispatch_callback_wrap);                  \
 173     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                             \
 174   }
 175
 176 #define DISPATCH_INTERCEPT_SYNC_B(name, barrier)                             \
 177   TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \
 178     SCOPED_TSAN_INTERCEPTOR(name, q, block);                                 \
 179     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
 180     dispatch_block_t heap_block = Block_copy(block);                         \
 181     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                             \
 182     tsan_block_context_t new_context = {                                     \
 183         q, heap_block, &invoke_and_release_block, false, true, barrier, 0};  \
 184     Release(thr, pc, (uptr)&new_context);                                    \
 185     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
 186     REAL(name##_f)(q, &new_context, dispatch_callback_wrap);                 \
 187     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                             \
 188     Acquire(thr, pc, (uptr)&new_context);                                    \
 189   }
 190
 191 #define DISPATCH_INTERCEPT_F(name, barrier)                       \
 192   TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \
 193                    dispatch_function_t work) {                    \
 194     SCOPED_TSAN_INTERCEPTOR(name, q, context, work);              \
 195     tsan_block_context_t *new_context =                           \
 196         AllocContext(thr, pc, q, context, work);                  \
 197     new_context->is_barrier_block = barrier;                      \
 198     Release(thr, pc, (uptr)new_context);                          \
 199     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                \
 200     REAL(name)(q, new_context, dispatch_callback_wrap);           \
 201     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                  \
 202   }
 203
 204 #define DISPATCH_INTERCEPT_SYNC_F(name, barrier)                              \
 205   TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context,             \
 206                    dispatch_function_t work) {                                \
 207     SCOPED_TSAN_INTERCEPTOR(name, q, context, work);                          \
 208     tsan_block_context_t new_context = {                                      \
 209         q, context, work, false, true, barrier, 0};                           \
 210     Release(thr, pc, (uptr)&new_context);                                     \
 211     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                            \
 212     REAL(name)(q, &new_context, dispatch_callback_wrap);                      \
 213     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                              \
 214     Acquire(thr, pc, (uptr)&new_context);                                     \
 215   }
 216
 217 // We wrap dispatch_async, dispatch_sync and friends where we allocate a new
 218 // context, which is used to synchronize (we release the context before
 219 // submitting, and the callback acquires it before executing the original
 220 // callback).
 221 DISPATCH_INTERCEPT_B(dispatch_async, false)
 222 DISPATCH_INTERCEPT_B(dispatch_barrier_async, true)
 223 DISPATCH_INTERCEPT_F(dispatch_async_f, false)
 224 DISPATCH_INTERCEPT_F(dispatch_barrier_async_f, true)
 225 DISPATCH_INTERCEPT_SYNC_B(dispatch_sync, false)
 226 DISPATCH_INTERCEPT_SYNC_B(dispatch_barrier_sync, true)
 227 DISPATCH_INTERCEPT_SYNC_F(dispatch_sync_f, false)
 228 DISPATCH_INTERCEPT_SYNC_F(dispatch_barrier_sync_f, true)
 229
 230 TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when,
 231                  dispatch_queue_t queue, dispatch_block_t block) {
 232   SCOPED_TSAN_INTERCEPTOR(dispatch_after, when, queue, block);
 233   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 234   dispatch_block_t heap_block = Block_copy(block);
 235   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 236   tsan_block_context_t *new_context =
 237       AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block);
 238   Release(thr, pc, (uptr)new_context);
 239   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 240   REAL(dispatch_after_f)(when, queue, new_context, dispatch_callback_wrap);
 241   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 242 }
 243
 244 TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when,
 245                  dispatch_queue_t queue, void *context,
 246                  dispatch_function_t work) {
 247   SCOPED_TSAN_INTERCEPTOR(dispatch_after_f, when, queue, context, work);
 248   WRAP(dispatch_after)(when, queue, ^(void) {
 249     work(context);
 250   });
 251 }
 252
 253 // GCD's dispatch_once implementation has a fast path that contains a racy read
 254 // and it's inlined into user's code. Furthermore, this fast path doesn't
 255 // establish a proper happens-before relations between the initialization and
 256 // code following the call to dispatch_once. We could deal with this in
 257 // instrumented code, but there's not much we can do about it in system
 258 // libraries. Let's disable the fast path (by never storing the value ~0 to
 259 // predicate), so the interceptor is always called, and let's add proper release
 260 // and acquire semantics. Since TSan does not see its own atomic stores, the
 261 // race on predicate won't be reported - the only accesses to it that TSan sees
 262 // are the loads on the fast path. Loads don't race. Secondly, dispatch_once is
 263 // both a macro and a real function, we want to intercept the function, so we
 264 // need to undefine the macro.
 265 #undef dispatch_once
 266 TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate,
 267                  dispatch_block_t block) {
 268   SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block);
 269   atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate);
 270   u32 v = atomic_load(a, memory_order_acquire);
 271   if (v == 0 &&
 272       atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) {
 273     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 274     block();
 275     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 276     Release(thr, pc, (uptr)a);
 277     atomic_store(a, 2, memory_order_release);
 278   } else {
 279     while (v != 2) {
 280       internal_sched_yield();
 281       v = atomic_load(a, memory_order_acquire);
 282     }
 283     Acquire(thr, pc, (uptr)a);
 284   }
 285 }
 286
 287 #undef dispatch_once_f
 288 TSAN_INTERCEPTOR(void, dispatch_once_f, dispatch_once_t *predicate,
 289                  void *context, dispatch_function_t function) {
 290   SCOPED_INTERCEPTOR_RAW(dispatch_once_f, predicate, context, function);
 291   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 292   WRAP(dispatch_once)(predicate, ^(void) {
 293     function(context);
 294   });
 295   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 296 }
 297
 298 TSAN_INTERCEPTOR(long_t, dispatch_semaphore_signal,
 299                  dispatch_semaphore_t dsema) {
 300   SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal, dsema);
 301   Release(thr, pc, (uptr)dsema);
 302   return REAL(dispatch_semaphore_signal)(dsema);
 303 }
 304
 305 TSAN_INTERCEPTOR(long_t, dispatch_semaphore_wait, dispatch_semaphore_t dsema,
 306                  dispatch_time_t timeout) {
 307   SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait, dsema, timeout);
 308   long_t result = REAL(dispatch_semaphore_wait)(dsema, timeout);
 309   if (result == 0) Acquire(thr, pc, (uptr)dsema);
 310   return result;
 311 }
 312
 313 TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group,
 314                  dispatch_time_t timeout) {
 315   SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait, group, timeout);
 316   long_t result = REAL(dispatch_group_wait)(group, timeout);
 317   if (result == 0) Acquire(thr, pc, (uptr)group);
 318   return result;
 319 }
 320
 321 TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) {
 322   SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group);
 323   // Acquired in the group noticifaction callback in dispatch_group_notify[_f].
 324   Release(thr, pc, (uptr)group);
 325   REAL(dispatch_group_leave)(group);
 326 }
 327
 328 TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group,
 329                  dispatch_queue_t queue, dispatch_block_t block) {
 330   SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block);
 331   dispatch_retain(group);
 332   dispatch_group_enter(group);
 333   __block dispatch_block_t block_copy = (dispatch_block_t)_Block_copy(block);
 334   WRAP(dispatch_async)(queue, ^(void) {
 335     block_copy();
 336     _Block_release(block_copy);
 337     WRAP(dispatch_group_leave)(group);
 338     dispatch_release(group);
 339   });
 340 }
 341
 342 TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group,
 343                  dispatch_queue_t queue, void *context,
 344                  dispatch_function_t work) {
 345   SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f, group, queue, context, work);
 346   dispatch_retain(group);
 347   dispatch_group_enter(group);
 348   WRAP(dispatch_async)(queue, ^(void) {
 349     work(context);
 350     WRAP(dispatch_group_leave)(group);
 351     dispatch_release(group);
 352   });
 353 }
 354
 355 TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group,
 356                  dispatch_queue_t q, dispatch_block_t block) {
 357   SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block);
 358
 359   // To make sure the group is still available in the callback (otherwise
 360   // it can be already destroyed).  Will be released in the callback.
 361   dispatch_retain(group);
 362
 363   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 364   dispatch_block_t heap_block = Block_copy(^(void) {
 365     {
 366       SCOPED_INTERCEPTOR_RAW(dispatch_read_callback);
 367       // Released when leaving the group (dispatch_group_leave).
 368       Acquire(thr, pc, (uptr)group);
 369     }
 370     dispatch_release(group);
 371     block();
 372   });
 373   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 374   tsan_block_context_t *new_context =
 375       AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
 376   new_context->is_barrier_block = true;
 377   Release(thr, pc, (uptr)new_context);
 378   REAL(dispatch_group_notify_f)(group, q, new_context, dispatch_callback_wrap);
 379 }
 380
 381 TSAN_INTERCEPTOR(void, dispatch_group_notify_f, dispatch_group_t group,
 382                  dispatch_queue_t q, void *context, dispatch_function_t work) {
 383   WRAP(dispatch_group_notify)(group, q, ^(void) { work(context); });
 384 }
 385
 386 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler,
 387                  dispatch_source_t source, dispatch_block_t handler) {
 388   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler, source, handler);
 389   if (handler == nullptr)
 390     return REAL(dispatch_source_set_event_handler)(source, nullptr);
 391   dispatch_queue_t q = GetTargetQueueFromSource(source);
 392   __block tsan_block_context_t new_context = {
 393       q, handler, &invoke_block, false, false, false, 0 };
 394   dispatch_block_t new_handler = Block_copy(^(void) {
 395     new_context.orig_context = handler;  // To explicitly capture "handler".
 396     dispatch_callback_wrap(&new_context);
 397   });
 398   uptr submit_sync = (uptr)&new_context;
 399   Release(thr, pc, submit_sync);
 400   REAL(dispatch_source_set_event_handler)(source, new_handler);
 401   Block_release(new_handler);
 402 }
 403
 404 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler_f,
 405                  dispatch_source_t source, dispatch_function_t handler) {
 406   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler_f, source, handler);
 407   if (handler == nullptr)
 408     return REAL(dispatch_source_set_event_handler)(source, nullptr);
 409   dispatch_block_t block = ^(void) {
 410     handler(dispatch_get_context(source));
 411   };
 412   WRAP(dispatch_source_set_event_handler)(source, block);
 413 }
 414
 415 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler,
 416                  dispatch_source_t source, dispatch_block_t handler) {
 417   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler, source, handler);
 418   if (handler == nullptr)
 419     return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
 420   dispatch_queue_t q = GetTargetQueueFromSource(source);
 421   __block tsan_block_context_t new_context = {
 422       q, handler, &invoke_block, false, false, false, 0};
 423   dispatch_block_t new_handler = Block_copy(^(void) {
 424     new_context.orig_context = handler;  // To explicitly capture "handler".
 425     dispatch_callback_wrap(&new_context);
 426   });
 427   uptr submit_sync = (uptr)&new_context;
 428   Release(thr, pc, submit_sync);
 429   REAL(dispatch_source_set_cancel_handler)(source, new_handler);
 430   Block_release(new_handler);
 431 }
 432
 433 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler_f,
 434                  dispatch_source_t source, dispatch_function_t handler) {
 435   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler_f, source,
 436                           handler);
 437   if (handler == nullptr)
 438     return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
 439   dispatch_block_t block = ^(void) {
 440     handler(dispatch_get_context(source));
 441   };
 442   WRAP(dispatch_source_set_cancel_handler)(source, block);
 443 }
 444
 445 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler,
 446                  dispatch_source_t source, dispatch_block_t handler) {
 447   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler, source,
 448                           handler);
 449   if (handler == nullptr)
 450     return REAL(dispatch_source_set_registration_handler)(source, nullptr);
 451   dispatch_queue_t q = GetTargetQueueFromSource(source);
 452   __block tsan_block_context_t new_context = {
 453       q, handler, &invoke_block, false, false, false, 0};
 454   dispatch_block_t new_handler = Block_copy(^(void) {
 455     new_context.orig_context = handler;  // To explicitly capture "handler".
 456     dispatch_callback_wrap(&new_context);
 457   });
 458   uptr submit_sync = (uptr)&new_context;
 459   Release(thr, pc, submit_sync);
 460   REAL(dispatch_source_set_registration_handler)(source, new_handler);
 461   Block_release(new_handler);
 462 }
 463
 464 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f,
 465                  dispatch_source_t source, dispatch_function_t handler) {
 466   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler_f, source,
 467                           handler);
 468   if (handler == nullptr)
 469     return REAL(dispatch_source_set_registration_handler)(source, nullptr);
 470   dispatch_block_t block = ^(void) {
 471     handler(dispatch_get_context(source));
 472   };
 473   WRAP(dispatch_source_set_registration_handler)(source, block);
 474 }
 475
 476 TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations,
 477                  dispatch_queue_t queue, void (^block)(size_t)) {
 478   SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block);
 479
 480   void *parent_to_child_sync = nullptr;
 481   uptr parent_to_child_sync_uptr = (uptr)&parent_to_child_sync;
 482   void *child_to_parent_sync = nullptr;
 483   uptr child_to_parent_sync_uptr = (uptr)&child_to_parent_sync;
 484
 485   Release(thr, pc, parent_to_child_sync_uptr);
 486   void (^new_block)(size_t) = ^(size_t iteration) {
 487     SCOPED_INTERCEPTOR_RAW(dispatch_apply);
 488     Acquire(thr, pc, parent_to_child_sync_uptr);
 489     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 490     block(iteration);
 491     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 492     Release(thr, pc, child_to_parent_sync_uptr);
 493   };
 494   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 495   REAL(dispatch_apply)(iterations, queue, new_block);
 496   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 497   Acquire(thr, pc, child_to_parent_sync_uptr);
 498 }
 499
 500 TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations,
 501                  dispatch_queue_t queue, void *context,
 502                  void (*work)(void *, size_t)) {
 503   SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work);
 504   void (^new_block)(size_t) = ^(size_t iteration) {
 505     work(context, iteration);
 506   };
 507   WRAP(dispatch_apply)(iterations, queue, new_block);
 508 }
 509
 510 DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
 511 DECLARE_REAL_AND_INTERCEPTOR(int, munmap, void *addr, long_t sz)
 512
 513 TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer,
 514                  size_t size, dispatch_queue_t q, dispatch_block_t destructor) {
 515   SCOPED_TSAN_INTERCEPTOR(dispatch_data_create, buffer, size, q, destructor);
 516   if ((q == nullptr) || (destructor == DISPATCH_DATA_DESTRUCTOR_DEFAULT))
 517     return REAL(dispatch_data_create)(buffer, size, q, destructor);
 518
 519   if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE)
 520     destructor = ^(void) { WRAP(free)((void *)buffer); };
 521   else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP)
 522     destructor = ^(void) { WRAP(munmap)((void *)buffer, size); };
 523
 524   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 525   dispatch_block_t heap_block = Block_copy(destructor);
 526   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 527   tsan_block_context_t *new_context =
 528       AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
 529   uptr submit_sync = (uptr)new_context;
 530   Release(thr, pc, submit_sync);
 531   return REAL(dispatch_data_create)(buffer, size, q, ^(void) {
 532     dispatch_callback_wrap(new_context);
 533   });
 534 }
 535
 536 typedef void (^fd_handler_t)(dispatch_data_t data, int error);
 537 typedef void (^cleanup_handler_t)(int error);
 538
 539 TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length,
 540                  dispatch_queue_t q, fd_handler_t h) {
 541   SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h);
 542   __block tsan_block_context_t new_context = {
 543       q, nullptr, &invoke_block, false, false, false, 0};
 544   fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
 545     new_context.orig_context = ^(void) {
 546       h(data, error);
 547     };
 548     dispatch_callback_wrap(&new_context);
 549   });
 550   uptr submit_sync = (uptr)&new_context;
 551   Release(thr, pc, submit_sync);
 552   REAL(dispatch_read)(fd, length, q, new_h);
 553   Block_release(new_h);
 554 }
 555
 556 TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data,
 557                  dispatch_queue_t q, fd_handler_t h) {
 558   SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h);
 559   __block tsan_block_context_t new_context = {
 560       q, nullptr, &invoke_block, false, false, false, 0};
 561   fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
 562     new_context.orig_context = ^(void) {
 563       h(data, error);
 564     };
 565     dispatch_callback_wrap(&new_context);
 566   });
 567   uptr submit_sync = (uptr)&new_context;
 568   Release(thr, pc, submit_sync);
 569   REAL(dispatch_write)(fd, data, q, new_h);
 570   Block_release(new_h);
 571 }
 572
 573 TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset,
 574                  size_t length, dispatch_queue_t q, dispatch_io_handler_t h) {
 575   SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h);
 576   __block tsan_block_context_t new_context = {
 577       q, nullptr, &invoke_block, false, false, false, 0};
 578   dispatch_io_handler_t new_h =
 579       Block_copy(^(bool done, dispatch_data_t data, int error) {
 580         new_context.orig_context = ^(void) {
 581           h(done, data, error);
 582         };
 583         dispatch_callback_wrap(&new_context);
 584       });
 585   uptr submit_sync = (uptr)&new_context;
 586   Release(thr, pc, submit_sync);
 587   REAL(dispatch_io_read)(channel, offset, length, q, new_h);
 588   Block_release(new_h);
 589 }
 590
 591 TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset,
 592                  dispatch_data_t data, dispatch_queue_t q,
 593                  dispatch_io_handler_t h) {
 594   SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h);
 595   __block tsan_block_context_t new_context = {
 596       q, nullptr, &invoke_block, false, false, false, 0};
 597   dispatch_io_handler_t new_h =
 598       Block_copy(^(bool done, dispatch_data_t data, int error) {
 599         new_context.orig_context = ^(void) {
 600           h(done, data, error);
 601         };
 602         dispatch_callback_wrap(&new_context);
 603       });
 604   uptr submit_sync = (uptr)&new_context;
 605   Release(thr, pc, submit_sync);
 606   REAL(dispatch_io_write)(channel, offset, data, q, new_h);
 607   Block_release(new_h);
 608 }
 609
 610 TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel,
 611                  dispatch_block_t barrier) {
 612   SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier);
 613   __block tsan_block_context_t new_context = {
 614       nullptr, nullptr, &invoke_block, false, false, false, 0};
 615   new_context.non_queue_sync_object = (uptr)channel;
 616   new_context.is_barrier_block = true;
 617   dispatch_block_t new_block = Block_copy(^(void) {
 618     new_context.orig_context = ^(void) {
 619       barrier();
 620     };
 621     dispatch_callback_wrap(&new_context);
 622   });
 623   uptr submit_sync = (uptr)&new_context;
 624   Release(thr, pc, submit_sync);
 625   REAL(dispatch_io_barrier)(channel, new_block);
 626   Block_release(new_block);
 627 }
 628
 629 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type,
 630                  dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) {
 631   SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h);
 632   __block dispatch_io_t new_channel = nullptr;
 633   __block tsan_block_context_t new_context = {
 634       q, nullptr, &invoke_block, false, false, false, 0};
 635   cleanup_handler_t new_h = Block_copy(^(int error) {
 636     {
 637       SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
 638       Acquire(thr, pc, (uptr)new_channel);  // Release() in dispatch_io_close.
 639     }
 640     new_context.orig_context = ^(void) {
 641       h(error);
 642     };
 643     dispatch_callback_wrap(&new_context);
 644   });
 645   uptr submit_sync = (uptr)&new_context;
 646   Release(thr, pc, submit_sync);
 647   new_channel = REAL(dispatch_io_create)(type, fd, q, new_h);
 648   Block_release(new_h);
 649   return new_channel;
 650 }
 651
 652 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path,
 653                  dispatch_io_type_t type, const char *path, int oflag,
 654                  mode_t mode, dispatch_queue_t q, cleanup_handler_t h) {
 655   SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode,
 656                           q, h);
 657   __block dispatch_io_t new_channel = nullptr;
 658   __block tsan_block_context_t new_context = {
 659       q, nullptr, &invoke_block, false, false, false, 0};
 660   cleanup_handler_t new_h = Block_copy(^(int error) {
 661     {
 662       SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
 663       Acquire(thr, pc, (uptr)new_channel);  // Release() in dispatch_io_close.
 664     }
 665     new_context.orig_context = ^(void) {
 666       h(error);
 667     };
 668     dispatch_callback_wrap(&new_context);
 669   });
 670   uptr submit_sync = (uptr)&new_context;
 671   Release(thr, pc, submit_sync);
 672   new_channel =
 673       REAL(dispatch_io_create_with_path)(type, path, oflag, mode, q, new_h);
 674   Block_release(new_h);
 675   return new_channel;
 676 }
 677
 678 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io,
 679                  dispatch_io_type_t type, dispatch_io_t io, dispatch_queue_t q,
 680                  cleanup_handler_t h) {
 681   SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h);
 682   __block dispatch_io_t new_channel = nullptr;
 683   __block tsan_block_context_t new_context = {
 684       q, nullptr, &invoke_block, false, false, false, 0};
 685   cleanup_handler_t new_h = Block_copy(^(int error) {
 686     {
 687       SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
 688       Acquire(thr, pc, (uptr)new_channel);  // Release() in dispatch_io_close.
 689     }
 690     new_context.orig_context = ^(void) {
 691       h(error);
 692     };
 693     dispatch_callback_wrap(&new_context);
 694   });
 695   uptr submit_sync = (uptr)&new_context;
 696   Release(thr, pc, submit_sync);
 697   new_channel = REAL(dispatch_io_create_with_io)(type, io, q, new_h);
 698   Block_release(new_h);
 699   return new_channel;
 700 }
 701
 702 TSAN_INTERCEPTOR(void, dispatch_io_close, dispatch_io_t channel,
 703                  dispatch_io_close_flags_t flags) {
 704   SCOPED_TSAN_INTERCEPTOR(dispatch_io_close, channel, flags);
 705   Release(thr, pc, (uptr)channel);  // Acquire() in dispatch_io_create[_*].
 706   return REAL(dispatch_io_close)(channel, flags);
 707 }
 708
 709 // Resuming a suspended queue needs to synchronize with all subsequent
 710 // executions of blocks in that queue.
 711 TSAN_INTERCEPTOR(void, dispatch_resume, dispatch_object_t o) {
 712   SCOPED_TSAN_INTERCEPTOR(dispatch_resume, o);
 713   Release(thr, pc, (uptr)o);  // Synchronizes with the Acquire() on serial_sync
 714                               // in dispatch_sync_pre_execute
 715   return REAL(dispatch_resume)(o);
 716 }
 717
 718 }  // namespace __tsan
 719
 720 #endif  // SANITIZER_MAC