contrib/compiler-rt/lib/tsan/rtl/tsan_libdispatch_mac.cc

   1 //===-- tsan_libdispatch_mac.cc -------------------------------------------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // This file is a part of ThreadSanitizer (TSan), a race detector.
  11 //
  12 // Mac-specific libdispatch (GCD) support.
  13 //===----------------------------------------------------------------------===//
  14
  15 #include "sanitizer_common/sanitizer_platform.h"
  16 #if SANITIZER_MAC
  17
  18 #include "sanitizer_common/sanitizer_common.h"
  19 #include "interception/interception.h"
  20 #include "tsan_interceptors.h"
  21 #include "tsan_platform.h"
  22 #include "tsan_rtl.h"
  23
  24 #include <Block.h>
  25 #include <dispatch/dispatch.h>
  26 #include <pthread.h>
  27
  28 typedef long long_t;  // NOLINT
  29
  30 namespace __tsan {
  31
  32 typedef struct {
  33   dispatch_queue_t queue;
  34   void *orig_context;
  35   dispatch_function_t orig_work;
  36   bool free_context_in_callback;
  37   bool submitted_synchronously;
  38   bool is_barrier_block;
  39   uptr non_queue_sync_object;
  40 } tsan_block_context_t;
  41
  42 // The offsets of different fields of the dispatch_queue_t structure, exported
  43 // by libdispatch.dylib.
  44 extern "C" struct dispatch_queue_offsets_s {
  45   const uint16_t dqo_version;
  46   const uint16_t dqo_label;
  47   const uint16_t dqo_label_size;
  48   const uint16_t dqo_flags;
  49   const uint16_t dqo_flags_size;
  50   const uint16_t dqo_serialnum;
  51   const uint16_t dqo_serialnum_size;
  52   const uint16_t dqo_width;
  53   const uint16_t dqo_width_size;
  54   const uint16_t dqo_running;
  55   const uint16_t dqo_running_size;
  56   const uint16_t dqo_suspend_cnt;
  57   const uint16_t dqo_suspend_cnt_size;
  58   const uint16_t dqo_target_queue;
  59   const uint16_t dqo_target_queue_size;
  60   const uint16_t dqo_priority;
  61   const uint16_t dqo_priority_size;
  62 } dispatch_queue_offsets;
  63
  64 static bool IsQueueSerial(dispatch_queue_t q) {
  65   CHECK_EQ(dispatch_queue_offsets.dqo_width_size, 2);
  66   uptr width = *(uint16_t *)(((uptr)q) + dispatch_queue_offsets.dqo_width);
  67   CHECK_NE(width, 0);
  68   return width == 1;
  69 }
  70
  71 static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) {
  72   CHECK_EQ(dispatch_queue_offsets.dqo_target_queue_size, 8);
  73   dispatch_queue_t target_queue =
  74       *(dispatch_queue_t *)(((uptr)source) +
  75                             dispatch_queue_offsets.dqo_target_queue);
  76   CHECK_NE(target_queue, 0);
  77   return target_queue;
  78 }
  79
  80 static tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc,
  81                                           dispatch_queue_t queue,
  82                                           void *orig_context,
  83                                           dispatch_function_t orig_work) {
  84   tsan_block_context_t *new_context =
  85       (tsan_block_context_t *)user_alloc(thr, pc, sizeof(tsan_block_context_t));
  86   new_context->queue = queue;
  87   new_context->orig_context = orig_context;
  88   new_context->orig_work = orig_work;
  89   new_context->free_context_in_callback = true;
  90   new_context->submitted_synchronously = false;
  91   new_context->is_barrier_block = false;
  92   return new_context;
  93 }
  94
  95 static void dispatch_callback_wrap(void *param) {
  96   SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap);
  97   tsan_block_context_t *context = (tsan_block_context_t *)param;
  98   bool is_queue_serial = context->queue && IsQueueSerial(context->queue);
  99   uptr sync_ptr = (uptr)context->queue ?: context->non_queue_sync_object;
 100
 101   uptr serial_sync = (uptr)sync_ptr;
 102   uptr concurrent_sync = ((uptr)sync_ptr) + sizeof(uptr);
 103   uptr submit_sync = (uptr)context;
 104   bool serial_task = context->is_barrier_block || is_queue_serial;
 105
 106   Acquire(thr, pc, submit_sync);
 107   Acquire(thr, pc, serial_sync);
 108   if (serial_task) Acquire(thr, pc, concurrent_sync);
 109
 110   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 111   context->orig_work(context->orig_context);
 112   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 113
 114   Release(thr, pc, serial_task ? serial_sync : concurrent_sync);
 115   if (context->submitted_synchronously) Release(thr, pc, submit_sync);
 116
 117   if (context->free_context_in_callback) user_free(thr, pc, context);
 118 }
 119
 120 static void invoke_block(void *param) {
 121   dispatch_block_t block = (dispatch_block_t)param;
 122   block();
 123 }
 124
 125 static void invoke_and_release_block(void *param) {
 126   dispatch_block_t block = (dispatch_block_t)param;
 127   block();
 128   Block_release(block);
 129 }
 130
 131 #define DISPATCH_INTERCEPT_B(name, barrier)                                  \
 132   TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \
 133     SCOPED_TSAN_INTERCEPTOR(name, q, block);                                 \
 134     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
 135     dispatch_block_t heap_block = Block_copy(block);                         \
 136     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                             \
 137     tsan_block_context_t *new_context =                                      \
 138         AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);     \
 139     new_context->is_barrier_block = barrier;                                 \
 140     Release(thr, pc, (uptr)new_context);                                     \
 141     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
 142     REAL(name##_f)(q, new_context, dispatch_callback_wrap);                  \
 143     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                             \
 144   }
 145
 146 #define DISPATCH_INTERCEPT_SYNC_B(name, barrier)                             \
 147   TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \
 148     SCOPED_TSAN_INTERCEPTOR(name, q, block);                                 \
 149     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
 150     dispatch_block_t heap_block = Block_copy(block);                         \
 151     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                             \
 152     tsan_block_context_t new_context = {                                     \
 153         q, heap_block, &invoke_and_release_block, false, true, barrier, 0};  \
 154     Release(thr, pc, (uptr)&new_context);                                    \
 155     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
 156     REAL(name##_f)(q, &new_context, dispatch_callback_wrap);                 \
 157     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                             \
 158     Acquire(thr, pc, (uptr)&new_context);                                    \
 159   }
 160
 161 #define DISPATCH_INTERCEPT_F(name, barrier)                       \
 162   TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \
 163                    dispatch_function_t work) {                    \
 164     SCOPED_TSAN_INTERCEPTOR(name, q, context, work);              \
 165     tsan_block_context_t *new_context =                           \
 166         AllocContext(thr, pc, q, context, work);                  \
 167     new_context->is_barrier_block = barrier;                      \
 168     Release(thr, pc, (uptr)new_context);                          \
 169     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                \
 170     REAL(name)(q, new_context, dispatch_callback_wrap);           \
 171     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                  \
 172   }
 173
 174 #define DISPATCH_INTERCEPT_SYNC_F(name, barrier)                              \
 175   TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context,             \
 176                    dispatch_function_t work) {                                \
 177     SCOPED_TSAN_INTERCEPTOR(name, q, context, work);                          \
 178     tsan_block_context_t new_context = {                                      \
 179         q, context, work, false, true, barrier, 0};                           \
 180     Release(thr, pc, (uptr)&new_context);                                     \
 181     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                            \
 182     REAL(name)(q, &new_context, dispatch_callback_wrap);                      \
 183     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                              \
 184     Acquire(thr, pc, (uptr)&new_context);                                     \
 185   }
 186
 187 // We wrap dispatch_async, dispatch_sync and friends where we allocate a new
 188 // context, which is used to synchronize (we release the context before
 189 // submitting, and the callback acquires it before executing the original
 190 // callback).
 191 DISPATCH_INTERCEPT_B(dispatch_async, false)
 192 DISPATCH_INTERCEPT_B(dispatch_barrier_async, true)
 193 DISPATCH_INTERCEPT_F(dispatch_async_f, false)
 194 DISPATCH_INTERCEPT_F(dispatch_barrier_async_f, true)
 195 DISPATCH_INTERCEPT_SYNC_B(dispatch_sync, false)
 196 DISPATCH_INTERCEPT_SYNC_B(dispatch_barrier_sync, true)
 197 DISPATCH_INTERCEPT_SYNC_F(dispatch_sync_f, false)
 198 DISPATCH_INTERCEPT_SYNC_F(dispatch_barrier_sync_f, true)
 199
 200 TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when,
 201                  dispatch_queue_t queue, dispatch_block_t block) {
 202   SCOPED_TSAN_INTERCEPTOR(dispatch_after, when, queue, block);
 203   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 204   dispatch_block_t heap_block = Block_copy(block);
 205   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 206   tsan_block_context_t *new_context =
 207       AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block);
 208   Release(thr, pc, (uptr)new_context);
 209   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 210   REAL(dispatch_after_f)(when, queue, new_context, dispatch_callback_wrap);
 211   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 212 }
 213
 214 TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when,
 215                  dispatch_queue_t queue, void *context,
 216                  dispatch_function_t work) {
 217   SCOPED_TSAN_INTERCEPTOR(dispatch_after_f, when, queue, context, work);
 218   WRAP(dispatch_after)(when, queue, ^(void) {
 219     work(context);
 220   });
 221 }
 222
 223 // GCD's dispatch_once implementation has a fast path that contains a racy read
 224 // and it's inlined into user's code. Furthermore, this fast path doesn't
 225 // establish a proper happens-before relations between the initialization and
 226 // code following the call to dispatch_once. We could deal with this in
 227 // instrumented code, but there's not much we can do about it in system
 228 // libraries. Let's disable the fast path (by never storing the value ~0 to
 229 // predicate), so the interceptor is always called, and let's add proper release
 230 // and acquire semantics. Since TSan does not see its own atomic stores, the
 231 // race on predicate won't be reported - the only accesses to it that TSan sees
 232 // are the loads on the fast path. Loads don't race. Secondly, dispatch_once is
 233 // both a macro and a real function, we want to intercept the function, so we
 234 // need to undefine the macro.
 235 #undef dispatch_once
 236 TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate,
 237                  dispatch_block_t block) {
 238   SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block);
 239   atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate);
 240   u32 v = atomic_load(a, memory_order_acquire);
 241   if (v == 0 &&
 242       atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) {
 243     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 244     block();
 245     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 246     Release(thr, pc, (uptr)a);
 247     atomic_store(a, 2, memory_order_release);
 248   } else {
 249     while (v != 2) {
 250       internal_sched_yield();
 251       v = atomic_load(a, memory_order_acquire);
 252     }
 253     Acquire(thr, pc, (uptr)a);
 254   }
 255 }
 256
 257 #undef dispatch_once_f
 258 TSAN_INTERCEPTOR(void, dispatch_once_f, dispatch_once_t *predicate,
 259                  void *context, dispatch_function_t function) {
 260   SCOPED_INTERCEPTOR_RAW(dispatch_once_f, predicate, context, function);
 261   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 262   WRAP(dispatch_once)(predicate, ^(void) {
 263     function(context);
 264   });
 265   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 266 }
 267
 268 TSAN_INTERCEPTOR(long_t, dispatch_semaphore_signal,
 269                  dispatch_semaphore_t dsema) {
 270   SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal, dsema);
 271   Release(thr, pc, (uptr)dsema);
 272   return REAL(dispatch_semaphore_signal)(dsema);
 273 }
 274
 275 TSAN_INTERCEPTOR(long_t, dispatch_semaphore_wait, dispatch_semaphore_t dsema,
 276                  dispatch_time_t timeout) {
 277   SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait, dsema, timeout);
 278   long_t result = REAL(dispatch_semaphore_wait)(dsema, timeout);
 279   if (result == 0) Acquire(thr, pc, (uptr)dsema);
 280   return result;
 281 }
 282
 283 TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group,
 284                  dispatch_time_t timeout) {
 285   SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait, group, timeout);
 286   long_t result = REAL(dispatch_group_wait)(group, timeout);
 287   if (result == 0) Acquire(thr, pc, (uptr)group);
 288   return result;
 289 }
 290
 291 TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) {
 292   SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group);
 293   // Acquired in the group noticifaction callback in dispatch_group_notify[_f].
 294   Release(thr, pc, (uptr)group);
 295   REAL(dispatch_group_leave)(group);
 296 }
 297
 298 TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group,
 299                  dispatch_queue_t queue, dispatch_block_t block) {
 300   SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block);
 301   dispatch_retain(group);
 302   dispatch_group_enter(group);
 303   __block dispatch_block_t block_copy = (dispatch_block_t)_Block_copy(block);
 304   WRAP(dispatch_async)(queue, ^(void) {
 305     block_copy();
 306     _Block_release(block_copy);
 307     WRAP(dispatch_group_leave)(group);
 308     dispatch_release(group);
 309   });
 310 }
 311
 312 TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group,
 313                  dispatch_queue_t queue, void *context,
 314                  dispatch_function_t work) {
 315   SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f, group, queue, context, work);
 316   dispatch_retain(group);
 317   dispatch_group_enter(group);
 318   WRAP(dispatch_async)(queue, ^(void) {
 319     work(context);
 320     WRAP(dispatch_group_leave)(group);
 321     dispatch_release(group);
 322   });
 323 }
 324
 325 TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group,
 326                  dispatch_queue_t q, dispatch_block_t block) {
 327   SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block);
 328
 329   // To make sure the group is still available in the callback (otherwise
 330   // it can be already destroyed).  Will be released in the callback.
 331   dispatch_retain(group);
 332
 333   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 334   dispatch_block_t heap_block = Block_copy(^(void) {
 335     {
 336       SCOPED_INTERCEPTOR_RAW(dispatch_read_callback);
 337       // Released when leaving the group (dispatch_group_leave).
 338       Acquire(thr, pc, (uptr)group);
 339     }
 340     dispatch_release(group);
 341     block();
 342   });
 343   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 344   tsan_block_context_t *new_context =
 345       AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
 346   new_context->is_barrier_block = true;
 347   Release(thr, pc, (uptr)new_context);
 348   REAL(dispatch_group_notify_f)(group, q, new_context, dispatch_callback_wrap);
 349 }
 350
 351 TSAN_INTERCEPTOR(void, dispatch_group_notify_f, dispatch_group_t group,
 352                  dispatch_queue_t q, void *context, dispatch_function_t work) {
 353   WRAP(dispatch_group_notify)(group, q, ^(void) { work(context); });
 354 }
 355
 356 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler,
 357                  dispatch_source_t source, dispatch_block_t handler) {
 358   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler, source, handler);
 359   if (handler == nullptr)
 360     return REAL(dispatch_source_set_event_handler)(source, nullptr);
 361   dispatch_queue_t q = GetTargetQueueFromSource(source);
 362   __block tsan_block_context_t new_context = {
 363       q, handler, &invoke_block, false, false, false, 0 };
 364   dispatch_block_t new_handler = Block_copy(^(void) {
 365     new_context.orig_context = handler;  // To explicitly capture "handler".
 366     dispatch_callback_wrap(&new_context);
 367   });
 368   uptr submit_sync = (uptr)&new_context;
 369   Release(thr, pc, submit_sync);
 370   REAL(dispatch_source_set_event_handler)(source, new_handler);
 371   Block_release(new_handler);
 372 }
 373
 374 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler_f,
 375                  dispatch_source_t source, dispatch_function_t handler) {
 376   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler_f, source, handler);
 377   if (handler == nullptr)
 378     return REAL(dispatch_source_set_event_handler)(source, nullptr);
 379   dispatch_block_t block = ^(void) {
 380     handler(dispatch_get_context(source));
 381   };
 382   WRAP(dispatch_source_set_event_handler)(source, block);
 383 }
 384
 385 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler,
 386                  dispatch_source_t source, dispatch_block_t handler) {
 387   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler, source, handler);
 388   if (handler == nullptr)
 389     return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
 390   dispatch_queue_t q = GetTargetQueueFromSource(source);
 391   __block tsan_block_context_t new_context = {
 392       q, handler, &invoke_block, false, false, false, 0};
 393   dispatch_block_t new_handler = Block_copy(^(void) {
 394     new_context.orig_context = handler;  // To explicitly capture "handler".
 395     dispatch_callback_wrap(&new_context);
 396   });
 397   uptr submit_sync = (uptr)&new_context;
 398   Release(thr, pc, submit_sync);
 399   REAL(dispatch_source_set_cancel_handler)(source, new_handler);
 400   Block_release(new_handler);
 401 }
 402
 403 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler_f,
 404                  dispatch_source_t source, dispatch_function_t handler) {
 405   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler_f, source,
 406                           handler);
 407   if (handler == nullptr)
 408     return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
 409   dispatch_block_t block = ^(void) {
 410     handler(dispatch_get_context(source));
 411   };
 412   WRAP(dispatch_source_set_cancel_handler)(source, block);
 413 }
 414
 415 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler,
 416                  dispatch_source_t source, dispatch_block_t handler) {
 417   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler, source,
 418                           handler);
 419   if (handler == nullptr)
 420     return REAL(dispatch_source_set_registration_handler)(source, nullptr);
 421   dispatch_queue_t q = GetTargetQueueFromSource(source);
 422   __block tsan_block_context_t new_context = {
 423       q, handler, &invoke_block, false, false, false, 0};
 424   dispatch_block_t new_handler = Block_copy(^(void) {
 425     new_context.orig_context = handler;  // To explicitly capture "handler".
 426     dispatch_callback_wrap(&new_context);
 427   });
 428   uptr submit_sync = (uptr)&new_context;
 429   Release(thr, pc, submit_sync);
 430   REAL(dispatch_source_set_registration_handler)(source, new_handler);
 431   Block_release(new_handler);
 432 }
 433
 434 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f,
 435                  dispatch_source_t source, dispatch_function_t handler) {
 436   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler_f, source,
 437                           handler);
 438   if (handler == nullptr)
 439     return REAL(dispatch_source_set_registration_handler)(source, nullptr);
 440   dispatch_block_t block = ^(void) {
 441     handler(dispatch_get_context(source));
 442   };
 443   WRAP(dispatch_source_set_registration_handler)(source, block);
 444 }
 445
 446 TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations,
 447                  dispatch_queue_t queue, void (^block)(size_t)) {
 448   SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block);
 449
 450   void *parent_to_child_sync = nullptr;
 451   uptr parent_to_child_sync_uptr = (uptr)&parent_to_child_sync;
 452   void *child_to_parent_sync = nullptr;
 453   uptr child_to_parent_sync_uptr = (uptr)&child_to_parent_sync;
 454
 455   Release(thr, pc, parent_to_child_sync_uptr);
 456   void (^new_block)(size_t) = ^(size_t iteration) {
 457     SCOPED_INTERCEPTOR_RAW(dispatch_apply);
 458     Acquire(thr, pc, parent_to_child_sync_uptr);
 459     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 460     block(iteration);
 461     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 462     Release(thr, pc, child_to_parent_sync_uptr);
 463   };
 464   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 465   REAL(dispatch_apply)(iterations, queue, new_block);
 466   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 467   Acquire(thr, pc, child_to_parent_sync_uptr);
 468 }
 469
 470 TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations,
 471                  dispatch_queue_t queue, void *context,
 472                  void (*work)(void *, size_t)) {
 473   SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work);
 474   void (^new_block)(size_t) = ^(size_t iteration) {
 475     work(context, iteration);
 476   };
 477   WRAP(dispatch_apply)(iterations, queue, new_block);
 478 }
 479
 480 DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
 481 DECLARE_REAL_AND_INTERCEPTOR(int, munmap, void *addr, long_t sz)
 482
 483 TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer,
 484                  size_t size, dispatch_queue_t q, dispatch_block_t destructor) {
 485   SCOPED_TSAN_INTERCEPTOR(dispatch_data_create, buffer, size, q, destructor);
 486   if ((q == nullptr) || (destructor == DISPATCH_DATA_DESTRUCTOR_DEFAULT))
 487     return REAL(dispatch_data_create)(buffer, size, q, destructor);
 488
 489   if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE)
 490     destructor = ^(void) { WRAP(free)((void *)buffer); };
 491   else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP)
 492     destructor = ^(void) { WRAP(munmap)((void *)buffer, size); };
 493
 494   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 495   dispatch_block_t heap_block = Block_copy(destructor);
 496   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 497   tsan_block_context_t *new_context =
 498       AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
 499   uptr submit_sync = (uptr)new_context;
 500   Release(thr, pc, submit_sync);
 501   return REAL(dispatch_data_create)(buffer, size, q, ^(void) {
 502     dispatch_callback_wrap(new_context);
 503   });
 504 }
 505
 506 typedef void (^fd_handler_t)(dispatch_data_t data, int error);
 507 typedef void (^cleanup_handler_t)(int error);
 508
 509 TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length,
 510                  dispatch_queue_t q, fd_handler_t h) {
 511   SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h);
 512   __block tsan_block_context_t new_context = {
 513       q, nullptr, &invoke_block, false, false, false, 0};
 514   fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
 515     new_context.orig_context = ^(void) {
 516       h(data, error);
 517     };
 518     dispatch_callback_wrap(&new_context);
 519   });
 520   uptr submit_sync = (uptr)&new_context;
 521   Release(thr, pc, submit_sync);
 522   REAL(dispatch_read)(fd, length, q, new_h);
 523   Block_release(new_h);
 524 }
 525
 526 TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data,
 527                  dispatch_queue_t q, fd_handler_t h) {
 528   SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h);
 529   __block tsan_block_context_t new_context = {
 530       q, nullptr, &invoke_block, false, false, false, 0};
 531   fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
 532     new_context.orig_context = ^(void) {
 533       h(data, error);
 534     };
 535     dispatch_callback_wrap(&new_context);
 536   });
 537   uptr submit_sync = (uptr)&new_context;
 538   Release(thr, pc, submit_sync);
 539   REAL(dispatch_write)(fd, data, q, new_h);
 540   Block_release(new_h);
 541 }
 542
 543 TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset,
 544                  size_t length, dispatch_queue_t q, dispatch_io_handler_t h) {
 545   SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h);
 546   __block tsan_block_context_t new_context = {
 547       q, nullptr, &invoke_block, false, false, false, 0};
 548   dispatch_io_handler_t new_h =
 549       Block_copy(^(bool done, dispatch_data_t data, int error) {
 550         new_context.orig_context = ^(void) {
 551           h(done, data, error);
 552         };
 553         dispatch_callback_wrap(&new_context);
 554       });
 555   uptr submit_sync = (uptr)&new_context;
 556   Release(thr, pc, submit_sync);
 557   REAL(dispatch_io_read)(channel, offset, length, q, new_h);
 558   Block_release(new_h);
 559 }
 560
 561 TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset,
 562                  dispatch_data_t data, dispatch_queue_t q,
 563                  dispatch_io_handler_t h) {
 564   SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h);
 565   __block tsan_block_context_t new_context = {
 566       q, nullptr, &invoke_block, false, false, false, 0};
 567   dispatch_io_handler_t new_h =
 568       Block_copy(^(bool done, dispatch_data_t data, int error) {
 569         new_context.orig_context = ^(void) {
 570           h(done, data, error);
 571         };
 572         dispatch_callback_wrap(&new_context);
 573       });
 574   uptr submit_sync = (uptr)&new_context;
 575   Release(thr, pc, submit_sync);
 576   REAL(dispatch_io_write)(channel, offset, data, q, new_h);
 577   Block_release(new_h);
 578 }
 579
 580 TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel,
 581                  dispatch_block_t barrier) {
 582   SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier);
 583   __block tsan_block_context_t new_context = {
 584       nullptr, nullptr, &invoke_block, false, false, false, 0};
 585   new_context.non_queue_sync_object = (uptr)channel;
 586   new_context.is_barrier_block = true;
 587   dispatch_block_t new_block = Block_copy(^(void) {
 588     new_context.orig_context = ^(void) {
 589       barrier();
 590     };
 591     dispatch_callback_wrap(&new_context);
 592   });
 593   uptr submit_sync = (uptr)&new_context;
 594   Release(thr, pc, submit_sync);
 595   REAL(dispatch_io_barrier)(channel, new_block);
 596   Block_release(new_block);
 597 }
 598
 599 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type,
 600                  dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) {
 601   SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h);
 602   __block dispatch_io_t new_channel = nullptr;
 603   __block tsan_block_context_t new_context = {
 604       q, nullptr, &invoke_block, false, false, false, 0};
 605   cleanup_handler_t new_h = Block_copy(^(int error) {
 606     {
 607       SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
 608       Acquire(thr, pc, (uptr)new_channel);  // Release() in dispatch_io_close.
 609     }
 610     new_context.orig_context = ^(void) {
 611       h(error);
 612     };
 613     dispatch_callback_wrap(&new_context);
 614   });
 615   uptr submit_sync = (uptr)&new_context;
 616   Release(thr, pc, submit_sync);
 617   new_channel = REAL(dispatch_io_create)(type, fd, q, new_h);
 618   Block_release(new_h);
 619   return new_channel;
 620 }
 621
 622 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path,
 623                  dispatch_io_type_t type, const char *path, int oflag,
 624                  mode_t mode, dispatch_queue_t q, cleanup_handler_t h) {
 625   SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode,
 626                           q, h);
 627   __block dispatch_io_t new_channel = nullptr;
 628   __block tsan_block_context_t new_context = {
 629       q, nullptr, &invoke_block, false, false, false, 0};
 630   cleanup_handler_t new_h = Block_copy(^(int error) {
 631     {
 632       SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
 633       Acquire(thr, pc, (uptr)new_channel);  // Release() in dispatch_io_close.
 634     }
 635     new_context.orig_context = ^(void) {
 636       h(error);
 637     };
 638     dispatch_callback_wrap(&new_context);
 639   });
 640   uptr submit_sync = (uptr)&new_context;
 641   Release(thr, pc, submit_sync);
 642   new_channel =
 643       REAL(dispatch_io_create_with_path)(type, path, oflag, mode, q, new_h);
 644   Block_release(new_h);
 645   return new_channel;
 646 }
 647
 648 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io,
 649                  dispatch_io_type_t type, dispatch_io_t io, dispatch_queue_t q,
 650                  cleanup_handler_t h) {
 651   SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h);
 652   __block dispatch_io_t new_channel = nullptr;
 653   __block tsan_block_context_t new_context = {
 654       q, nullptr, &invoke_block, false, false, false, 0};
 655   cleanup_handler_t new_h = Block_copy(^(int error) {
 656     {
 657       SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
 658       Acquire(thr, pc, (uptr)new_channel);  // Release() in dispatch_io_close.
 659     }
 660     new_context.orig_context = ^(void) {
 661       h(error);
 662     };
 663     dispatch_callback_wrap(&new_context);
 664   });
 665   uptr submit_sync = (uptr)&new_context;
 666   Release(thr, pc, submit_sync);
 667   new_channel = REAL(dispatch_io_create_with_io)(type, io, q, new_h);
 668   Block_release(new_h);
 669   return new_channel;
 670 }
 671
 672 TSAN_INTERCEPTOR(void, dispatch_io_close, dispatch_io_t channel,
 673                  dispatch_io_close_flags_t flags) {
 674   SCOPED_TSAN_INTERCEPTOR(dispatch_io_close, channel, flags);
 675   Release(thr, pc, (uptr)channel);  // Acquire() in dispatch_io_create[_*].
 676   return REAL(dispatch_io_close)(channel, flags);
 677 }
 678
 679 }  // namespace __tsan
 680
 681 #endif  // SANITIZER_MAC