1 //===-- tsan_libdispatch_mac.cc -------------------------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file is a part of ThreadSanitizer (TSan), a race detector.
12 // Mac-specific libdispatch (GCD) support.
13 //===----------------------------------------------------------------------===//
15 #include "sanitizer_common/sanitizer_platform.h"
18 #include "sanitizer_common/sanitizer_common.h"
19 #include "interception/interception.h"
20 #include "tsan_interceptors.h"
21 #include "tsan_platform.h"
25 #include <dispatch/dispatch.h>
28 // DISPATCH_NOESCAPE is not defined prior to XCode 8.
29 #ifndef DISPATCH_NOESCAPE
30 #define DISPATCH_NOESCAPE
33 typedef long long_t; // NOLINT
38 dispatch_queue_t queue;
40 dispatch_function_t orig_work;
41 bool free_context_in_callback;
42 bool submitted_synchronously;
43 bool is_barrier_block;
44 uptr non_queue_sync_object;
45 } tsan_block_context_t;
47 // The offsets of different fields of the dispatch_queue_t structure, exported
48 // by libdispatch.dylib.
49 extern "C" struct dispatch_queue_offsets_s {
50 const uint16_t dqo_version;
51 const uint16_t dqo_label;
52 const uint16_t dqo_label_size;
53 const uint16_t dqo_flags;
54 const uint16_t dqo_flags_size;
55 const uint16_t dqo_serialnum;
56 const uint16_t dqo_serialnum_size;
57 const uint16_t dqo_width;
58 const uint16_t dqo_width_size;
59 const uint16_t dqo_running;
60 const uint16_t dqo_running_size;
61 const uint16_t dqo_suspend_cnt;
62 const uint16_t dqo_suspend_cnt_size;
63 const uint16_t dqo_target_queue;
64 const uint16_t dqo_target_queue_size;
65 const uint16_t dqo_priority;
66 const uint16_t dqo_priority_size;
67 } dispatch_queue_offsets;
69 static bool IsQueueSerial(dispatch_queue_t q) {
70 CHECK_EQ(dispatch_queue_offsets.dqo_width_size, 2);
71 uptr width = *(uint16_t *)(((uptr)q) + dispatch_queue_offsets.dqo_width);
76 static dispatch_queue_t GetTargetQueueFromQueue(dispatch_queue_t q) {
77 CHECK_EQ(dispatch_queue_offsets.dqo_target_queue_size, 8);
78 dispatch_queue_t tq = *(
79 dispatch_queue_t *)(((uptr)q) + dispatch_queue_offsets.dqo_target_queue);
83 static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) {
84 dispatch_queue_t tq = GetTargetQueueFromQueue((dispatch_queue_t)source);
89 static tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc,
90 dispatch_queue_t queue,
92 dispatch_function_t orig_work) {
93 tsan_block_context_t *new_context =
94 (tsan_block_context_t *)user_alloc_internal(thr, pc,
95 sizeof(tsan_block_context_t));
96 new_context->queue = queue;
97 new_context->orig_context = orig_context;
98 new_context->orig_work = orig_work;
99 new_context->free_context_in_callback = true;
100 new_context->submitted_synchronously = false;
101 new_context->is_barrier_block = false;
102 new_context->non_queue_sync_object = 0;
106 #define GET_QUEUE_SYNC_VARS(context, q) \
107 bool is_queue_serial = q && IsQueueSerial(q); \
108 uptr sync_ptr = (uptr)q ?: context->non_queue_sync_object; \
109 uptr serial_sync = (uptr)sync_ptr; \
110 uptr concurrent_sync = sync_ptr ? ((uptr)sync_ptr) + sizeof(uptr) : 0; \
111 bool serial_task = context->is_barrier_block || is_queue_serial
113 static void dispatch_sync_pre_execute(ThreadState *thr, uptr pc,
114 tsan_block_context_t *context) {
115 uptr submit_sync = (uptr)context;
116 Acquire(thr, pc, submit_sync);
118 dispatch_queue_t q = context->queue;
120 GET_QUEUE_SYNC_VARS(context, q);
121 if (serial_sync) Acquire(thr, pc, serial_sync);
122 if (serial_task && concurrent_sync) Acquire(thr, pc, concurrent_sync);
124 if (q) q = GetTargetQueueFromQueue(q);
128 static void dispatch_sync_post_execute(ThreadState *thr, uptr pc,
129 tsan_block_context_t *context) {
130 uptr submit_sync = (uptr)context;
131 if (context->submitted_synchronously) Release(thr, pc, submit_sync);
133 dispatch_queue_t q = context->queue;
135 GET_QUEUE_SYNC_VARS(context, q);
136 if (serial_task && serial_sync) Release(thr, pc, serial_sync);
137 if (!serial_task && concurrent_sync) Release(thr, pc, concurrent_sync);
139 if (q) q = GetTargetQueueFromQueue(q);
143 static void dispatch_callback_wrap(void *param) {
144 SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap);
145 tsan_block_context_t *context = (tsan_block_context_t *)param;
147 dispatch_sync_pre_execute(thr, pc, context);
149 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
150 context->orig_work(context->orig_context);
151 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
153 dispatch_sync_post_execute(thr, pc, context);
155 if (context->free_context_in_callback) user_free(thr, pc, context);
158 static void invoke_block(void *param) {
159 dispatch_block_t block = (dispatch_block_t)param;
163 static void invoke_and_release_block(void *param) {
164 dispatch_block_t block = (dispatch_block_t)param;
166 Block_release(block);
169 #define DISPATCH_INTERCEPT_B(name, barrier) \
170 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \
171 SCOPED_TSAN_INTERCEPTOR(name, q, block); \
172 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
173 dispatch_block_t heap_block = Block_copy(block); \
174 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
175 tsan_block_context_t *new_context = \
176 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); \
177 new_context->is_barrier_block = barrier; \
178 Release(thr, pc, (uptr)new_context); \
179 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
180 REAL(name##_f)(q, new_context, dispatch_callback_wrap); \
181 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
184 #define DISPATCH_INTERCEPT_SYNC_B(name, barrier) \
185 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, \
186 DISPATCH_NOESCAPE dispatch_block_t block) { \
187 SCOPED_TSAN_INTERCEPTOR(name, q, block); \
188 tsan_block_context_t new_context = { \
189 q, block, &invoke_block, false, true, barrier, 0}; \
190 Release(thr, pc, (uptr)&new_context); \
191 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
192 REAL(name##_f)(q, &new_context, dispatch_callback_wrap); \
193 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
194 Acquire(thr, pc, (uptr)&new_context); \
197 #define DISPATCH_INTERCEPT_F(name, barrier) \
198 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \
199 dispatch_function_t work) { \
200 SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \
201 tsan_block_context_t *new_context = \
202 AllocContext(thr, pc, q, context, work); \
203 new_context->is_barrier_block = barrier; \
204 Release(thr, pc, (uptr)new_context); \
205 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
206 REAL(name)(q, new_context, dispatch_callback_wrap); \
207 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
210 #define DISPATCH_INTERCEPT_SYNC_F(name, barrier) \
211 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \
212 dispatch_function_t work) { \
213 SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \
214 tsan_block_context_t new_context = { \
215 q, context, work, false, true, barrier, 0}; \
216 Release(thr, pc, (uptr)&new_context); \
217 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
218 REAL(name)(q, &new_context, dispatch_callback_wrap); \
219 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
220 Acquire(thr, pc, (uptr)&new_context); \
223 // We wrap dispatch_async, dispatch_sync and friends where we allocate a new
224 // context, which is used to synchronize (we release the context before
225 // submitting, and the callback acquires it before executing the original
227 DISPATCH_INTERCEPT_B(dispatch_async, false)
228 DISPATCH_INTERCEPT_B(dispatch_barrier_async, true)
229 DISPATCH_INTERCEPT_F(dispatch_async_f, false)
230 DISPATCH_INTERCEPT_F(dispatch_barrier_async_f, true)
231 DISPATCH_INTERCEPT_SYNC_B(dispatch_sync, false)
232 DISPATCH_INTERCEPT_SYNC_B(dispatch_barrier_sync, true)
233 DISPATCH_INTERCEPT_SYNC_F(dispatch_sync_f, false)
234 DISPATCH_INTERCEPT_SYNC_F(dispatch_barrier_sync_f, true)
236 TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when,
237 dispatch_queue_t queue, dispatch_block_t block) {
238 SCOPED_TSAN_INTERCEPTOR(dispatch_after, when, queue, block);
239 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
240 dispatch_block_t heap_block = Block_copy(block);
241 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
242 tsan_block_context_t *new_context =
243 AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block);
244 Release(thr, pc, (uptr)new_context);
245 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
246 REAL(dispatch_after_f)(when, queue, new_context, dispatch_callback_wrap);
247 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
250 TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when,
251 dispatch_queue_t queue, void *context,
252 dispatch_function_t work) {
253 SCOPED_TSAN_INTERCEPTOR(dispatch_after_f, when, queue, context, work);
254 WRAP(dispatch_after)(when, queue, ^(void) {
259 // GCD's dispatch_once implementation has a fast path that contains a racy read
260 // and it's inlined into user's code. Furthermore, this fast path doesn't
261 // establish a proper happens-before relations between the initialization and
262 // code following the call to dispatch_once. We could deal with this in
263 // instrumented code, but there's not much we can do about it in system
264 // libraries. Let's disable the fast path (by never storing the value ~0 to
265 // predicate), so the interceptor is always called, and let's add proper release
266 // and acquire semantics. Since TSan does not see its own atomic stores, the
267 // race on predicate won't be reported - the only accesses to it that TSan sees
268 // are the loads on the fast path. Loads don't race. Secondly, dispatch_once is
269 // both a macro and a real function, we want to intercept the function, so we
270 // need to undefine the macro.
272 TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate,
273 DISPATCH_NOESCAPE dispatch_block_t block) {
274 SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block);
275 atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate);
276 u32 v = atomic_load(a, memory_order_acquire);
278 atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) {
279 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
281 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
282 Release(thr, pc, (uptr)a);
283 atomic_store(a, 2, memory_order_release);
286 internal_sched_yield();
287 v = atomic_load(a, memory_order_acquire);
289 Acquire(thr, pc, (uptr)a);
293 #undef dispatch_once_f
294 TSAN_INTERCEPTOR(void, dispatch_once_f, dispatch_once_t *predicate,
295 void *context, dispatch_function_t function) {
296 SCOPED_INTERCEPTOR_RAW(dispatch_once_f, predicate, context, function);
297 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
298 WRAP(dispatch_once)(predicate, ^(void) {
301 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
304 TSAN_INTERCEPTOR(long_t, dispatch_semaphore_signal,
305 dispatch_semaphore_t dsema) {
306 SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal, dsema);
307 Release(thr, pc, (uptr)dsema);
308 return REAL(dispatch_semaphore_signal)(dsema);
311 TSAN_INTERCEPTOR(long_t, dispatch_semaphore_wait, dispatch_semaphore_t dsema,
312 dispatch_time_t timeout) {
313 SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait, dsema, timeout);
314 long_t result = REAL(dispatch_semaphore_wait)(dsema, timeout);
315 if (result == 0) Acquire(thr, pc, (uptr)dsema);
319 TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group,
320 dispatch_time_t timeout) {
321 SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait, group, timeout);
322 long_t result = REAL(dispatch_group_wait)(group, timeout);
323 if (result == 0) Acquire(thr, pc, (uptr)group);
327 TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) {
328 SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group);
329 // Acquired in the group noticifaction callback in dispatch_group_notify[_f].
330 Release(thr, pc, (uptr)group);
331 REAL(dispatch_group_leave)(group);
334 TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group,
335 dispatch_queue_t queue, dispatch_block_t block) {
336 SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block);
337 dispatch_retain(group);
338 dispatch_group_enter(group);
339 __block dispatch_block_t block_copy = (dispatch_block_t)_Block_copy(block);
340 WRAP(dispatch_async)(queue, ^(void) {
342 _Block_release(block_copy);
343 WRAP(dispatch_group_leave)(group);
344 dispatch_release(group);
348 TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group,
349 dispatch_queue_t queue, void *context,
350 dispatch_function_t work) {
351 SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f, group, queue, context, work);
352 dispatch_retain(group);
353 dispatch_group_enter(group);
354 WRAP(dispatch_async)(queue, ^(void) {
356 WRAP(dispatch_group_leave)(group);
357 dispatch_release(group);
361 TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group,
362 dispatch_queue_t q, dispatch_block_t block) {
363 SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block);
365 // To make sure the group is still available in the callback (otherwise
366 // it can be already destroyed). Will be released in the callback.
367 dispatch_retain(group);
369 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
370 dispatch_block_t heap_block = Block_copy(^(void) {
372 SCOPED_INTERCEPTOR_RAW(dispatch_read_callback);
373 // Released when leaving the group (dispatch_group_leave).
374 Acquire(thr, pc, (uptr)group);
376 dispatch_release(group);
379 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
380 tsan_block_context_t *new_context =
381 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
382 new_context->is_barrier_block = true;
383 Release(thr, pc, (uptr)new_context);
384 REAL(dispatch_group_notify_f)(group, q, new_context, dispatch_callback_wrap);
387 TSAN_INTERCEPTOR(void, dispatch_group_notify_f, dispatch_group_t group,
388 dispatch_queue_t q, void *context, dispatch_function_t work) {
389 WRAP(dispatch_group_notify)(group, q, ^(void) { work(context); });
392 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler,
393 dispatch_source_t source, dispatch_block_t handler) {
394 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler, source, handler);
395 if (handler == nullptr)
396 return REAL(dispatch_source_set_event_handler)(source, nullptr);
397 dispatch_queue_t q = GetTargetQueueFromSource(source);
398 __block tsan_block_context_t new_context = {
399 q, handler, &invoke_block, false, false, false, 0 };
400 dispatch_block_t new_handler = Block_copy(^(void) {
401 new_context.orig_context = handler; // To explicitly capture "handler".
402 dispatch_callback_wrap(&new_context);
404 uptr submit_sync = (uptr)&new_context;
405 Release(thr, pc, submit_sync);
406 REAL(dispatch_source_set_event_handler)(source, new_handler);
407 Block_release(new_handler);
410 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler_f,
411 dispatch_source_t source, dispatch_function_t handler) {
412 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler_f, source, handler);
413 if (handler == nullptr)
414 return REAL(dispatch_source_set_event_handler)(source, nullptr);
415 dispatch_block_t block = ^(void) {
416 handler(dispatch_get_context(source));
418 WRAP(dispatch_source_set_event_handler)(source, block);
421 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler,
422 dispatch_source_t source, dispatch_block_t handler) {
423 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler, source, handler);
424 if (handler == nullptr)
425 return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
426 dispatch_queue_t q = GetTargetQueueFromSource(source);
427 __block tsan_block_context_t new_context = {
428 q, handler, &invoke_block, false, false, false, 0};
429 dispatch_block_t new_handler = Block_copy(^(void) {
430 new_context.orig_context = handler; // To explicitly capture "handler".
431 dispatch_callback_wrap(&new_context);
433 uptr submit_sync = (uptr)&new_context;
434 Release(thr, pc, submit_sync);
435 REAL(dispatch_source_set_cancel_handler)(source, new_handler);
436 Block_release(new_handler);
439 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler_f,
440 dispatch_source_t source, dispatch_function_t handler) {
441 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler_f, source,
443 if (handler == nullptr)
444 return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
445 dispatch_block_t block = ^(void) {
446 handler(dispatch_get_context(source));
448 WRAP(dispatch_source_set_cancel_handler)(source, block);
451 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler,
452 dispatch_source_t source, dispatch_block_t handler) {
453 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler, source,
455 if (handler == nullptr)
456 return REAL(dispatch_source_set_registration_handler)(source, nullptr);
457 dispatch_queue_t q = GetTargetQueueFromSource(source);
458 __block tsan_block_context_t new_context = {
459 q, handler, &invoke_block, false, false, false, 0};
460 dispatch_block_t new_handler = Block_copy(^(void) {
461 new_context.orig_context = handler; // To explicitly capture "handler".
462 dispatch_callback_wrap(&new_context);
464 uptr submit_sync = (uptr)&new_context;
465 Release(thr, pc, submit_sync);
466 REAL(dispatch_source_set_registration_handler)(source, new_handler);
467 Block_release(new_handler);
470 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f,
471 dispatch_source_t source, dispatch_function_t handler) {
472 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler_f, source,
474 if (handler == nullptr)
475 return REAL(dispatch_source_set_registration_handler)(source, nullptr);
476 dispatch_block_t block = ^(void) {
477 handler(dispatch_get_context(source));
479 WRAP(dispatch_source_set_registration_handler)(source, block);
482 TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations,
483 dispatch_queue_t queue,
484 DISPATCH_NOESCAPE void (^block)(size_t)) {
485 SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block);
487 void *parent_to_child_sync = nullptr;
488 uptr parent_to_child_sync_uptr = (uptr)&parent_to_child_sync;
489 void *child_to_parent_sync = nullptr;
490 uptr child_to_parent_sync_uptr = (uptr)&child_to_parent_sync;
492 Release(thr, pc, parent_to_child_sync_uptr);
493 void (^new_block)(size_t) = ^(size_t iteration) {
494 SCOPED_INTERCEPTOR_RAW(dispatch_apply);
495 Acquire(thr, pc, parent_to_child_sync_uptr);
496 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
498 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
499 Release(thr, pc, child_to_parent_sync_uptr);
501 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
502 REAL(dispatch_apply)(iterations, queue, new_block);
503 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
504 Acquire(thr, pc, child_to_parent_sync_uptr);
507 TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations,
508 dispatch_queue_t queue, void *context,
509 void (*work)(void *, size_t)) {
510 SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work);
511 void (^new_block)(size_t) = ^(size_t iteration) {
512 work(context, iteration);
514 WRAP(dispatch_apply)(iterations, queue, new_block);
517 DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
518 DECLARE_REAL_AND_INTERCEPTOR(int, munmap, void *addr, long_t sz)
520 TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer,
521 size_t size, dispatch_queue_t q, dispatch_block_t destructor) {
522 SCOPED_TSAN_INTERCEPTOR(dispatch_data_create, buffer, size, q, destructor);
523 if ((q == nullptr) || (destructor == DISPATCH_DATA_DESTRUCTOR_DEFAULT))
524 return REAL(dispatch_data_create)(buffer, size, q, destructor);
526 if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE)
527 destructor = ^(void) { WRAP(free)((void *)(uintptr_t)buffer); };
528 else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP)
529 destructor = ^(void) { WRAP(munmap)((void *)(uintptr_t)buffer, size); };
531 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
532 dispatch_block_t heap_block = Block_copy(destructor);
533 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
534 tsan_block_context_t *new_context =
535 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
536 uptr submit_sync = (uptr)new_context;
537 Release(thr, pc, submit_sync);
538 return REAL(dispatch_data_create)(buffer, size, q, ^(void) {
539 dispatch_callback_wrap(new_context);
543 typedef void (^fd_handler_t)(dispatch_data_t data, int error);
544 typedef void (^cleanup_handler_t)(int error);
546 TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length,
547 dispatch_queue_t q, fd_handler_t h) {
548 SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h);
549 __block tsan_block_context_t new_context = {
550 q, nullptr, &invoke_block, false, false, false, 0};
551 fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
552 new_context.orig_context = ^(void) {
555 dispatch_callback_wrap(&new_context);
557 uptr submit_sync = (uptr)&new_context;
558 Release(thr, pc, submit_sync);
559 REAL(dispatch_read)(fd, length, q, new_h);
560 Block_release(new_h);
563 TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data,
564 dispatch_queue_t q, fd_handler_t h) {
565 SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h);
566 __block tsan_block_context_t new_context = {
567 q, nullptr, &invoke_block, false, false, false, 0};
568 fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
569 new_context.orig_context = ^(void) {
572 dispatch_callback_wrap(&new_context);
574 uptr submit_sync = (uptr)&new_context;
575 Release(thr, pc, submit_sync);
576 REAL(dispatch_write)(fd, data, q, new_h);
577 Block_release(new_h);
580 TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset,
581 size_t length, dispatch_queue_t q, dispatch_io_handler_t h) {
582 SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h);
583 __block tsan_block_context_t new_context = {
584 q, nullptr, &invoke_block, false, false, false, 0};
585 dispatch_io_handler_t new_h =
586 Block_copy(^(bool done, dispatch_data_t data, int error) {
587 new_context.orig_context = ^(void) {
588 h(done, data, error);
590 dispatch_callback_wrap(&new_context);
592 uptr submit_sync = (uptr)&new_context;
593 Release(thr, pc, submit_sync);
594 REAL(dispatch_io_read)(channel, offset, length, q, new_h);
595 Block_release(new_h);
598 TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset,
599 dispatch_data_t data, dispatch_queue_t q,
600 dispatch_io_handler_t h) {
601 SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h);
602 __block tsan_block_context_t new_context = {
603 q, nullptr, &invoke_block, false, false, false, 0};
604 dispatch_io_handler_t new_h =
605 Block_copy(^(bool done, dispatch_data_t data, int error) {
606 new_context.orig_context = ^(void) {
607 h(done, data, error);
609 dispatch_callback_wrap(&new_context);
611 uptr submit_sync = (uptr)&new_context;
612 Release(thr, pc, submit_sync);
613 REAL(dispatch_io_write)(channel, offset, data, q, new_h);
614 Block_release(new_h);
617 TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel,
618 dispatch_block_t barrier) {
619 SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier);
620 __block tsan_block_context_t new_context = {
621 nullptr, nullptr, &invoke_block, false, false, false, 0};
622 new_context.non_queue_sync_object = (uptr)channel;
623 new_context.is_barrier_block = true;
624 dispatch_block_t new_block = Block_copy(^(void) {
625 new_context.orig_context = ^(void) {
628 dispatch_callback_wrap(&new_context);
630 uptr submit_sync = (uptr)&new_context;
631 Release(thr, pc, submit_sync);
632 REAL(dispatch_io_barrier)(channel, new_block);
633 Block_release(new_block);
636 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type,
637 dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) {
638 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h);
639 __block dispatch_io_t new_channel = nullptr;
640 __block tsan_block_context_t new_context = {
641 q, nullptr, &invoke_block, false, false, false, 0};
642 cleanup_handler_t new_h = Block_copy(^(int error) {
644 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
645 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close.
647 new_context.orig_context = ^(void) {
650 dispatch_callback_wrap(&new_context);
652 uptr submit_sync = (uptr)&new_context;
653 Release(thr, pc, submit_sync);
654 new_channel = REAL(dispatch_io_create)(type, fd, q, new_h);
655 Block_release(new_h);
659 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path,
660 dispatch_io_type_t type, const char *path, int oflag,
661 mode_t mode, dispatch_queue_t q, cleanup_handler_t h) {
662 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode,
664 __block dispatch_io_t new_channel = nullptr;
665 __block tsan_block_context_t new_context = {
666 q, nullptr, &invoke_block, false, false, false, 0};
667 cleanup_handler_t new_h = Block_copy(^(int error) {
669 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
670 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close.
672 new_context.orig_context = ^(void) {
675 dispatch_callback_wrap(&new_context);
677 uptr submit_sync = (uptr)&new_context;
678 Release(thr, pc, submit_sync);
680 REAL(dispatch_io_create_with_path)(type, path, oflag, mode, q, new_h);
681 Block_release(new_h);
685 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io,
686 dispatch_io_type_t type, dispatch_io_t io, dispatch_queue_t q,
687 cleanup_handler_t h) {
688 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h);
689 __block dispatch_io_t new_channel = nullptr;
690 __block tsan_block_context_t new_context = {
691 q, nullptr, &invoke_block, false, false, false, 0};
692 cleanup_handler_t new_h = Block_copy(^(int error) {
694 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
695 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close.
697 new_context.orig_context = ^(void) {
700 dispatch_callback_wrap(&new_context);
702 uptr submit_sync = (uptr)&new_context;
703 Release(thr, pc, submit_sync);
704 new_channel = REAL(dispatch_io_create_with_io)(type, io, q, new_h);
705 Block_release(new_h);
709 TSAN_INTERCEPTOR(void, dispatch_io_close, dispatch_io_t channel,
710 dispatch_io_close_flags_t flags) {
711 SCOPED_TSAN_INTERCEPTOR(dispatch_io_close, channel, flags);
712 Release(thr, pc, (uptr)channel); // Acquire() in dispatch_io_create[_*].
713 return REAL(dispatch_io_close)(channel, flags);
716 // Resuming a suspended queue needs to synchronize with all subsequent
717 // executions of blocks in that queue.
718 TSAN_INTERCEPTOR(void, dispatch_resume, dispatch_object_t o) {
719 SCOPED_TSAN_INTERCEPTOR(dispatch_resume, o);
720 Release(thr, pc, (uptr)o); // Synchronizes with the Acquire() on serial_sync
721 // in dispatch_sync_pre_execute
722 return REAL(dispatch_resume)(o);
725 } // namespace __tsan
727 #endif // SANITIZER_MAC