1 //===-- tsan_libdispatch_mac.cc -------------------------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file is a part of ThreadSanitizer (TSan), a race detector.
12 // Mac-specific libdispatch (GCD) support.
13 //===----------------------------------------------------------------------===//
15 #include "sanitizer_common/sanitizer_platform.h"
18 #include "sanitizer_common/sanitizer_common.h"
19 #include "interception/interception.h"
20 #include "tsan_interceptors.h"
21 #include "tsan_platform.h"
25 #include <dispatch/dispatch.h>
28 // DISPATCH_NOESCAPE is not defined prior to XCode 8.
29 #ifndef DISPATCH_NOESCAPE
30 #define DISPATCH_NOESCAPE
33 typedef long long_t; // NOLINT
38 dispatch_queue_t queue;
40 dispatch_function_t orig_work;
41 bool free_context_in_callback;
42 bool submitted_synchronously;
43 bool is_barrier_block;
44 uptr non_queue_sync_object;
45 } tsan_block_context_t;
47 // The offsets of different fields of the dispatch_queue_t structure, exported
48 // by libdispatch.dylib.
49 extern "C" struct dispatch_queue_offsets_s {
50 const uint16_t dqo_version;
51 const uint16_t dqo_label;
52 const uint16_t dqo_label_size;
53 const uint16_t dqo_flags;
54 const uint16_t dqo_flags_size;
55 const uint16_t dqo_serialnum;
56 const uint16_t dqo_serialnum_size;
57 const uint16_t dqo_width;
58 const uint16_t dqo_width_size;
59 const uint16_t dqo_running;
60 const uint16_t dqo_running_size;
61 const uint16_t dqo_suspend_cnt;
62 const uint16_t dqo_suspend_cnt_size;
63 const uint16_t dqo_target_queue;
64 const uint16_t dqo_target_queue_size;
65 const uint16_t dqo_priority;
66 const uint16_t dqo_priority_size;
67 } dispatch_queue_offsets;
69 static bool IsQueueSerial(dispatch_queue_t q) {
70 CHECK_EQ(dispatch_queue_offsets.dqo_width_size, 2);
71 uptr width = *(uint16_t *)(((uptr)q) + dispatch_queue_offsets.dqo_width);
76 static dispatch_queue_t GetTargetQueueFromQueue(dispatch_queue_t q) {
77 CHECK_EQ(dispatch_queue_offsets.dqo_target_queue_size, 8);
78 dispatch_queue_t tq = *(
79 dispatch_queue_t *)(((uptr)q) + dispatch_queue_offsets.dqo_target_queue);
83 static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) {
84 dispatch_queue_t tq = GetTargetQueueFromQueue((dispatch_queue_t)source);
89 static tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc,
90 dispatch_queue_t queue,
92 dispatch_function_t orig_work) {
93 tsan_block_context_t *new_context =
94 (tsan_block_context_t *)user_alloc_internal(thr, pc,
95 sizeof(tsan_block_context_t));
96 new_context->queue = queue;
97 new_context->orig_context = orig_context;
98 new_context->orig_work = orig_work;
99 new_context->free_context_in_callback = true;
100 new_context->submitted_synchronously = false;
101 new_context->is_barrier_block = false;
102 new_context->non_queue_sync_object = 0;
106 #define GET_QUEUE_SYNC_VARS(context, q) \
107 bool is_queue_serial = q && IsQueueSerial(q); \
108 uptr sync_ptr = (uptr)q ?: context->non_queue_sync_object; \
109 uptr serial_sync = (uptr)sync_ptr; \
110 uptr concurrent_sync = sync_ptr ? ((uptr)sync_ptr) + sizeof(uptr) : 0; \
111 bool serial_task = context->is_barrier_block || is_queue_serial
113 static void dispatch_sync_pre_execute(ThreadState *thr, uptr pc,
114 tsan_block_context_t *context) {
115 uptr submit_sync = (uptr)context;
116 Acquire(thr, pc, submit_sync);
118 dispatch_queue_t q = context->queue;
120 GET_QUEUE_SYNC_VARS(context, q);
121 if (serial_sync) Acquire(thr, pc, serial_sync);
122 if (serial_task && concurrent_sync) Acquire(thr, pc, concurrent_sync);
124 if (q) q = GetTargetQueueFromQueue(q);
128 static void dispatch_sync_post_execute(ThreadState *thr, uptr pc,
129 tsan_block_context_t *context) {
130 uptr submit_sync = (uptr)context;
131 if (context->submitted_synchronously) Release(thr, pc, submit_sync);
133 dispatch_queue_t q = context->queue;
135 GET_QUEUE_SYNC_VARS(context, q);
136 if (serial_task && serial_sync) Release(thr, pc, serial_sync);
137 if (!serial_task && concurrent_sync) Release(thr, pc, concurrent_sync);
139 if (q) q = GetTargetQueueFromQueue(q);
143 static void dispatch_callback_wrap(void *param) {
144 SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap);
145 tsan_block_context_t *context = (tsan_block_context_t *)param;
147 dispatch_sync_pre_execute(thr, pc, context);
149 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
150 context->orig_work(context->orig_context);
151 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
153 dispatch_sync_post_execute(thr, pc, context);
155 if (context->free_context_in_callback) user_free(thr, pc, context);
158 static void invoke_block(void *param) {
159 dispatch_block_t block = (dispatch_block_t)param;
163 static void invoke_and_release_block(void *param) {
164 dispatch_block_t block = (dispatch_block_t)param;
166 Block_release(block);
169 #define DISPATCH_INTERCEPT_B(name, barrier) \
170 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \
171 SCOPED_TSAN_INTERCEPTOR(name, q, block); \
172 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
173 dispatch_block_t heap_block = Block_copy(block); \
174 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
175 tsan_block_context_t *new_context = \
176 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); \
177 new_context->is_barrier_block = barrier; \
178 Release(thr, pc, (uptr)new_context); \
179 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
180 REAL(name##_f)(q, new_context, dispatch_callback_wrap); \
181 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
184 #define DISPATCH_INTERCEPT_SYNC_B(name, barrier) \
185 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, \
186 DISPATCH_NOESCAPE dispatch_block_t block) { \
187 SCOPED_TSAN_INTERCEPTOR(name, q, block); \
188 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
189 dispatch_block_t heap_block = Block_copy(block); \
190 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
191 tsan_block_context_t new_context = { \
192 q, heap_block, &invoke_and_release_block, false, true, barrier, 0}; \
193 Release(thr, pc, (uptr)&new_context); \
194 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
195 REAL(name##_f)(q, &new_context, dispatch_callback_wrap); \
196 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
197 Acquire(thr, pc, (uptr)&new_context); \
200 #define DISPATCH_INTERCEPT_F(name, barrier) \
201 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \
202 dispatch_function_t work) { \
203 SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \
204 tsan_block_context_t *new_context = \
205 AllocContext(thr, pc, q, context, work); \
206 new_context->is_barrier_block = barrier; \
207 Release(thr, pc, (uptr)new_context); \
208 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
209 REAL(name)(q, new_context, dispatch_callback_wrap); \
210 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
213 #define DISPATCH_INTERCEPT_SYNC_F(name, barrier) \
214 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \
215 dispatch_function_t work) { \
216 SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \
217 tsan_block_context_t new_context = { \
218 q, context, work, false, true, barrier, 0}; \
219 Release(thr, pc, (uptr)&new_context); \
220 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
221 REAL(name)(q, &new_context, dispatch_callback_wrap); \
222 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
223 Acquire(thr, pc, (uptr)&new_context); \
226 // We wrap dispatch_async, dispatch_sync and friends where we allocate a new
227 // context, which is used to synchronize (we release the context before
228 // submitting, and the callback acquires it before executing the original
230 DISPATCH_INTERCEPT_B(dispatch_async, false)
231 DISPATCH_INTERCEPT_B(dispatch_barrier_async, true)
232 DISPATCH_INTERCEPT_F(dispatch_async_f, false)
233 DISPATCH_INTERCEPT_F(dispatch_barrier_async_f, true)
234 DISPATCH_INTERCEPT_SYNC_B(dispatch_sync, false)
235 DISPATCH_INTERCEPT_SYNC_B(dispatch_barrier_sync, true)
236 DISPATCH_INTERCEPT_SYNC_F(dispatch_sync_f, false)
237 DISPATCH_INTERCEPT_SYNC_F(dispatch_barrier_sync_f, true)
239 TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when,
240 dispatch_queue_t queue, dispatch_block_t block) {
241 SCOPED_TSAN_INTERCEPTOR(dispatch_after, when, queue, block);
242 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
243 dispatch_block_t heap_block = Block_copy(block);
244 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
245 tsan_block_context_t *new_context =
246 AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block);
247 Release(thr, pc, (uptr)new_context);
248 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
249 REAL(dispatch_after_f)(when, queue, new_context, dispatch_callback_wrap);
250 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
253 TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when,
254 dispatch_queue_t queue, void *context,
255 dispatch_function_t work) {
256 SCOPED_TSAN_INTERCEPTOR(dispatch_after_f, when, queue, context, work);
257 WRAP(dispatch_after)(when, queue, ^(void) {
262 // GCD's dispatch_once implementation has a fast path that contains a racy read
263 // and it's inlined into user's code. Furthermore, this fast path doesn't
264 // establish a proper happens-before relations between the initialization and
265 // code following the call to dispatch_once. We could deal with this in
266 // instrumented code, but there's not much we can do about it in system
267 // libraries. Let's disable the fast path (by never storing the value ~0 to
268 // predicate), so the interceptor is always called, and let's add proper release
269 // and acquire semantics. Since TSan does not see its own atomic stores, the
270 // race on predicate won't be reported - the only accesses to it that TSan sees
271 // are the loads on the fast path. Loads don't race. Secondly, dispatch_once is
272 // both a macro and a real function, we want to intercept the function, so we
273 // need to undefine the macro.
275 TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate,
276 DISPATCH_NOESCAPE dispatch_block_t block) {
277 SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block);
278 atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate);
279 u32 v = atomic_load(a, memory_order_acquire);
281 atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) {
282 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
284 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
285 Release(thr, pc, (uptr)a);
286 atomic_store(a, 2, memory_order_release);
289 internal_sched_yield();
290 v = atomic_load(a, memory_order_acquire);
292 Acquire(thr, pc, (uptr)a);
296 #undef dispatch_once_f
297 TSAN_INTERCEPTOR(void, dispatch_once_f, dispatch_once_t *predicate,
298 void *context, dispatch_function_t function) {
299 SCOPED_INTERCEPTOR_RAW(dispatch_once_f, predicate, context, function);
300 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
301 WRAP(dispatch_once)(predicate, ^(void) {
304 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
307 TSAN_INTERCEPTOR(long_t, dispatch_semaphore_signal,
308 dispatch_semaphore_t dsema) {
309 SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal, dsema);
310 Release(thr, pc, (uptr)dsema);
311 return REAL(dispatch_semaphore_signal)(dsema);
314 TSAN_INTERCEPTOR(long_t, dispatch_semaphore_wait, dispatch_semaphore_t dsema,
315 dispatch_time_t timeout) {
316 SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait, dsema, timeout);
317 long_t result = REAL(dispatch_semaphore_wait)(dsema, timeout);
318 if (result == 0) Acquire(thr, pc, (uptr)dsema);
322 TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group,
323 dispatch_time_t timeout) {
324 SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait, group, timeout);
325 long_t result = REAL(dispatch_group_wait)(group, timeout);
326 if (result == 0) Acquire(thr, pc, (uptr)group);
330 TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) {
331 SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group);
332 // Acquired in the group noticifaction callback in dispatch_group_notify[_f].
333 Release(thr, pc, (uptr)group);
334 REAL(dispatch_group_leave)(group);
337 TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group,
338 dispatch_queue_t queue, dispatch_block_t block) {
339 SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block);
340 dispatch_retain(group);
341 dispatch_group_enter(group);
342 __block dispatch_block_t block_copy = (dispatch_block_t)_Block_copy(block);
343 WRAP(dispatch_async)(queue, ^(void) {
345 _Block_release(block_copy);
346 WRAP(dispatch_group_leave)(group);
347 dispatch_release(group);
351 TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group,
352 dispatch_queue_t queue, void *context,
353 dispatch_function_t work) {
354 SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f, group, queue, context, work);
355 dispatch_retain(group);
356 dispatch_group_enter(group);
357 WRAP(dispatch_async)(queue, ^(void) {
359 WRAP(dispatch_group_leave)(group);
360 dispatch_release(group);
364 TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group,
365 dispatch_queue_t q, dispatch_block_t block) {
366 SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block);
368 // To make sure the group is still available in the callback (otherwise
369 // it can be already destroyed). Will be released in the callback.
370 dispatch_retain(group);
372 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
373 dispatch_block_t heap_block = Block_copy(^(void) {
375 SCOPED_INTERCEPTOR_RAW(dispatch_read_callback);
376 // Released when leaving the group (dispatch_group_leave).
377 Acquire(thr, pc, (uptr)group);
379 dispatch_release(group);
382 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
383 tsan_block_context_t *new_context =
384 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
385 new_context->is_barrier_block = true;
386 Release(thr, pc, (uptr)new_context);
387 REAL(dispatch_group_notify_f)(group, q, new_context, dispatch_callback_wrap);
390 TSAN_INTERCEPTOR(void, dispatch_group_notify_f, dispatch_group_t group,
391 dispatch_queue_t q, void *context, dispatch_function_t work) {
392 WRAP(dispatch_group_notify)(group, q, ^(void) { work(context); });
395 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler,
396 dispatch_source_t source, dispatch_block_t handler) {
397 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler, source, handler);
398 if (handler == nullptr)
399 return REAL(dispatch_source_set_event_handler)(source, nullptr);
400 dispatch_queue_t q = GetTargetQueueFromSource(source);
401 __block tsan_block_context_t new_context = {
402 q, handler, &invoke_block, false, false, false, 0 };
403 dispatch_block_t new_handler = Block_copy(^(void) {
404 new_context.orig_context = handler; // To explicitly capture "handler".
405 dispatch_callback_wrap(&new_context);
407 uptr submit_sync = (uptr)&new_context;
408 Release(thr, pc, submit_sync);
409 REAL(dispatch_source_set_event_handler)(source, new_handler);
410 Block_release(new_handler);
413 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler_f,
414 dispatch_source_t source, dispatch_function_t handler) {
415 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler_f, source, handler);
416 if (handler == nullptr)
417 return REAL(dispatch_source_set_event_handler)(source, nullptr);
418 dispatch_block_t block = ^(void) {
419 handler(dispatch_get_context(source));
421 WRAP(dispatch_source_set_event_handler)(source, block);
424 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler,
425 dispatch_source_t source, dispatch_block_t handler) {
426 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler, source, handler);
427 if (handler == nullptr)
428 return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
429 dispatch_queue_t q = GetTargetQueueFromSource(source);
430 __block tsan_block_context_t new_context = {
431 q, handler, &invoke_block, false, false, false, 0};
432 dispatch_block_t new_handler = Block_copy(^(void) {
433 new_context.orig_context = handler; // To explicitly capture "handler".
434 dispatch_callback_wrap(&new_context);
436 uptr submit_sync = (uptr)&new_context;
437 Release(thr, pc, submit_sync);
438 REAL(dispatch_source_set_cancel_handler)(source, new_handler);
439 Block_release(new_handler);
442 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler_f,
443 dispatch_source_t source, dispatch_function_t handler) {
444 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler_f, source,
446 if (handler == nullptr)
447 return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
448 dispatch_block_t block = ^(void) {
449 handler(dispatch_get_context(source));
451 WRAP(dispatch_source_set_cancel_handler)(source, block);
454 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler,
455 dispatch_source_t source, dispatch_block_t handler) {
456 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler, source,
458 if (handler == nullptr)
459 return REAL(dispatch_source_set_registration_handler)(source, nullptr);
460 dispatch_queue_t q = GetTargetQueueFromSource(source);
461 __block tsan_block_context_t new_context = {
462 q, handler, &invoke_block, false, false, false, 0};
463 dispatch_block_t new_handler = Block_copy(^(void) {
464 new_context.orig_context = handler; // To explicitly capture "handler".
465 dispatch_callback_wrap(&new_context);
467 uptr submit_sync = (uptr)&new_context;
468 Release(thr, pc, submit_sync);
469 REAL(dispatch_source_set_registration_handler)(source, new_handler);
470 Block_release(new_handler);
473 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f,
474 dispatch_source_t source, dispatch_function_t handler) {
475 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler_f, source,
477 if (handler == nullptr)
478 return REAL(dispatch_source_set_registration_handler)(source, nullptr);
479 dispatch_block_t block = ^(void) {
480 handler(dispatch_get_context(source));
482 WRAP(dispatch_source_set_registration_handler)(source, block);
485 TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations,
486 dispatch_queue_t queue,
487 DISPATCH_NOESCAPE void (^block)(size_t)) {
488 SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block);
490 void *parent_to_child_sync = nullptr;
491 uptr parent_to_child_sync_uptr = (uptr)&parent_to_child_sync;
492 void *child_to_parent_sync = nullptr;
493 uptr child_to_parent_sync_uptr = (uptr)&child_to_parent_sync;
495 Release(thr, pc, parent_to_child_sync_uptr);
496 void (^new_block)(size_t) = ^(size_t iteration) {
497 SCOPED_INTERCEPTOR_RAW(dispatch_apply);
498 Acquire(thr, pc, parent_to_child_sync_uptr);
499 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
501 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
502 Release(thr, pc, child_to_parent_sync_uptr);
504 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
505 REAL(dispatch_apply)(iterations, queue, new_block);
506 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
507 Acquire(thr, pc, child_to_parent_sync_uptr);
510 TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations,
511 dispatch_queue_t queue, void *context,
512 void (*work)(void *, size_t)) {
513 SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work);
514 void (^new_block)(size_t) = ^(size_t iteration) {
515 work(context, iteration);
517 WRAP(dispatch_apply)(iterations, queue, new_block);
520 DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
521 DECLARE_REAL_AND_INTERCEPTOR(int, munmap, void *addr, long_t sz)
523 TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer,
524 size_t size, dispatch_queue_t q, dispatch_block_t destructor) {
525 SCOPED_TSAN_INTERCEPTOR(dispatch_data_create, buffer, size, q, destructor);
526 if ((q == nullptr) || (destructor == DISPATCH_DATA_DESTRUCTOR_DEFAULT))
527 return REAL(dispatch_data_create)(buffer, size, q, destructor);
529 if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE)
530 destructor = ^(void) { WRAP(free)((void *)(uintptr_t)buffer); };
531 else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP)
532 destructor = ^(void) { WRAP(munmap)((void *)(uintptr_t)buffer, size); };
534 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
535 dispatch_block_t heap_block = Block_copy(destructor);
536 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
537 tsan_block_context_t *new_context =
538 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
539 uptr submit_sync = (uptr)new_context;
540 Release(thr, pc, submit_sync);
541 return REAL(dispatch_data_create)(buffer, size, q, ^(void) {
542 dispatch_callback_wrap(new_context);
546 typedef void (^fd_handler_t)(dispatch_data_t data, int error);
547 typedef void (^cleanup_handler_t)(int error);
549 TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length,
550 dispatch_queue_t q, fd_handler_t h) {
551 SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h);
552 __block tsan_block_context_t new_context = {
553 q, nullptr, &invoke_block, false, false, false, 0};
554 fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
555 new_context.orig_context = ^(void) {
558 dispatch_callback_wrap(&new_context);
560 uptr submit_sync = (uptr)&new_context;
561 Release(thr, pc, submit_sync);
562 REAL(dispatch_read)(fd, length, q, new_h);
563 Block_release(new_h);
566 TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data,
567 dispatch_queue_t q, fd_handler_t h) {
568 SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h);
569 __block tsan_block_context_t new_context = {
570 q, nullptr, &invoke_block, false, false, false, 0};
571 fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
572 new_context.orig_context = ^(void) {
575 dispatch_callback_wrap(&new_context);
577 uptr submit_sync = (uptr)&new_context;
578 Release(thr, pc, submit_sync);
579 REAL(dispatch_write)(fd, data, q, new_h);
580 Block_release(new_h);
583 TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset,
584 size_t length, dispatch_queue_t q, dispatch_io_handler_t h) {
585 SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h);
586 __block tsan_block_context_t new_context = {
587 q, nullptr, &invoke_block, false, false, false, 0};
588 dispatch_io_handler_t new_h =
589 Block_copy(^(bool done, dispatch_data_t data, int error) {
590 new_context.orig_context = ^(void) {
591 h(done, data, error);
593 dispatch_callback_wrap(&new_context);
595 uptr submit_sync = (uptr)&new_context;
596 Release(thr, pc, submit_sync);
597 REAL(dispatch_io_read)(channel, offset, length, q, new_h);
598 Block_release(new_h);
601 TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset,
602 dispatch_data_t data, dispatch_queue_t q,
603 dispatch_io_handler_t h) {
604 SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h);
605 __block tsan_block_context_t new_context = {
606 q, nullptr, &invoke_block, false, false, false, 0};
607 dispatch_io_handler_t new_h =
608 Block_copy(^(bool done, dispatch_data_t data, int error) {
609 new_context.orig_context = ^(void) {
610 h(done, data, error);
612 dispatch_callback_wrap(&new_context);
614 uptr submit_sync = (uptr)&new_context;
615 Release(thr, pc, submit_sync);
616 REAL(dispatch_io_write)(channel, offset, data, q, new_h);
617 Block_release(new_h);
620 TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel,
621 dispatch_block_t barrier) {
622 SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier);
623 __block tsan_block_context_t new_context = {
624 nullptr, nullptr, &invoke_block, false, false, false, 0};
625 new_context.non_queue_sync_object = (uptr)channel;
626 new_context.is_barrier_block = true;
627 dispatch_block_t new_block = Block_copy(^(void) {
628 new_context.orig_context = ^(void) {
631 dispatch_callback_wrap(&new_context);
633 uptr submit_sync = (uptr)&new_context;
634 Release(thr, pc, submit_sync);
635 REAL(dispatch_io_barrier)(channel, new_block);
636 Block_release(new_block);
639 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type,
640 dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) {
641 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h);
642 __block dispatch_io_t new_channel = nullptr;
643 __block tsan_block_context_t new_context = {
644 q, nullptr, &invoke_block, false, false, false, 0};
645 cleanup_handler_t new_h = Block_copy(^(int error) {
647 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
648 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close.
650 new_context.orig_context = ^(void) {
653 dispatch_callback_wrap(&new_context);
655 uptr submit_sync = (uptr)&new_context;
656 Release(thr, pc, submit_sync);
657 new_channel = REAL(dispatch_io_create)(type, fd, q, new_h);
658 Block_release(new_h);
662 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path,
663 dispatch_io_type_t type, const char *path, int oflag,
664 mode_t mode, dispatch_queue_t q, cleanup_handler_t h) {
665 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode,
667 __block dispatch_io_t new_channel = nullptr;
668 __block tsan_block_context_t new_context = {
669 q, nullptr, &invoke_block, false, false, false, 0};
670 cleanup_handler_t new_h = Block_copy(^(int error) {
672 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
673 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close.
675 new_context.orig_context = ^(void) {
678 dispatch_callback_wrap(&new_context);
680 uptr submit_sync = (uptr)&new_context;
681 Release(thr, pc, submit_sync);
683 REAL(dispatch_io_create_with_path)(type, path, oflag, mode, q, new_h);
684 Block_release(new_h);
688 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io,
689 dispatch_io_type_t type, dispatch_io_t io, dispatch_queue_t q,
690 cleanup_handler_t h) {
691 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h);
692 __block dispatch_io_t new_channel = nullptr;
693 __block tsan_block_context_t new_context = {
694 q, nullptr, &invoke_block, false, false, false, 0};
695 cleanup_handler_t new_h = Block_copy(^(int error) {
697 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
698 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close.
700 new_context.orig_context = ^(void) {
703 dispatch_callback_wrap(&new_context);
705 uptr submit_sync = (uptr)&new_context;
706 Release(thr, pc, submit_sync);
707 new_channel = REAL(dispatch_io_create_with_io)(type, io, q, new_h);
708 Block_release(new_h);
712 TSAN_INTERCEPTOR(void, dispatch_io_close, dispatch_io_t channel,
713 dispatch_io_close_flags_t flags) {
714 SCOPED_TSAN_INTERCEPTOR(dispatch_io_close, channel, flags);
715 Release(thr, pc, (uptr)channel); // Acquire() in dispatch_io_create[_*].
716 return REAL(dispatch_io_close)(channel, flags);
719 // Resuming a suspended queue needs to synchronize with all subsequent
720 // executions of blocks in that queue.
721 TSAN_INTERCEPTOR(void, dispatch_resume, dispatch_object_t o) {
722 SCOPED_TSAN_INTERCEPTOR(dispatch_resume, o);
723 Release(thr, pc, (uptr)o); // Synchronizes with the Acquire() on serial_sync
724 // in dispatch_sync_pre_execute
725 return REAL(dispatch_resume)(o);
728 } // namespace __tsan
730 #endif // SANITIZER_MAC