1 //===-- tsan_libdispatch_mac.cc -------------------------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file is a part of ThreadSanitizer (TSan), a race detector.
12 // Mac-specific libdispatch (GCD) support.
13 //===----------------------------------------------------------------------===//
15 #include "sanitizer_common/sanitizer_platform.h"
18 #include "sanitizer_common/sanitizer_common.h"
19 #include "interception/interception.h"
20 #include "tsan_interceptors.h"
21 #include "tsan_platform.h"
25 #include <dispatch/dispatch.h>
28 typedef long long_t; // NOLINT
33 dispatch_queue_t queue;
35 dispatch_function_t orig_work;
36 bool free_context_in_callback;
37 bool submitted_synchronously;
38 bool is_barrier_block;
39 uptr non_queue_sync_object;
40 } tsan_block_context_t;
42 // The offsets of different fields of the dispatch_queue_t structure, exported
43 // by libdispatch.dylib.
44 extern "C" struct dispatch_queue_offsets_s {
45 const uint16_t dqo_version;
46 const uint16_t dqo_label;
47 const uint16_t dqo_label_size;
48 const uint16_t dqo_flags;
49 const uint16_t dqo_flags_size;
50 const uint16_t dqo_serialnum;
51 const uint16_t dqo_serialnum_size;
52 const uint16_t dqo_width;
53 const uint16_t dqo_width_size;
54 const uint16_t dqo_running;
55 const uint16_t dqo_running_size;
56 const uint16_t dqo_suspend_cnt;
57 const uint16_t dqo_suspend_cnt_size;
58 const uint16_t dqo_target_queue;
59 const uint16_t dqo_target_queue_size;
60 const uint16_t dqo_priority;
61 const uint16_t dqo_priority_size;
62 } dispatch_queue_offsets;
64 static bool IsQueueSerial(dispatch_queue_t q) {
65 CHECK_EQ(dispatch_queue_offsets.dqo_width_size, 2);
66 uptr width = *(uint16_t *)(((uptr)q) + dispatch_queue_offsets.dqo_width);
71 static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) {
72 CHECK_EQ(dispatch_queue_offsets.dqo_target_queue_size, 8);
73 dispatch_queue_t target_queue =
74 *(dispatch_queue_t *)(((uptr)source) +
75 dispatch_queue_offsets.dqo_target_queue);
76 CHECK_NE(target_queue, 0);
80 static tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc,
81 dispatch_queue_t queue,
83 dispatch_function_t orig_work) {
84 tsan_block_context_t *new_context =
85 (tsan_block_context_t *)user_alloc(thr, pc, sizeof(tsan_block_context_t));
86 new_context->queue = queue;
87 new_context->orig_context = orig_context;
88 new_context->orig_work = orig_work;
89 new_context->free_context_in_callback = true;
90 new_context->submitted_synchronously = false;
91 new_context->is_barrier_block = false;
95 static void dispatch_callback_wrap(void *param) {
96 SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap);
97 tsan_block_context_t *context = (tsan_block_context_t *)param;
98 bool is_queue_serial = context->queue && IsQueueSerial(context->queue);
99 uptr sync_ptr = (uptr)context->queue ?: context->non_queue_sync_object;
101 uptr serial_sync = (uptr)sync_ptr;
102 uptr concurrent_sync = ((uptr)sync_ptr) + sizeof(uptr);
103 uptr submit_sync = (uptr)context;
104 bool serial_task = context->is_barrier_block || is_queue_serial;
106 Acquire(thr, pc, submit_sync);
107 Acquire(thr, pc, serial_sync);
108 if (serial_task) Acquire(thr, pc, concurrent_sync);
110 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
111 context->orig_work(context->orig_context);
112 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
114 Release(thr, pc, serial_task ? serial_sync : concurrent_sync);
115 if (context->submitted_synchronously) Release(thr, pc, submit_sync);
117 if (context->free_context_in_callback) user_free(thr, pc, context);
120 static void invoke_block(void *param) {
121 dispatch_block_t block = (dispatch_block_t)param;
125 static void invoke_and_release_block(void *param) {
126 dispatch_block_t block = (dispatch_block_t)param;
128 Block_release(block);
131 #define DISPATCH_INTERCEPT_B(name, barrier) \
132 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \
133 SCOPED_TSAN_INTERCEPTOR(name, q, block); \
134 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
135 dispatch_block_t heap_block = Block_copy(block); \
136 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
137 tsan_block_context_t *new_context = \
138 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); \
139 new_context->is_barrier_block = barrier; \
140 Release(thr, pc, (uptr)new_context); \
141 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
142 REAL(name##_f)(q, new_context, dispatch_callback_wrap); \
143 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
146 #define DISPATCH_INTERCEPT_SYNC_B(name, barrier) \
147 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \
148 SCOPED_TSAN_INTERCEPTOR(name, q, block); \
149 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
150 dispatch_block_t heap_block = Block_copy(block); \
151 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
152 tsan_block_context_t new_context = { \
153 q, heap_block, &invoke_and_release_block, false, true, barrier, 0}; \
154 Release(thr, pc, (uptr)&new_context); \
155 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
156 REAL(name##_f)(q, &new_context, dispatch_callback_wrap); \
157 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
158 Acquire(thr, pc, (uptr)&new_context); \
161 #define DISPATCH_INTERCEPT_F(name, barrier) \
162 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \
163 dispatch_function_t work) { \
164 SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \
165 tsan_block_context_t *new_context = \
166 AllocContext(thr, pc, q, context, work); \
167 new_context->is_barrier_block = barrier; \
168 Release(thr, pc, (uptr)new_context); \
169 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
170 REAL(name)(q, new_context, dispatch_callback_wrap); \
171 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
174 #define DISPATCH_INTERCEPT_SYNC_F(name, barrier) \
175 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \
176 dispatch_function_t work) { \
177 SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \
178 tsan_block_context_t new_context = { \
179 q, context, work, false, true, barrier, 0}; \
180 Release(thr, pc, (uptr)&new_context); \
181 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
182 REAL(name)(q, &new_context, dispatch_callback_wrap); \
183 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
184 Acquire(thr, pc, (uptr)&new_context); \
187 // We wrap dispatch_async, dispatch_sync and friends where we allocate a new
188 // context, which is used to synchronize (we release the context before
189 // submitting, and the callback acquires it before executing the original
191 DISPATCH_INTERCEPT_B(dispatch_async, false)
192 DISPATCH_INTERCEPT_B(dispatch_barrier_async, true)
193 DISPATCH_INTERCEPT_F(dispatch_async_f, false)
194 DISPATCH_INTERCEPT_F(dispatch_barrier_async_f, true)
195 DISPATCH_INTERCEPT_SYNC_B(dispatch_sync, false)
196 DISPATCH_INTERCEPT_SYNC_B(dispatch_barrier_sync, true)
197 DISPATCH_INTERCEPT_SYNC_F(dispatch_sync_f, false)
198 DISPATCH_INTERCEPT_SYNC_F(dispatch_barrier_sync_f, true)
200 TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when,
201 dispatch_queue_t queue, dispatch_block_t block) {
202 SCOPED_TSAN_INTERCEPTOR(dispatch_after, when, queue, block);
203 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
204 dispatch_block_t heap_block = Block_copy(block);
205 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
206 tsan_block_context_t *new_context =
207 AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block);
208 Release(thr, pc, (uptr)new_context);
209 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
210 REAL(dispatch_after_f)(when, queue, new_context, dispatch_callback_wrap);
211 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
214 TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when,
215 dispatch_queue_t queue, void *context,
216 dispatch_function_t work) {
217 SCOPED_TSAN_INTERCEPTOR(dispatch_after_f, when, queue, context, work);
218 WRAP(dispatch_after)(when, queue, ^(void) {
223 // GCD's dispatch_once implementation has a fast path that contains a racy read
224 // and it's inlined into user's code. Furthermore, this fast path doesn't
225 // establish a proper happens-before relations between the initialization and
226 // code following the call to dispatch_once. We could deal with this in
227 // instrumented code, but there's not much we can do about it in system
228 // libraries. Let's disable the fast path (by never storing the value ~0 to
229 // predicate), so the interceptor is always called, and let's add proper release
230 // and acquire semantics. Since TSan does not see its own atomic stores, the
231 // race on predicate won't be reported - the only accesses to it that TSan sees
232 // are the loads on the fast path. Loads don't race. Secondly, dispatch_once is
233 // both a macro and a real function, we want to intercept the function, so we
234 // need to undefine the macro.
236 TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate,
237 dispatch_block_t block) {
238 SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block);
239 atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate);
240 u32 v = atomic_load(a, memory_order_acquire);
242 atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) {
243 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
245 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
246 Release(thr, pc, (uptr)a);
247 atomic_store(a, 2, memory_order_release);
250 internal_sched_yield();
251 v = atomic_load(a, memory_order_acquire);
253 Acquire(thr, pc, (uptr)a);
257 #undef dispatch_once_f
258 TSAN_INTERCEPTOR(void, dispatch_once_f, dispatch_once_t *predicate,
259 void *context, dispatch_function_t function) {
260 SCOPED_INTERCEPTOR_RAW(dispatch_once_f, predicate, context, function);
261 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
262 WRAP(dispatch_once)(predicate, ^(void) {
265 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
268 TSAN_INTERCEPTOR(long_t, dispatch_semaphore_signal,
269 dispatch_semaphore_t dsema) {
270 SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal, dsema);
271 Release(thr, pc, (uptr)dsema);
272 return REAL(dispatch_semaphore_signal)(dsema);
275 TSAN_INTERCEPTOR(long_t, dispatch_semaphore_wait, dispatch_semaphore_t dsema,
276 dispatch_time_t timeout) {
277 SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait, dsema, timeout);
278 long_t result = REAL(dispatch_semaphore_wait)(dsema, timeout);
279 if (result == 0) Acquire(thr, pc, (uptr)dsema);
283 TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group,
284 dispatch_time_t timeout) {
285 SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait, group, timeout);
286 long_t result = REAL(dispatch_group_wait)(group, timeout);
287 if (result == 0) Acquire(thr, pc, (uptr)group);
291 TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) {
292 SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group);
293 // Acquired in the group noticifaction callback in dispatch_group_notify[_f].
294 Release(thr, pc, (uptr)group);
295 REAL(dispatch_group_leave)(group);
298 TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group,
299 dispatch_queue_t queue, dispatch_block_t block) {
300 SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block);
301 dispatch_retain(group);
302 dispatch_group_enter(group);
303 __block dispatch_block_t block_copy = (dispatch_block_t)_Block_copy(block);
304 WRAP(dispatch_async)(queue, ^(void) {
306 _Block_release(block_copy);
307 WRAP(dispatch_group_leave)(group);
308 dispatch_release(group);
312 TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group,
313 dispatch_queue_t queue, void *context,
314 dispatch_function_t work) {
315 SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f, group, queue, context, work);
316 dispatch_retain(group);
317 dispatch_group_enter(group);
318 WRAP(dispatch_async)(queue, ^(void) {
320 WRAP(dispatch_group_leave)(group);
321 dispatch_release(group);
325 TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group,
326 dispatch_queue_t q, dispatch_block_t block) {
327 SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block);
329 // To make sure the group is still available in the callback (otherwise
330 // it can be already destroyed). Will be released in the callback.
331 dispatch_retain(group);
333 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
334 dispatch_block_t heap_block = Block_copy(^(void) {
336 SCOPED_INTERCEPTOR_RAW(dispatch_read_callback);
337 // Released when leaving the group (dispatch_group_leave).
338 Acquire(thr, pc, (uptr)group);
340 dispatch_release(group);
343 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
344 tsan_block_context_t *new_context =
345 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
346 new_context->is_barrier_block = true;
347 Release(thr, pc, (uptr)new_context);
348 REAL(dispatch_group_notify_f)(group, q, new_context, dispatch_callback_wrap);
351 TSAN_INTERCEPTOR(void, dispatch_group_notify_f, dispatch_group_t group,
352 dispatch_queue_t q, void *context, dispatch_function_t work) {
353 WRAP(dispatch_group_notify)(group, q, ^(void) { work(context); });
356 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler,
357 dispatch_source_t source, dispatch_block_t handler) {
358 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler, source, handler);
359 if (handler == nullptr)
360 return REAL(dispatch_source_set_event_handler)(source, nullptr);
361 dispatch_queue_t q = GetTargetQueueFromSource(source);
362 __block tsan_block_context_t new_context = {
363 q, handler, &invoke_block, false, false, false, 0 };
364 dispatch_block_t new_handler = Block_copy(^(void) {
365 new_context.orig_context = handler; // To explicitly capture "handler".
366 dispatch_callback_wrap(&new_context);
368 uptr submit_sync = (uptr)&new_context;
369 Release(thr, pc, submit_sync);
370 REAL(dispatch_source_set_event_handler)(source, new_handler);
371 Block_release(new_handler);
374 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler_f,
375 dispatch_source_t source, dispatch_function_t handler) {
376 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler_f, source, handler);
377 if (handler == nullptr)
378 return REAL(dispatch_source_set_event_handler)(source, nullptr);
379 dispatch_block_t block = ^(void) {
380 handler(dispatch_get_context(source));
382 WRAP(dispatch_source_set_event_handler)(source, block);
385 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler,
386 dispatch_source_t source, dispatch_block_t handler) {
387 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler, source, handler);
388 if (handler == nullptr)
389 return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
390 dispatch_queue_t q = GetTargetQueueFromSource(source);
391 __block tsan_block_context_t new_context = {
392 q, handler, &invoke_block, false, false, false, 0};
393 dispatch_block_t new_handler = Block_copy(^(void) {
394 new_context.orig_context = handler; // To explicitly capture "handler".
395 dispatch_callback_wrap(&new_context);
397 uptr submit_sync = (uptr)&new_context;
398 Release(thr, pc, submit_sync);
399 REAL(dispatch_source_set_cancel_handler)(source, new_handler);
400 Block_release(new_handler);
403 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler_f,
404 dispatch_source_t source, dispatch_function_t handler) {
405 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler_f, source,
407 if (handler == nullptr)
408 return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
409 dispatch_block_t block = ^(void) {
410 handler(dispatch_get_context(source));
412 WRAP(dispatch_source_set_cancel_handler)(source, block);
415 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler,
416 dispatch_source_t source, dispatch_block_t handler) {
417 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler, source,
419 if (handler == nullptr)
420 return REAL(dispatch_source_set_registration_handler)(source, nullptr);
421 dispatch_queue_t q = GetTargetQueueFromSource(source);
422 __block tsan_block_context_t new_context = {
423 q, handler, &invoke_block, false, false, false, 0};
424 dispatch_block_t new_handler = Block_copy(^(void) {
425 new_context.orig_context = handler; // To explicitly capture "handler".
426 dispatch_callback_wrap(&new_context);
428 uptr submit_sync = (uptr)&new_context;
429 Release(thr, pc, submit_sync);
430 REAL(dispatch_source_set_registration_handler)(source, new_handler);
431 Block_release(new_handler);
434 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f,
435 dispatch_source_t source, dispatch_function_t handler) {
436 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler_f, source,
438 if (handler == nullptr)
439 return REAL(dispatch_source_set_registration_handler)(source, nullptr);
440 dispatch_block_t block = ^(void) {
441 handler(dispatch_get_context(source));
443 WRAP(dispatch_source_set_registration_handler)(source, block);
446 TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations,
447 dispatch_queue_t queue, void (^block)(size_t)) {
448 SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block);
450 void *parent_to_child_sync = nullptr;
451 uptr parent_to_child_sync_uptr = (uptr)&parent_to_child_sync;
452 void *child_to_parent_sync = nullptr;
453 uptr child_to_parent_sync_uptr = (uptr)&child_to_parent_sync;
455 Release(thr, pc, parent_to_child_sync_uptr);
456 void (^new_block)(size_t) = ^(size_t iteration) {
457 SCOPED_INTERCEPTOR_RAW(dispatch_apply);
458 Acquire(thr, pc, parent_to_child_sync_uptr);
459 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
461 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
462 Release(thr, pc, child_to_parent_sync_uptr);
464 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
465 REAL(dispatch_apply)(iterations, queue, new_block);
466 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
467 Acquire(thr, pc, child_to_parent_sync_uptr);
470 TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations,
471 dispatch_queue_t queue, void *context,
472 void (*work)(void *, size_t)) {
473 SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work);
474 void (^new_block)(size_t) = ^(size_t iteration) {
475 work(context, iteration);
477 WRAP(dispatch_apply)(iterations, queue, new_block);
480 DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
481 DECLARE_REAL_AND_INTERCEPTOR(int, munmap, void *addr, long_t sz)
483 TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer,
484 size_t size, dispatch_queue_t q, dispatch_block_t destructor) {
485 SCOPED_TSAN_INTERCEPTOR(dispatch_data_create, buffer, size, q, destructor);
486 if ((q == nullptr) || (destructor == DISPATCH_DATA_DESTRUCTOR_DEFAULT))
487 return REAL(dispatch_data_create)(buffer, size, q, destructor);
489 if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE)
490 destructor = ^(void) { WRAP(free)((void *)buffer); };
491 else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP)
492 destructor = ^(void) { WRAP(munmap)((void *)buffer, size); };
494 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
495 dispatch_block_t heap_block = Block_copy(destructor);
496 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
497 tsan_block_context_t *new_context =
498 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
499 uptr submit_sync = (uptr)new_context;
500 Release(thr, pc, submit_sync);
501 return REAL(dispatch_data_create)(buffer, size, q, ^(void) {
502 dispatch_callback_wrap(new_context);
506 typedef void (^fd_handler_t)(dispatch_data_t data, int error);
507 typedef void (^cleanup_handler_t)(int error);
509 TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length,
510 dispatch_queue_t q, fd_handler_t h) {
511 SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h);
512 __block tsan_block_context_t new_context = {
513 q, nullptr, &invoke_block, false, false, false, 0};
514 fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
515 new_context.orig_context = ^(void) {
518 dispatch_callback_wrap(&new_context);
520 uptr submit_sync = (uptr)&new_context;
521 Release(thr, pc, submit_sync);
522 REAL(dispatch_read)(fd, length, q, new_h);
523 Block_release(new_h);
526 TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data,
527 dispatch_queue_t q, fd_handler_t h) {
528 SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h);
529 __block tsan_block_context_t new_context = {
530 q, nullptr, &invoke_block, false, false, false, 0};
531 fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
532 new_context.orig_context = ^(void) {
535 dispatch_callback_wrap(&new_context);
537 uptr submit_sync = (uptr)&new_context;
538 Release(thr, pc, submit_sync);
539 REAL(dispatch_write)(fd, data, q, new_h);
540 Block_release(new_h);
543 TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset,
544 size_t length, dispatch_queue_t q, dispatch_io_handler_t h) {
545 SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h);
546 __block tsan_block_context_t new_context = {
547 q, nullptr, &invoke_block, false, false, false, 0};
548 dispatch_io_handler_t new_h =
549 Block_copy(^(bool done, dispatch_data_t data, int error) {
550 new_context.orig_context = ^(void) {
551 h(done, data, error);
553 dispatch_callback_wrap(&new_context);
555 uptr submit_sync = (uptr)&new_context;
556 Release(thr, pc, submit_sync);
557 REAL(dispatch_io_read)(channel, offset, length, q, new_h);
558 Block_release(new_h);
561 TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset,
562 dispatch_data_t data, dispatch_queue_t q,
563 dispatch_io_handler_t h) {
564 SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h);
565 __block tsan_block_context_t new_context = {
566 q, nullptr, &invoke_block, false, false, false, 0};
567 dispatch_io_handler_t new_h =
568 Block_copy(^(bool done, dispatch_data_t data, int error) {
569 new_context.orig_context = ^(void) {
570 h(done, data, error);
572 dispatch_callback_wrap(&new_context);
574 uptr submit_sync = (uptr)&new_context;
575 Release(thr, pc, submit_sync);
576 REAL(dispatch_io_write)(channel, offset, data, q, new_h);
577 Block_release(new_h);
580 TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel,
581 dispatch_block_t barrier) {
582 SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier);
583 __block tsan_block_context_t new_context = {
584 nullptr, nullptr, &invoke_block, false, false, false, 0};
585 new_context.non_queue_sync_object = (uptr)channel;
586 new_context.is_barrier_block = true;
587 dispatch_block_t new_block = Block_copy(^(void) {
588 new_context.orig_context = ^(void) {
591 dispatch_callback_wrap(&new_context);
593 uptr submit_sync = (uptr)&new_context;
594 Release(thr, pc, submit_sync);
595 REAL(dispatch_io_barrier)(channel, new_block);
596 Block_release(new_block);
599 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type,
600 dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) {
601 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h);
602 __block dispatch_io_t new_channel = nullptr;
603 __block tsan_block_context_t new_context = {
604 q, nullptr, &invoke_block, false, false, false, 0};
605 cleanup_handler_t new_h = Block_copy(^(int error) {
607 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
608 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close.
610 new_context.orig_context = ^(void) {
613 dispatch_callback_wrap(&new_context);
615 uptr submit_sync = (uptr)&new_context;
616 Release(thr, pc, submit_sync);
617 new_channel = REAL(dispatch_io_create)(type, fd, q, new_h);
618 Block_release(new_h);
622 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path,
623 dispatch_io_type_t type, const char *path, int oflag,
624 mode_t mode, dispatch_queue_t q, cleanup_handler_t h) {
625 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode,
627 __block dispatch_io_t new_channel = nullptr;
628 __block tsan_block_context_t new_context = {
629 q, nullptr, &invoke_block, false, false, false, 0};
630 cleanup_handler_t new_h = Block_copy(^(int error) {
632 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
633 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close.
635 new_context.orig_context = ^(void) {
638 dispatch_callback_wrap(&new_context);
640 uptr submit_sync = (uptr)&new_context;
641 Release(thr, pc, submit_sync);
643 REAL(dispatch_io_create_with_path)(type, path, oflag, mode, q, new_h);
644 Block_release(new_h);
648 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io,
649 dispatch_io_type_t type, dispatch_io_t io, dispatch_queue_t q,
650 cleanup_handler_t h) {
651 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h);
652 __block dispatch_io_t new_channel = nullptr;
653 __block tsan_block_context_t new_context = {
654 q, nullptr, &invoke_block, false, false, false, 0};
655 cleanup_handler_t new_h = Block_copy(^(int error) {
657 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
658 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close.
660 new_context.orig_context = ^(void) {
663 dispatch_callback_wrap(&new_context);
665 uptr submit_sync = (uptr)&new_context;
666 Release(thr, pc, submit_sync);
667 new_channel = REAL(dispatch_io_create_with_io)(type, io, q, new_h);
668 Block_release(new_h);
672 TSAN_INTERCEPTOR(void, dispatch_io_close, dispatch_io_t channel,
673 dispatch_io_close_flags_t flags) {
674 SCOPED_TSAN_INTERCEPTOR(dispatch_io_close, channel, flags);
675 Release(thr, pc, (uptr)channel); // Acquire() in dispatch_io_create[_*].
676 return REAL(dispatch_io_close)(channel, flags);
679 } // namespace __tsan
681 #endif // SANITIZER_MAC