1 //===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file is a part of ThreadSanitizer (TSan), a race detector.
12 // Main internal TSan header file.
15 // - C++ run-time should not be used (static CTORs, RTTI, exceptions, static
16 // function-scope locals)
17 // - All functions/classes/etc reside in namespace __tsan, except for those
18 // declared in tsan_interface.h.
19 // - Platform-specific files should be used instead of ifdefs (*).
20 // - No system headers included in header files (*).
21 // - Platform specific headres included only into platform-specific files (*).
23 // (*) Except when inlining is critical for performance.
24 //===----------------------------------------------------------------------===//
29 #include "sanitizer_common/sanitizer_allocator.h"
30 #include "sanitizer_common/sanitizer_allocator_internal.h"
31 #include "sanitizer_common/sanitizer_asm.h"
32 #include "sanitizer_common/sanitizer_common.h"
33 #include "sanitizer_common/sanitizer_deadlock_detector_interface.h"
34 #include "sanitizer_common/sanitizer_libignore.h"
35 #include "sanitizer_common/sanitizer_suppressions.h"
36 #include "sanitizer_common/sanitizer_thread_registry.h"
37 #include "tsan_clock.h"
38 #include "tsan_defs.h"
39 #include "tsan_flags.h"
40 #include "tsan_sync.h"
41 #include "tsan_trace.h"
42 #include "tsan_vector.h"
43 #include "tsan_report.h"
44 #include "tsan_platform.h"
45 #include "tsan_mutexset.h"
46 #include "tsan_ignoreset.h"
47 #include "tsan_stack_trace.h"
49 #if SANITIZER_WORDSIZE != 64
50 # error "ThreadSanitizer is supported only on 64-bit platforms"
56 struct MapUnmapCallback;
57 #if defined(__mips64) || defined(__aarch64__) || defined(__powerpc__)
58 static const uptr kAllocatorRegionSizeLog = 20;
59 static const uptr kAllocatorNumRegions =
60 SANITIZER_MMAP_RANGE_SIZE >> kAllocatorRegionSizeLog;
61 typedef TwoLevelByteMap<(kAllocatorNumRegions >> 12), 1 << 12,
62 MapUnmapCallback> ByteMap;
64 static const uptr kSpaceBeg = 0;
65 static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
66 static const uptr kMetadataSize = 0;
67 typedef __sanitizer::CompactSizeClassMap SizeClassMap;
68 static const uptr kRegionSizeLog = kAllocatorRegionSizeLog;
69 typedef __tsan::ByteMap ByteMap;
70 typedef __tsan::MapUnmapCallback MapUnmapCallback;
71 static const uptr kFlags = 0;
73 typedef SizeClassAllocator32<AP32> PrimaryAllocator;
75 struct AP64 { // Allocator64 parameters. Deliberately using a short name.
76 static const uptr kSpaceBeg = Mapping::kHeapMemBeg;
77 static const uptr kSpaceSize = Mapping::kHeapMemEnd - Mapping::kHeapMemBeg;
78 static const uptr kMetadataSize = 0;
79 typedef DefaultSizeClassMap SizeClassMap;
80 typedef __tsan::MapUnmapCallback MapUnmapCallback;
81 static const uptr kFlags = 0;
83 typedef SizeClassAllocator64<AP64> PrimaryAllocator;
85 typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache;
86 typedef LargeMmapAllocator<MapUnmapCallback> SecondaryAllocator;
87 typedef CombinedAllocator<PrimaryAllocator, AllocatorCache,
88 SecondaryAllocator> Allocator;
89 Allocator *allocator();
92 void TsanCheckFailed(const char *file, int line, const char *cond,
95 const u64 kShadowRodata = (u64)-1; // .rodata shadow marker
97 // FastState (from most significant bit):
105 FastState(u64 tid, u64 epoch) {
106 x_ = tid << kTidShift;
108 DCHECK_EQ(tid, this->tid());
109 DCHECK_EQ(epoch, this->epoch());
110 DCHECK_EQ(GetIgnoreBit(), false);
113 explicit FastState(u64 x)
122 u64 res = (x_ & ~kIgnoreBit) >> kTidShift;
126 u64 TidWithIgnore() const {
127 u64 res = x_ >> kTidShift;
132 u64 res = x_ & ((1ull << kClkBits) - 1);
136 void IncrementEpoch() {
137 u64 old_epoch = epoch();
139 DCHECK_EQ(old_epoch + 1, epoch());
143 void SetIgnoreBit() { x_ |= kIgnoreBit; }
144 void ClearIgnoreBit() { x_ &= ~kIgnoreBit; }
145 bool GetIgnoreBit() const { return (s64)x_ < 0; }
147 void SetHistorySize(int hs) {
150 x_ = (x_ & ~(kHistoryMask << kHistoryShift)) | (u64(hs) << kHistoryShift);
154 int GetHistorySize() const {
155 return (int)((x_ >> kHistoryShift) & kHistoryMask);
158 void ClearHistorySize() {
163 u64 GetTracePos() const {
164 const int hs = GetHistorySize();
165 // When hs == 0, the trace consists of 2 parts.
166 const u64 mask = (1ull << (kTracePartSizeBits + hs + 1)) - 1;
167 return epoch() & mask;
172 static const int kTidShift = 64 - kTidBits - 1;
173 static const u64 kIgnoreBit = 1ull << 63;
174 static const u64 kFreedBit = 1ull << 63;
175 static const u64 kHistoryShift = kClkBits;
176 static const u64 kHistoryMask = 7;
180 // Shadow (from most significant bit):
188 class Shadow : public FastState {
190 explicit Shadow(u64 x)
194 explicit Shadow(const FastState &s)
199 void SetAddr0AndSizeLog(u64 addr0, unsigned kAccessSizeLog) {
200 DCHECK_EQ((x_ >> kClkBits) & 31, 0);
202 DCHECK_LE(kAccessSizeLog, 3);
203 x_ |= ((kAccessSizeLog << 3) | addr0) << kClkBits;
204 DCHECK_EQ(kAccessSizeLog, size_log());
205 DCHECK_EQ(addr0, this->addr0());
208 void SetWrite(unsigned kAccessIsWrite) {
209 DCHECK_EQ(x_ & kReadBit, 0);
212 DCHECK_EQ(kAccessIsWrite, IsWrite());
215 void SetAtomic(bool kIsAtomic) {
219 DCHECK_EQ(IsAtomic(), kIsAtomic);
222 bool IsAtomic() const {
223 return x_ & kAtomicBit;
226 bool IsZero() const {
230 static inline bool TidsAreEqual(const Shadow s1, const Shadow s2) {
231 u64 shifted_xor = (s1.x_ ^ s2.x_) >> kTidShift;
232 DCHECK_EQ(shifted_xor == 0, s1.TidWithIgnore() == s2.TidWithIgnore());
233 return shifted_xor == 0;
237 bool Addr0AndSizeAreEqual(const Shadow s1, const Shadow s2) {
238 u64 masked_xor = ((s1.x_ ^ s2.x_) >> kClkBits) & 31;
239 return masked_xor == 0;
242 static ALWAYS_INLINE bool TwoRangesIntersect(Shadow s1, Shadow s2,
243 unsigned kS2AccessSize) {
245 u64 diff = s1.addr0() - s2.addr0();
246 if ((s64)diff < 0) { // s1.addr0 < s2.addr0 // NOLINT
247 // if (s1.addr0() + size1) > s2.addr0()) return true;
248 if (s1.size() > -diff)
251 // if (s2.addr0() + kS2AccessSize > s1.addr0()) return true;
252 if (kS2AccessSize > diff)
255 DCHECK_EQ(res, TwoRangesIntersectSlow(s1, s2));
256 DCHECK_EQ(res, TwoRangesIntersectSlow(s2, s1));
260 u64 ALWAYS_INLINE addr0() const { return (x_ >> kClkBits) & 7; }
261 u64 ALWAYS_INLINE size() const { return 1ull << size_log(); }
262 bool ALWAYS_INLINE IsWrite() const { return !IsRead(); }
263 bool ALWAYS_INLINE IsRead() const { return x_ & kReadBit; }
265 // The idea behind the freed bit is as follows.
266 // When the memory is freed (or otherwise unaccessible) we write to the shadow
267 // values with tid/epoch related to the free and the freed bit set.
268 // During memory accesses processing the freed bit is considered
269 // as msb of tid. So any access races with shadow with freed bit set
270 // (it is as if write from a thread with which we never synchronized before).
271 // This allows us to detect accesses to freed memory w/o additional
272 // overheads in memory access processing and at the same time restore
273 // tid/epoch of free.
278 bool IsFreed() const {
279 return x_ & kFreedBit;
282 bool GetFreedAndReset() {
283 bool res = x_ & kFreedBit;
288 bool ALWAYS_INLINE IsBothReadsOrAtomic(bool kIsWrite, bool kIsAtomic) const {
289 bool v = x_ & ((u64(kIsWrite ^ 1) << kReadShift)
290 | (u64(kIsAtomic) << kAtomicShift));
291 DCHECK_EQ(v, (!IsWrite() && !kIsWrite) || (IsAtomic() && kIsAtomic));
295 bool ALWAYS_INLINE IsRWNotWeaker(bool kIsWrite, bool kIsAtomic) const {
296 bool v = ((x_ >> kReadShift) & 3)
297 <= u64((kIsWrite ^ 1) | (kIsAtomic << 1));
298 DCHECK_EQ(v, (IsAtomic() < kIsAtomic) ||
299 (IsAtomic() == kIsAtomic && !IsWrite() <= !kIsWrite));
303 bool ALWAYS_INLINE IsRWWeakerOrEqual(bool kIsWrite, bool kIsAtomic) const {
304 bool v = ((x_ >> kReadShift) & 3)
305 >= u64((kIsWrite ^ 1) | (kIsAtomic << 1));
306 DCHECK_EQ(v, (IsAtomic() > kIsAtomic) ||
307 (IsAtomic() == kIsAtomic && !IsWrite() >= !kIsWrite));
312 static const u64 kReadShift = 5 + kClkBits;
313 static const u64 kReadBit = 1ull << kReadShift;
314 static const u64 kAtomicShift = 6 + kClkBits;
315 static const u64 kAtomicBit = 1ull << kAtomicShift;
317 u64 size_log() const { return (x_ >> (3 + kClkBits)) & 3; }
319 static bool TwoRangesIntersectSlow(const Shadow s1, const Shadow s2) {
320 if (s1.addr0() == s2.addr0()) return true;
321 if (s1.addr0() < s2.addr0() && s1.addr0() + s1.size() > s2.addr0())
323 if (s2.addr0() < s1.addr0() && s2.addr0() + s2.size() > s1.addr0())
329 struct ThreadSignalContext;
335 bool in_blocking_func;
336 uptr in_signal_handler;
337 uptr *shadow_stack_pos;
340 // A Processor represents a physical thread, or a P for Go.
341 // It is used to store internal resources like allocate cache, and does not
342 // participate in race-detection logic (invisible to end user).
343 // In C++ it is tied to an OS thread just like ThreadState, however ideally
344 // it should be tied to a CPU (this way we will have fewer allocator caches).
345 // In Go it is tied to a P, so there are significantly fewer Processor's than
346 // ThreadState's (which are tied to Gs).
347 // A ThreadState must be wired with a Processor to handle events.
349 ThreadState *thr; // currently wired thread, or nullptr
351 AllocatorCache alloc_cache;
352 InternalAllocatorCache internal_alloc_cache;
354 DenseSlabAllocCache block_cache;
355 DenseSlabAllocCache sync_cache;
356 DenseSlabAllocCache clock_cache;
357 DDPhysicalThread *dd_pt;
361 // ScopedGlobalProcessor temporary setups a global processor for the current
362 // thread, if it does not have one. Intended for interceptors that can run
363 // at the very thread end, when we already destroyed the thread processor.
364 struct ScopedGlobalProcessor {
365 ScopedGlobalProcessor();
366 ~ScopedGlobalProcessor();
370 // This struct is stored in TLS.
372 FastState fast_state;
373 // Synch epoch represents the threads's epoch before the last synchronization
374 // action. It allows to reduce number of shadow state updates.
375 // For example, fast_synch_epoch=100, last write to addr X was at epoch=150,
376 // if we are processing write to X from the same thread at epoch=200,
377 // we do nothing, because both writes happen in the same 'synch epoch'.
378 // That is, if another memory access does not race with the former write,
379 // it does not race with the latter as well.
380 // QUESTION: can we can squeeze this into ThreadState::Fast?
381 // E.g. ThreadState::Fast is a 44-bit, 32 are taken by synch_epoch and 12 are
382 // taken by epoch between synchs.
383 // This way we can save one load from tls.
384 u64 fast_synch_epoch;
385 // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read.
386 // We do not distinguish beteween ignoring reads and writes
387 // for better performance.
388 int ignore_reads_and_writes;
390 int suppress_reports;
391 // Go does not support ignores.
393 IgnoreSet mop_ignore_set;
394 IgnoreSet sync_ignore_set;
396 // C/C++ uses fixed size shadow stack embed into Trace.
397 // Go uses malloc-allocated shadow stack with dynamic size.
399 uptr *shadow_stack_end;
400 uptr *shadow_stack_pos;
401 u64 *racy_shadow_addr;
406 Vector<JmpBuf> jmp_bufs;
407 int ignore_interceptors;
409 #if TSAN_COLLECT_STATS
426 #if SANITIZER_DEBUG && !SANITIZER_GO
427 InternalDeadlockDetector internal_deadlock_detector;
429 DDLogicalThread *dd_lt;
431 // Current wired Processor, or nullptr. Required to handle any events.
434 Processor *proc() { return proc1; }
439 atomic_uintptr_t in_signal_handler;
440 ThreadSignalContext *signal_ctx;
443 u32 last_sleep_stack_id;
444 ThreadClock last_sleep_clock;
447 // Set in regions of runtime that must be signal-safe and fork-safe.
448 // If set, malloc must not be called.
451 const ReportDesc *current_report;
453 explicit ThreadState(Context *ctx, int tid, int unique_id, u64 epoch,
454 unsigned reuse_count,
455 uptr stk_addr, uptr stk_size,
456 uptr tls_addr, uptr tls_size);
460 #if SANITIZER_MAC || SANITIZER_ANDROID
461 ThreadState *cur_thread();
462 void cur_thread_finalize();
464 __attribute__((tls_model("initial-exec")))
465 extern THREADLOCAL char cur_thread_placeholder[];
466 INLINE ThreadState *cur_thread() {
467 return reinterpret_cast<ThreadState *>(&cur_thread_placeholder);
469 INLINE void cur_thread_finalize() { }
470 #endif // SANITIZER_MAC || SANITIZER_ANDROID
471 #endif // SANITIZER_GO
473 class ThreadContext : public ThreadContextBase {
475 explicit ThreadContext(int tid);
478 u32 creation_stack_id;
480 // Epoch at which the thread had started.
481 // If we see an event from the thread stamped by an older epoch,
482 // the event is from a dead thread that shared tid with this thread.
486 // Override superclass callbacks.
487 void OnDead() override;
488 void OnJoined(void *arg) override;
489 void OnFinished() override;
490 void OnStarted(void *arg) override;
491 void OnCreated(void *arg) override;
492 void OnReset() override;
493 void OnDetached(void *arg) override;
498 bool operator==(const RacyStacks &other) const {
499 if (hash[0] == other.hash[0] && hash[1] == other.hash[1])
501 if (hash[0] == other.hash[1] && hash[1] == other.hash[0])
512 struct FiredSuppression {
522 bool after_multithreaded_fork;
528 int nmissed_expected;
529 atomic_uint64_t last_symbolize_time_ns;
531 void *background_thread;
532 atomic_uint32_t stop_background_thread;
534 ThreadRegistry *thread_registry;
537 Vector<RacyStacks> racy_stacks;
538 Vector<RacyAddress> racy_addresses;
539 // Number of fired suppressions may be large enough.
540 Mutex fired_suppressions_mtx;
541 InternalMmapVector<FiredSuppression> fired_suppressions;
544 ClockAlloc clock_alloc;
549 u64 int_alloc_cnt[MBlockTypeCount];
550 u64 int_alloc_siz[MBlockTypeCount];
553 extern Context *ctx; // The one and the only global runtime context.
555 ALWAYS_INLINE Flags *flags() {
559 struct ScopedIgnoreInterceptors {
560 ScopedIgnoreInterceptors() {
562 cur_thread()->ignore_interceptors++;
566 ~ScopedIgnoreInterceptors() {
568 cur_thread()->ignore_interceptors--;
573 const char *GetObjectTypeFromTag(uptr tag);
574 const char *GetReportHeaderFromTag(uptr tag);
575 uptr TagFromShadowStackFrame(uptr pc);
579 explicit ScopedReport(ReportType typ, uptr tag = kExternalTagNone);
582 void AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, StackTrace stack,
583 const MutexSet *mset);
584 void AddStack(StackTrace stack, bool suppressable = false);
585 void AddThread(const ThreadContext *tctx, bool suppressable = false);
586 void AddThread(int unique_tid, bool suppressable = false);
587 void AddUniqueTid(int unique_tid);
588 void AddMutex(const SyncVar *s);
589 u64 AddMutex(u64 id);
590 void AddLocation(uptr addr, uptr size);
591 void AddSleep(u32 stack_id);
592 void SetCount(int count);
594 const ReportDesc *GetReport() const;
598 // Symbolizer makes lots of intercepted calls. If we try to process them,
599 // at best it will cause deadlocks on internal mutexes.
600 ScopedIgnoreInterceptors ignore_interceptors_;
602 void AddDeadMutex(u64 id);
604 ScopedReport(const ScopedReport&);
605 void operator = (const ScopedReport&);
608 ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack);
609 void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk,
610 MutexSet *mset, uptr *tag = nullptr);
612 // The stack could look like:
613 // <start> | <main> | <foo> | tag | <bar>
614 // This will extract the tag and keep:
615 // <start> | <main> | <foo> | <bar>
616 template<typename StackTraceTy>
617 void ExtractTagFromStack(StackTraceTy *stack, uptr *tag = nullptr) {
618 if (stack->size < 2) return;
619 uptr possible_tag_pc = stack->trace[stack->size - 2];
620 uptr possible_tag = TagFromShadowStackFrame(possible_tag_pc);
621 if (possible_tag == kExternalTagNone) return;
622 stack->trace_buffer[stack->size - 2] = stack->trace_buffer[stack->size - 1];
624 if (tag) *tag = possible_tag;
627 template<typename StackTraceTy>
628 void ObtainCurrentStack(ThreadState *thr, uptr toppc, StackTraceTy *stack,
629 uptr *tag = nullptr) {
630 uptr size = thr->shadow_stack_pos - thr->shadow_stack;
632 if (size + !!toppc > kStackTraceMax) {
633 start = size + !!toppc - kStackTraceMax;
634 size = kStackTraceMax - !!toppc;
636 stack->Init(&thr->shadow_stack[start], size, toppc);
637 ExtractTagFromStack(stack, tag);
641 #if TSAN_COLLECT_STATS
642 void StatAggregate(u64 *dst, u64 *src);
643 void StatOutput(u64 *stat);
646 void ALWAYS_INLINE StatInc(ThreadState *thr, StatType typ, u64 n = 1) {
647 #if TSAN_COLLECT_STATS
651 void ALWAYS_INLINE StatSet(ThreadState *thr, StatType typ, u64 n) {
652 #if TSAN_COLLECT_STATS
657 void MapShadow(uptr addr, uptr size);
658 void MapThreadTrace(uptr addr, uptr size, const char *name);
659 void DontNeedShadowFor(uptr addr, uptr size);
660 void InitializeShadowMemory();
661 void InitializeInterceptors();
662 void InitializeLibIgnore();
663 void InitializeDynamicAnnotations();
665 void ForkBefore(ThreadState *thr, uptr pc);
666 void ForkParentAfter(ThreadState *thr, uptr pc);
667 void ForkChildAfter(ThreadState *thr, uptr pc);
669 void ReportRace(ThreadState *thr);
670 bool OutputReport(ThreadState *thr, const ScopedReport &srep);
671 bool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace);
672 bool IsExpectedReport(uptr addr, uptr size);
673 void PrintMatchedBenignRaces();
675 #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1
676 # define DPrintf Printf
678 # define DPrintf(...)
681 #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 2
682 # define DPrintf2 Printf
684 # define DPrintf2(...)
687 u32 CurrentStackId(ThreadState *thr, uptr pc);
688 ReportStack *SymbolizeStackId(u32 stack_id);
689 void PrintCurrentStack(ThreadState *thr, uptr pc);
690 void PrintCurrentStackSlow(uptr pc); // uses libunwind
692 void Initialize(ThreadState *thr);
693 int Finalize(ThreadState *thr);
695 void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write);
696 void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write);
698 void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,
699 int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic);
700 void MemoryAccessImpl(ThreadState *thr, uptr addr,
701 int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic,
702 u64 *shadow_mem, Shadow cur);
703 void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr,
704 uptr size, bool is_write);
705 void MemoryAccessRangeStep(ThreadState *thr, uptr pc, uptr addr,
706 uptr size, uptr step, bool is_write);
707 void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr,
708 int size, bool kAccessIsWrite, bool kIsAtomic);
710 const int kSizeLog1 = 0;
711 const int kSizeLog2 = 1;
712 const int kSizeLog4 = 2;
713 const int kSizeLog8 = 3;
715 void ALWAYS_INLINE MemoryRead(ThreadState *thr, uptr pc,
716 uptr addr, int kAccessSizeLog) {
717 MemoryAccess(thr, pc, addr, kAccessSizeLog, false, false);
720 void ALWAYS_INLINE MemoryWrite(ThreadState *thr, uptr pc,
721 uptr addr, int kAccessSizeLog) {
722 MemoryAccess(thr, pc, addr, kAccessSizeLog, true, false);
725 void ALWAYS_INLINE MemoryReadAtomic(ThreadState *thr, uptr pc,
726 uptr addr, int kAccessSizeLog) {
727 MemoryAccess(thr, pc, addr, kAccessSizeLog, false, true);
730 void ALWAYS_INLINE MemoryWriteAtomic(ThreadState *thr, uptr pc,
731 uptr addr, int kAccessSizeLog) {
732 MemoryAccess(thr, pc, addr, kAccessSizeLog, true, true);
735 void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size);
736 void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size);
737 void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size);
739 void ThreadIgnoreBegin(ThreadState *thr, uptr pc, bool save_stack = true);
740 void ThreadIgnoreEnd(ThreadState *thr, uptr pc);
741 void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc, bool save_stack = true);
742 void ThreadIgnoreSyncEnd(ThreadState *thr, uptr pc);
744 void FuncEntry(ThreadState *thr, uptr pc);
745 void FuncExit(ThreadState *thr);
747 int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached);
748 void ThreadStart(ThreadState *thr, int tid, tid_t os_id, bool workerthread);
749 void ThreadFinish(ThreadState *thr);
750 int ThreadTid(ThreadState *thr, uptr pc, uptr uid);
751 void ThreadJoin(ThreadState *thr, uptr pc, int tid);
752 void ThreadDetach(ThreadState *thr, uptr pc, int tid);
753 void ThreadFinalize(ThreadState *thr);
754 void ThreadSetName(ThreadState *thr, const char *name);
755 int ThreadCount(ThreadState *thr);
756 void ProcessPendingSignals(ThreadState *thr);
758 Processor *ProcCreate();
759 void ProcDestroy(Processor *proc);
760 void ProcWire(Processor *proc, ThreadState *thr);
761 void ProcUnwire(Processor *proc, ThreadState *thr);
763 // Note: the parameter is called flagz, because flags is already taken
764 // by the global function that returns flags.
765 void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
766 void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
767 void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
768 void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0,
770 int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
771 void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
772 void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
773 void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr);
774 void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr);
775 void MutexRepair(ThreadState *thr, uptr pc, uptr addr); // call on EOWNERDEAD
776 void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr);
778 void Acquire(ThreadState *thr, uptr pc, uptr addr);
779 // AcquireGlobal synchronizes the current thread with all other threads.
780 // In terms of happens-before relation, it draws a HB edge from all threads
781 // (where they happen to execute right now) to the current thread. We use it to
782 // handle Go finalizers. Namely, finalizer goroutine executes AcquireGlobal
783 // right before executing finalizers. This provides a coarse, but simple
784 // approximation of the actual required synchronization.
785 void AcquireGlobal(ThreadState *thr, uptr pc);
786 void Release(ThreadState *thr, uptr pc, uptr addr);
787 void ReleaseStore(ThreadState *thr, uptr pc, uptr addr);
788 void AfterSleep(ThreadState *thr, uptr pc);
789 void AcquireImpl(ThreadState *thr, uptr pc, SyncClock *c);
790 void ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c);
791 void ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c);
792 void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c);
794 // The hacky call uses custom calling convention and an assembly thunk.
795 // It is considerably faster that a normal call for the caller
796 // if it is not executed (it is intended for slow paths from hot functions).
797 // The trick is that the call preserves all registers and the compiler
798 // does not treat it as a call.
799 // If it does not work for you, use normal call.
800 #if !SANITIZER_DEBUG && defined(__x86_64__) && !SANITIZER_MAC
801 // The caller may not create the stack frame for itself at all,
802 // so we create a reserve stack frame for it (1024b must be enough).
803 #define HACKY_CALL(f) \
804 __asm__ __volatile__("sub $1024, %%rsp;" \
805 CFI_INL_ADJUST_CFA_OFFSET(1024) \
806 ".hidden " #f "_thunk;" \
807 "call " #f "_thunk;" \
808 "add $1024, %%rsp;" \
809 CFI_INL_ADJUST_CFA_OFFSET(-1024) \
812 #define HACKY_CALL(f) f()
815 void TraceSwitch(ThreadState *thr);
816 uptr TraceTopPC(ThreadState *thr);
819 Trace *ThreadTrace(int tid);
821 extern "C" void __tsan_trace_switch();
822 void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs,
823 EventType typ, u64 addr) {
824 if (!kCollectHistory)
826 DCHECK_GE((int)typ, 0);
827 DCHECK_LE((int)typ, 7);
828 DCHECK_EQ(GetLsb(addr, 61), addr);
829 StatInc(thr, StatEvents);
830 u64 pos = fs.GetTracePos();
831 if (UNLIKELY((pos % kTracePartSize) == 0)) {
833 HACKY_CALL(__tsan_trace_switch);
838 Event *trace = (Event*)GetThreadTrace(fs.tid());
839 Event *evp = &trace[pos];
840 Event ev = (u64)addr | ((u64)typ << 61);
845 uptr ALWAYS_INLINE HeapEnd() {
846 return HeapMemEnd() + PrimaryAllocator::AdditionalSize();
850 } // namespace __tsan