1 //===-- sanitizer_atomic_msvc.h ---------------------------------*- C++ -*-===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file is a part of ThreadSanitizer/AddressSanitizer runtime.
10 // Not intended for direct inclusion. Include sanitizer_atomic.h.
12 //===----------------------------------------------------------------------===//
14 #ifndef SANITIZER_ATOMIC_MSVC_H
15 #define SANITIZER_ATOMIC_MSVC_H
17 extern "C" void _ReadWriteBarrier();
18 #pragma intrinsic(_ReadWriteBarrier)
19 extern "C" void _mm_mfence();
20 #pragma intrinsic(_mm_mfence)
21 extern "C" void _mm_pause();
22 #pragma intrinsic(_mm_pause)
23 extern "C" char _InterlockedExchange8( // NOLINT
24 char volatile *Addend, char Value); // NOLINT
25 #pragma intrinsic(_InterlockedExchange8)
26 extern "C" short _InterlockedExchange16( // NOLINT
27 short volatile *Addend, short Value); // NOLINT
28 #pragma intrinsic(_InterlockedExchange16)
29 extern "C" long _InterlockedExchange( // NOLINT
30 long volatile *Addend, long Value); // NOLINT
31 #pragma intrinsic(_InterlockedExchange)
32 extern "C" long _InterlockedExchangeAdd( // NOLINT
33 long volatile * Addend, long Value); // NOLINT
34 #pragma intrinsic(_InterlockedExchangeAdd)
35 extern "C" char _InterlockedCompareExchange8( // NOLINT
36 char volatile *Destination, // NOLINT
37 char Exchange, char Comparand); // NOLINT
38 #pragma intrinsic(_InterlockedCompareExchange8)
39 extern "C" short _InterlockedCompareExchange16( // NOLINT
40 short volatile *Destination, // NOLINT
41 short Exchange, short Comparand); // NOLINT
42 #pragma intrinsic(_InterlockedCompareExchange16)
44 long long _InterlockedCompareExchange64( // NOLINT
45 long long volatile *Destination, // NOLINT
46 long long Exchange, long long Comparand); // NOLINT
47 #pragma intrinsic(_InterlockedCompareExchange64)
48 extern "C" void *_InterlockedCompareExchangePointer(
49 void *volatile *Destination,
50 void *Exchange, void *Comparand);
51 #pragma intrinsic(_InterlockedCompareExchangePointer)
53 long __cdecl _InterlockedCompareExchange( // NOLINT
54 long volatile *Destination, // NOLINT
55 long Exchange, long Comparand); // NOLINT
56 #pragma intrinsic(_InterlockedCompareExchange)
59 extern "C" long long _InterlockedExchangeAdd64( // NOLINT
60 long long volatile * Addend, long long Value); // NOLINT
61 #pragma intrinsic(_InterlockedExchangeAdd64)
64 namespace __sanitizer {
66 INLINE void atomic_signal_fence(memory_order) {
70 INLINE void atomic_thread_fence(memory_order) {
74 INLINE void proc_yield(int cnt) {
75 for (int i = 0; i < cnt; i++)
80 INLINE typename T::Type atomic_load(
81 const volatile T *a, memory_order mo) {
82 DCHECK(mo & (memory_order_relaxed | memory_order_consume
83 | memory_order_acquire | memory_order_seq_cst));
84 DCHECK(!((uptr)a % sizeof(*a)));
86 // FIXME(dvyukov): 64-bit load is not atomic on 32-bits.
87 if (mo == memory_order_relaxed) {
90 atomic_signal_fence(memory_order_seq_cst);
92 atomic_signal_fence(memory_order_seq_cst);
98 INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
99 DCHECK(mo & (memory_order_relaxed | memory_order_release
100 | memory_order_seq_cst));
101 DCHECK(!((uptr)a % sizeof(*a)));
102 // FIXME(dvyukov): 64-bit store is not atomic on 32-bits.
103 if (mo == memory_order_relaxed) {
106 atomic_signal_fence(memory_order_seq_cst);
108 atomic_signal_fence(memory_order_seq_cst);
110 if (mo == memory_order_seq_cst)
111 atomic_thread_fence(memory_order_seq_cst);
114 INLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a,
115 u32 v, memory_order mo) {
117 DCHECK(!((uptr)a % sizeof(*a)));
118 return (u32)_InterlockedExchangeAdd(
119 (volatile long*)&a->val_dont_use, (long)v); // NOLINT
122 INLINE uptr atomic_fetch_add(volatile atomic_uintptr_t *a,
123 uptr v, memory_order mo) {
125 DCHECK(!((uptr)a % sizeof(*a)));
127 return (uptr)_InterlockedExchangeAdd64(
128 (volatile long long*)&a->val_dont_use, (long long)v); // NOLINT
130 return (uptr)_InterlockedExchangeAdd(
131 (volatile long*)&a->val_dont_use, (long)v); // NOLINT
135 INLINE u32 atomic_fetch_sub(volatile atomic_uint32_t *a,
136 u32 v, memory_order mo) {
138 DCHECK(!((uptr)a % sizeof(*a)));
139 return (u32)_InterlockedExchangeAdd(
140 (volatile long*)&a->val_dont_use, -(long)v); // NOLINT
143 INLINE uptr atomic_fetch_sub(volatile atomic_uintptr_t *a,
144 uptr v, memory_order mo) {
146 DCHECK(!((uptr)a % sizeof(*a)));
148 return (uptr)_InterlockedExchangeAdd64(
149 (volatile long long*)&a->val_dont_use, -(long long)v); // NOLINT
151 return (uptr)_InterlockedExchangeAdd(
152 (volatile long*)&a->val_dont_use, -(long)v); // NOLINT
156 INLINE u8 atomic_exchange(volatile atomic_uint8_t *a,
157 u8 v, memory_order mo) {
159 DCHECK(!((uptr)a % sizeof(*a)));
160 return (u8)_InterlockedExchange8((volatile char*)&a->val_dont_use, v);
163 INLINE u16 atomic_exchange(volatile atomic_uint16_t *a,
164 u16 v, memory_order mo) {
166 DCHECK(!((uptr)a % sizeof(*a)));
167 return (u16)_InterlockedExchange16((volatile short*)&a->val_dont_use, v);
170 INLINE u32 atomic_exchange(volatile atomic_uint32_t *a,
171 u32 v, memory_order mo) {
173 DCHECK(!((uptr)a % sizeof(*a)));
174 return (u32)_InterlockedExchange((volatile long*)&a->val_dont_use, v);
177 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a,
182 DCHECK(!((uptr)a % sizeof(*a)));
185 u8 prev = (u8)_InterlockedCompareExchange8(
186 (volatile char*)&a->val_dont_use, (char)xchgv, (char)cmpv);
193 lock cmpxchg [ecx], dl
203 INLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a,
208 uptr prev = (uptr)_InterlockedCompareExchangePointer(
209 (void*volatile*)&a->val_dont_use, (void*)xchg, (void*)cmpv);
216 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a,
221 u16 prev = (u16)_InterlockedCompareExchange16(
222 (volatile short*)&a->val_dont_use, (short)xchg, (short)cmpv);
229 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a,
234 u32 prev = (u32)_InterlockedCompareExchange(
235 (volatile long*)&a->val_dont_use, (long)xchg, (long)cmpv);
242 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a,
247 u64 prev = (u64)_InterlockedCompareExchange64(
248 (volatile long long*)&a->val_dont_use, (long long)xchg, (long long)cmpv);
256 INLINE bool atomic_compare_exchange_weak(volatile T *a,
257 typename T::Type *cmp,
258 typename T::Type xchg,
260 return atomic_compare_exchange_strong(a, cmp, xchg, mo);
263 } // namespace __sanitizer
265 #endif // SANITIZER_ATOMIC_CLANG_H