contrib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_msvc.h

   1 //===-- sanitizer_atomic_msvc.h ---------------------------------*- C++ -*-===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // This file is a part of ThreadSanitizer/AddressSanitizer runtime.
  11 // Not intended for direct inclusion. Include sanitizer_atomic.h.
  12 //
  13 //===----------------------------------------------------------------------===//
  14
  15 #ifndef SANITIZER_ATOMIC_MSVC_H
  16 #define SANITIZER_ATOMIC_MSVC_H
  17
  18 extern "C" void _ReadWriteBarrier();
  19 #pragma intrinsic(_ReadWriteBarrier)
  20 extern "C" void _mm_mfence();
  21 #pragma intrinsic(_mm_mfence)
  22 extern "C" void _mm_pause();
  23 #pragma intrinsic(_mm_pause)
  24 extern "C" char _InterlockedExchange8(   // NOLINT
  25     char volatile *Addend, char Value);  // NOLINT
  26 #pragma intrinsic(_InterlockedExchange8)
  27 extern "C" short _InterlockedExchange16(   // NOLINT
  28     short volatile *Addend, short Value);  // NOLINT
  29 #pragma intrinsic(_InterlockedExchange16)
  30 extern "C" long _InterlockedExchange(    // NOLINT
  31     long volatile *Addend, long Value);  // NOLINT
  32 #pragma intrinsic(_InterlockedExchange)
  33 extern "C" long _InterlockedExchangeAdd(  // NOLINT
  34     long volatile * Addend, long Value);  // NOLINT
  35 #pragma intrinsic(_InterlockedExchangeAdd)
  36 extern "C" char _InterlockedCompareExchange8(  // NOLINT
  37     char volatile *Destination,                // NOLINT
  38     char Exchange, char Comparand);            // NOLINT
  39 #pragma intrinsic(_InterlockedCompareExchange8)
  40 extern "C" short _InterlockedCompareExchange16(  // NOLINT
  41     short volatile *Destination,                 // NOLINT
  42     short Exchange, short Comparand);            // NOLINT
  43 #pragma intrinsic(_InterlockedCompareExchange16)
  44 extern "C"
  45 long long _InterlockedCompareExchange64(  // NOLINT
  46     long long volatile *Destination,              // NOLINT
  47     long long Exchange, long long Comparand);     // NOLINT
  48 #pragma intrinsic(_InterlockedCompareExchange64)
  49 extern "C" void *_InterlockedCompareExchangePointer(
  50     void *volatile *Destination,
  51     void *Exchange, void *Comparand);
  52 #pragma intrinsic(_InterlockedCompareExchangePointer)
  53 extern "C"
  54 long __cdecl _InterlockedCompareExchange(  // NOLINT
  55     long volatile *Destination,            // NOLINT
  56     long Exchange, long Comparand);        // NOLINT
  57 #pragma intrinsic(_InterlockedCompareExchange)
  58
  59 #ifdef _WIN64
  60 extern "C" long long _InterlockedExchangeAdd64(     // NOLINT
  61     long long volatile * Addend, long long Value);  // NOLINT
  62 #pragma intrinsic(_InterlockedExchangeAdd64)
  63 #endif
  64
  65 namespace __sanitizer {
  66
  67 INLINE void atomic_signal_fence(memory_order) {
  68   _ReadWriteBarrier();
  69 }
  70
  71 INLINE void atomic_thread_fence(memory_order) {
  72   _mm_mfence();
  73 }
  74
  75 INLINE void proc_yield(int cnt) {
  76   for (int i = 0; i < cnt; i++)
  77     _mm_pause();
  78 }
  79
  80 template<typename T>
  81 INLINE typename T::Type atomic_load(
  82     const volatile T *a, memory_order mo) {
  83   DCHECK(mo & (memory_order_relaxed | memory_order_consume
  84       | memory_order_acquire | memory_order_seq_cst));
  85   DCHECK(!((uptr)a % sizeof(*a)));
  86   typename T::Type v;
  87   // FIXME(dvyukov): 64-bit load is not atomic on 32-bits.
  88   if (mo == memory_order_relaxed) {
  89     v = a->val_dont_use;
  90   } else {
  91     atomic_signal_fence(memory_order_seq_cst);
  92     v = a->val_dont_use;
  93     atomic_signal_fence(memory_order_seq_cst);
  94   }
  95   return v;
  96 }
  97
  98 template<typename T>
  99 INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
 100   DCHECK(mo & (memory_order_relaxed | memory_order_release
 101       | memory_order_seq_cst));
 102   DCHECK(!((uptr)a % sizeof(*a)));
 103   // FIXME(dvyukov): 64-bit store is not atomic on 32-bits.
 104   if (mo == memory_order_relaxed) {
 105     a->val_dont_use = v;
 106   } else {
 107     atomic_signal_fence(memory_order_seq_cst);
 108     a->val_dont_use = v;
 109     atomic_signal_fence(memory_order_seq_cst);
 110   }
 111   if (mo == memory_order_seq_cst)
 112     atomic_thread_fence(memory_order_seq_cst);
 113 }
 114
 115 INLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a,
 116     u32 v, memory_order mo) {
 117   (void)mo;
 118   DCHECK(!((uptr)a % sizeof(*a)));
 119   return (u32)_InterlockedExchangeAdd(
 120       (volatile long*)&a->val_dont_use, (long)v);  // NOLINT
 121 }
 122
 123 INLINE uptr atomic_fetch_add(volatile atomic_uintptr_t *a,
 124     uptr v, memory_order mo) {
 125   (void)mo;
 126   DCHECK(!((uptr)a % sizeof(*a)));
 127 #ifdef _WIN64
 128   return (uptr)_InterlockedExchangeAdd64(
 129       (volatile long long*)&a->val_dont_use, (long long)v);  // NOLINT
 130 #else
 131   return (uptr)_InterlockedExchangeAdd(
 132       (volatile long*)&a->val_dont_use, (long)v);  // NOLINT
 133 #endif
 134 }
 135
 136 INLINE u32 atomic_fetch_sub(volatile atomic_uint32_t *a,
 137     u32 v, memory_order mo) {
 138   (void)mo;
 139   DCHECK(!((uptr)a % sizeof(*a)));
 140   return (u32)_InterlockedExchangeAdd(
 141       (volatile long*)&a->val_dont_use, -(long)v);  // NOLINT
 142 }
 143
 144 INLINE uptr atomic_fetch_sub(volatile atomic_uintptr_t *a,
 145     uptr v, memory_order mo) {
 146   (void)mo;
 147   DCHECK(!((uptr)a % sizeof(*a)));
 148 #ifdef _WIN64
 149   return (uptr)_InterlockedExchangeAdd64(
 150       (volatile long long*)&a->val_dont_use, -(long long)v);  // NOLINT
 151 #else
 152   return (uptr)_InterlockedExchangeAdd(
 153       (volatile long*)&a->val_dont_use, -(long)v);  // NOLINT
 154 #endif
 155 }
 156
 157 INLINE u8 atomic_exchange(volatile atomic_uint8_t *a,
 158     u8 v, memory_order mo) {
 159   (void)mo;
 160   DCHECK(!((uptr)a % sizeof(*a)));
 161   return (u8)_InterlockedExchange8((volatile char*)&a->val_dont_use, v);
 162 }
 163
 164 INLINE u16 atomic_exchange(volatile atomic_uint16_t *a,
 165     u16 v, memory_order mo) {
 166   (void)mo;
 167   DCHECK(!((uptr)a % sizeof(*a)));
 168   return (u16)_InterlockedExchange16((volatile short*)&a->val_dont_use, v);
 169 }
 170
 171 INLINE u32 atomic_exchange(volatile atomic_uint32_t *a,
 172     u32 v, memory_order mo) {
 173   (void)mo;
 174   DCHECK(!((uptr)a % sizeof(*a)));
 175   return (u32)_InterlockedExchange((volatile long*)&a->val_dont_use, v);
 176 }
 177
 178 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a,
 179                                            u8 *cmp,
 180                                            u8 xchgv,
 181                                            memory_order mo) {
 182   (void)mo;
 183   DCHECK(!((uptr)a % sizeof(*a)));
 184   u8 cmpv = *cmp;
 185 #ifdef _WIN64
 186   u8 prev = (u8)_InterlockedCompareExchange8(
 187       (volatile char*)&a->val_dont_use, (char)xchgv, (char)cmpv);
 188 #else
 189   u8 prev;
 190   __asm {
 191     mov al, cmpv
 192     mov ecx, a
 193     mov dl, xchgv
 194     lock cmpxchg [ecx], dl
 195     mov prev, al
 196   }
 197 #endif
 198   if (prev == cmpv)
 199     return true;
 200   *cmp = prev;
 201   return false;
 202 }
 203
 204 INLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a,
 205                                            uptr *cmp,
 206                                            uptr xchg,
 207                                            memory_order mo) {
 208   uptr cmpv = *cmp;
 209   uptr prev = (uptr)_InterlockedCompareExchangePointer(
 210       (void*volatile*)&a->val_dont_use, (void*)xchg, (void*)cmpv);
 211   if (prev == cmpv)
 212     return true;
 213   *cmp = prev;
 214   return false;
 215 }
 216
 217 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a,
 218                                            u16 *cmp,
 219                                            u16 xchg,
 220                                            memory_order mo) {
 221   u16 cmpv = *cmp;
 222   u16 prev = (u16)_InterlockedCompareExchange16(
 223       (volatile short*)&a->val_dont_use, (short)xchg, (short)cmpv);
 224   if (prev == cmpv)
 225     return true;
 226   *cmp = prev;
 227   return false;
 228 }
 229
 230 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a,
 231                                            u32 *cmp,
 232                                            u32 xchg,
 233                                            memory_order mo) {
 234   u32 cmpv = *cmp;
 235   u32 prev = (u32)_InterlockedCompareExchange(
 236       (volatile long*)&a->val_dont_use, (long)xchg, (long)cmpv);
 237   if (prev == cmpv)
 238     return true;
 239   *cmp = prev;
 240   return false;
 241 }
 242
 243 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a,
 244                                            u64 *cmp,
 245                                            u64 xchg,
 246                                            memory_order mo) {
 247   u64 cmpv = *cmp;
 248   u64 prev = (u64)_InterlockedCompareExchange64(
 249       (volatile long long*)&a->val_dont_use, (long long)xchg, (long long)cmpv);
 250   if (prev == cmpv)
 251     return true;
 252   *cmp = prev;
 253   return false;
 254 }
 255
 256 template<typename T>
 257 INLINE bool atomic_compare_exchange_weak(volatile T *a,
 258                                          typename T::Type *cmp,
 259                                          typename T::Type xchg,
 260                                          memory_order mo) {
 261   return atomic_compare_exchange_strong(a, cmp, xchg, mo);
 262 }
 263
 264 }  // namespace __sanitizer
 265
 266 #endif  // SANITIZER_ATOMIC_CLANG_H