sys/compat/linuxkpi/common/include/asm/atomic.h

   1 /*-
   2  * Copyright (c) 2010 Isilon Systems, Inc.
   3  * Copyright (c) 2010 iX Systems, Inc.
   4  * Copyright (c) 2010 Panasas, Inc.
   5  * Copyright (c) 2013-2017 Mellanox Technologies, Ltd.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice unmodified, this list of conditions, and the following
  13  *    disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  19  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  20  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  21  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  22  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  23  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  27  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28  *
  29  * $FreeBSD$
  30  */
  31
  32 #ifndef _ASM_ATOMIC_H_
  33 #define _ASM_ATOMIC_H_
  34
  35 #include <linux/compiler.h>
  36 #include <sys/types.h>
  37 #include <machine/atomic.h>
  38
  39 #define ATOMIC_INIT(x)  { .counter = (x) }
  40
  41 typedef struct {
  42         volatile int counter;
  43 } atomic_t;
  44
  45 /*------------------------------------------------------------------------*
  46  *      32-bit atomic operations
  47  *------------------------------------------------------------------------*/
  48
  49 #define atomic_add(i, v)                atomic_add_return((i), (v))
  50 #define atomic_sub(i, v)                atomic_sub_return((i), (v))
  51 #define atomic_inc_return(v)            atomic_add_return(1, (v))
  52 #define atomic_add_negative(i, v)       (atomic_add_return((i), (v)) < 0)
  53 #define atomic_add_and_test(i, v)       (atomic_add_return((i), (v)) == 0)
  54 #define atomic_sub_and_test(i, v)       (atomic_sub_return((i), (v)) == 0)
  55 #define atomic_dec_and_test(v)          (atomic_sub_return(1, (v)) == 0)
  56 #define atomic_inc_and_test(v)          (atomic_add_return(1, (v)) == 0)
  57 #define atomic_dec_return(v)            atomic_sub_return(1, (v))
  58 #define atomic_inc_not_zero(v)          atomic_add_unless((v), 1, 0)
  59
  60 static inline int
  61 atomic_add_return(int i, atomic_t *v)
  62 {
  63         return i + atomic_fetchadd_int(&v->counter, i);
  64 }
  65
  66 static inline int
  67 atomic_sub_return(int i, atomic_t *v)
  68 {
  69         return atomic_fetchadd_int(&v->counter, -i) - i;
  70 }
  71
  72 static inline void
  73 atomic_set(atomic_t *v, int i)
  74 {
  75         WRITE_ONCE(v->counter, i);
  76 }
  77
  78 static inline void
  79 atomic_set_release(atomic_t *v, int i)
  80 {
  81         atomic_store_rel_int(&v->counter, i);
  82 }
  83
  84 static inline void
  85 atomic_set_mask(unsigned int mask, atomic_t *v)
  86 {
  87         atomic_set_int(&v->counter, mask);
  88 }
  89
  90 static inline int
  91 atomic_read(const atomic_t *v)
  92 {
  93         return READ_ONCE(v->counter);
  94 }
  95
  96 static inline int
  97 atomic_inc(atomic_t *v)
  98 {
  99         return atomic_fetchadd_int(&v->counter, 1) + 1;
 100 }
 101
 102 static inline int
 103 atomic_dec(atomic_t *v)
 104 {
 105         return atomic_fetchadd_int(&v->counter, -1) - 1;
 106 }
 107
 108 static inline int
 109 atomic_add_unless(atomic_t *v, int a, int u)
 110 {
 111         int c;
 112
 113         for (;;) {
 114                 c = atomic_read(v);
 115                 if (unlikely(c == u))
 116                         break;
 117                 if (likely(atomic_cmpset_int(&v->counter, c, c + a)))
 118                         break;
 119         }
 120         return (c != u);
 121 }
 122
 123 static inline void
 124 atomic_clear_mask(unsigned int mask, atomic_t *v)
 125 {
 126         atomic_clear_int(&v->counter, mask);
 127 }
 128
 129 static inline int
 130 atomic_xchg(atomic_t *v, int i)
 131 {
 132 #if defined(__i386__) || defined(__amd64__) || \
 133     defined(__arm__) || defined(__aarch64__) || \
 134     defined(__powerpc__)
 135         return (atomic_swap_int(&v->counter, i));
 136 #else
 137         int ret;
 138         for (;;) {
 139                 ret = READ_ONCE(v->counter);
 140                 if (atomic_cmpset_int(&v->counter, ret, i))
 141                         break;
 142         }
 143         return (ret);
 144 #endif
 145 }
 146
 147 static inline int
 148 atomic_cmpxchg(atomic_t *v, int old, int new)
 149 {
 150         int ret = old;
 151
 152         for (;;) {
 153                 if (atomic_cmpset_int(&v->counter, old, new))
 154                         break;
 155                 ret = READ_ONCE(v->counter);
 156                 if (ret != old)
 157                         break;
 158         }
 159         return (ret);
 160 }
 161
 162 #define cmpxchg(ptr, old, new) ({                               \
 163         __typeof(*(ptr)) __ret;                                 \
 164                                                                 \
 165         CTASSERT(sizeof(__ret) == 1 || sizeof(__ret) == 2 ||    \
 166             sizeof(__ret) == 4 || sizeof(__ret) == 8);          \
 167                                                                 \
 168         __ret = (old);                                          \
 169         switch (sizeof(__ret)) {                                \
 170         case 1:                                                 \
 171                 while (!atomic_fcmpset_8((volatile int8_t *)(ptr), \
 172                     (int8_t *)&__ret, (new)) && __ret == (old)) \
 173                         ;                                       \
 174                 break;                                          \
 175         case 2:                                                 \
 176                 while (!atomic_fcmpset_16((volatile int16_t *)(ptr), \
 177                     (int16_t *)&__ret, (new)) && __ret == (old)) \
 178                         ;                                       \
 179                 break;                                          \
 180         case 4:                                                 \
 181                 while (!atomic_fcmpset_32((volatile int32_t *)(ptr), \
 182                     (int32_t *)&__ret, (new)) && __ret == (old)) \
 183                         ;                                       \
 184                 break;                                          \
 185         case 8:                                                 \
 186                 while (!atomic_fcmpset_64((volatile int64_t *)(ptr), \
 187                     (int64_t *)&__ret, (new)) && __ret == (old)) \
 188                         ;                                       \
 189                 break;                                          \
 190         }                                                       \
 191         __ret;                                                  \
 192 })
 193
 194 #define cmpxchg_relaxed(...)    cmpxchg(__VA_ARGS__)
 195
 196 #define xchg(ptr, v) ({                                         \
 197         __typeof(*(ptr)) __ret;                                 \
 198                                                                 \
 199         __ret = *(ptr);                                         \
 200         *(ptr) = v;                                             \
 201         __ret;                                                  \
 202 })
 203
 204 #define LINUX_ATOMIC_OP(op, c_op)                               \
 205 static inline void atomic_##op(int i, atomic_t *v)              \
 206 {                                                               \
 207         int c, old;                                             \
 208                                                                 \
 209         c = v->counter;                                         \
 210         while ((old = atomic_cmpxchg(v, c, c c_op i)) != c)     \
 211                 c = old;                                        \
 212 }
 213
 214 #define LINUX_ATOMIC_FETCH_OP(op, c_op)                         \
 215 static inline int atomic_fetch_##op(int i, atomic_t *v)         \
 216 {                                                               \
 217         int c, old;                                             \
 218                                                                 \
 219         c = v->counter;                                         \
 220         while ((old = atomic_cmpxchg(v, c, c c_op i)) != c)     \
 221                 c = old;                                        \
 222                                                                 \
 223         return (c);                                             \
 224 }
 225
 226 LINUX_ATOMIC_OP(or, |)
 227 LINUX_ATOMIC_OP(and, &)
 228 LINUX_ATOMIC_OP(andnot, &~)
 229 LINUX_ATOMIC_OP(xor, ^)
 230
 231 LINUX_ATOMIC_FETCH_OP(or, |)
 232 LINUX_ATOMIC_FETCH_OP(and, &)
 233 LINUX_ATOMIC_FETCH_OP(andnot, &~)
 234 LINUX_ATOMIC_FETCH_OP(xor, ^)
 235
 236 #endif                                  /* _ASM_ATOMIC_H_ */