sys/compat/linuxkpi/common/include/asm/atomic.h

   1 /*-
   2  * Copyright (c) 2010 Isilon Systems, Inc.
   3  * Copyright (c) 2010 iX Systems, Inc.
   4  * Copyright (c) 2010 Panasas, Inc.
   5  * Copyright (c) 2013-2018 Mellanox Technologies, Ltd.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice unmodified, this list of conditions, and the following
  13  *    disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  19  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  20  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  21  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  22  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  23  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  27  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28  *
  29  * $FreeBSD$
  30  */
  31
  32 #ifndef _ASM_ATOMIC_H_
  33 #define _ASM_ATOMIC_H_
  34
  35 #include <linux/compiler.h>
  36 #include <sys/types.h>
  37 #include <machine/atomic.h>
  38
  39 #define ATOMIC_INIT(x)  { .counter = (x) }
  40
  41 typedef struct {
  42         volatile int counter;
  43 } atomic_t;
  44
  45 /*------------------------------------------------------------------------*
  46  *      32-bit atomic operations
  47  *------------------------------------------------------------------------*/
  48
  49 #define atomic_add(i, v)                atomic_add_return((i), (v))
  50 #define atomic_sub(i, v)                atomic_sub_return((i), (v))
  51 #define atomic_inc_return(v)            atomic_add_return(1, (v))
  52 #define atomic_add_negative(i, v)       (atomic_add_return((i), (v)) < 0)
  53 #define atomic_add_and_test(i, v)       (atomic_add_return((i), (v)) == 0)
  54 #define atomic_sub_and_test(i, v)       (atomic_sub_return((i), (v)) == 0)
  55 #define atomic_dec_and_test(v)          (atomic_sub_return(1, (v)) == 0)
  56 #define atomic_inc_and_test(v)          (atomic_add_return(1, (v)) == 0)
  57 #define atomic_dec_return(v)            atomic_sub_return(1, (v))
  58 #define atomic_inc_not_zero(v)          atomic_add_unless((v), 1, 0)
  59
  60 static inline int
  61 atomic_add_return(int i, atomic_t *v)
  62 {
  63         return i + atomic_fetchadd_int(&v->counter, i);
  64 }
  65
  66 static inline int
  67 atomic_sub_return(int i, atomic_t *v)
  68 {
  69         return atomic_fetchadd_int(&v->counter, -i) - i;
  70 }
  71
  72 static inline void
  73 atomic_set(atomic_t *v, int i)
  74 {
  75         WRITE_ONCE(v->counter, i);
  76 }
  77
  78 static inline void
  79 atomic_set_release(atomic_t *v, int i)
  80 {
  81         atomic_store_rel_int(&v->counter, i);
  82 }
  83
  84 static inline void
  85 atomic_set_mask(unsigned int mask, atomic_t *v)
  86 {
  87         atomic_set_int(&v->counter, mask);
  88 }
  89
  90 static inline int
  91 atomic_read(const atomic_t *v)
  92 {
  93         return READ_ONCE(v->counter);
  94 }
  95
  96 static inline int
  97 atomic_inc(atomic_t *v)
  98 {
  99         return atomic_fetchadd_int(&v->counter, 1) + 1;
 100 }
 101
 102 static inline int
 103 atomic_dec(atomic_t *v)
 104 {
 105         return atomic_fetchadd_int(&v->counter, -1) - 1;
 106 }
 107
 108 static inline int
 109 atomic_add_unless(atomic_t *v, int a, int u)
 110 {
 111         int c;
 112
 113         for (;;) {
 114                 c = atomic_read(v);
 115                 if (unlikely(c == u))
 116                         break;
 117                 if (likely(atomic_cmpset_int(&v->counter, c, c + a)))
 118                         break;
 119         }
 120         return (c != u);
 121 }
 122
 123 static inline void
 124 atomic_clear_mask(unsigned int mask, atomic_t *v)
 125 {
 126         atomic_clear_int(&v->counter, mask);
 127 }
 128
 129 static inline int
 130 atomic_xchg(atomic_t *v, int i)
 131 {
 132 #if !defined(__mips__)
 133         return (atomic_swap_int(&v->counter, i));
 134 #else
 135         int ret;
 136         for (;;) {
 137                 ret = READ_ONCE(v->counter);
 138                 if (atomic_cmpset_int(&v->counter, ret, i))
 139                         break;
 140         }
 141         return (ret);
 142 #endif
 143 }
 144
 145 static inline int
 146 atomic_cmpxchg(atomic_t *v, int old, int new)
 147 {
 148         int ret = old;
 149
 150         for (;;) {
 151                 if (atomic_cmpset_int(&v->counter, old, new))
 152                         break;
 153                 ret = READ_ONCE(v->counter);
 154                 if (ret != old)
 155                         break;
 156         }
 157         return (ret);
 158 }
 159
 160 #if defined(__amd64__) || defined(__arm64__) || defined(__i386__)
 161 #define LINUXKPI_ATOMIC_8(...) __VA_ARGS__
 162 #define LINUXKPI_ATOMIC_16(...) __VA_ARGS__
 163 #else
 164 #define LINUXKPI_ATOMIC_8(...)
 165 #define LINUXKPI_ATOMIC_16(...)
 166 #endif
 167
 168 #if !(defined(i386) || (defined(__mips__) && !(defined(__mips_n32) ||   \
 169     defined(__mips_n64))) || (defined(__powerpc__) &&                   \
 170     !defined(__powerpc64__)))
 171 #define LINUXKPI_ATOMIC_64(...) __VA_ARGS__
 172 #else
 173 #define LINUXKPI_ATOMIC_64(...)
 174 #endif
 175
 176 #define cmpxchg(ptr, old, new) ({                                       \
 177         union {                                                         \
 178                 __typeof(*(ptr)) val;                                   \
 179                 u8 u8[0];                                               \
 180                 u16 u16[0];                                             \
 181                 u32 u32[0];                                             \
 182                 u64 u64[0];                                             \
 183         } __ret = { .val = (old) }, __new = { .val = (new) };           \
 184                                                                         \
 185         CTASSERT(                                                       \
 186             LINUXKPI_ATOMIC_8(sizeof(__ret.val) == 1 ||)                \
 187             LINUXKPI_ATOMIC_16(sizeof(__ret.val) == 2 ||)               \
 188             LINUXKPI_ATOMIC_64(sizeof(__ret.val) == 8 ||)               \
 189             sizeof(__ret.val) == 4);                                    \
 190                                                                         \
 191         switch (sizeof(__ret.val)) {                                    \
 192         LINUXKPI_ATOMIC_8(                                              \
 193         case 1:                                                         \
 194                 while (!atomic_fcmpset_8((volatile u8 *)(ptr),          \
 195                     __ret.u8, __new.u8[0]) && __ret.val == (old))       \
 196                         ;                                               \
 197                 break;                                                  \
 198         )                                                               \
 199         LINUXKPI_ATOMIC_16(                                             \
 200         case 2:                                                         \
 201                 while (!atomic_fcmpset_16((volatile u16 *)(ptr),        \
 202                     __ret.u16, __new.u16[0]) && __ret.val == (old))     \
 203                         ;                                               \
 204                 break;                                                  \
 205         )                                                               \
 206         case 4:                                                         \
 207                 while (!atomic_fcmpset_32((volatile u32 *)(ptr),        \
 208                     __ret.u32, __new.u32[0]) && __ret.val == (old))     \
 209                         ;                                               \
 210                 break;                                                  \
 211         LINUXKPI_ATOMIC_64(                                             \
 212         case 8:                                                         \
 213                 while (!atomic_fcmpset_64((volatile u64 *)(ptr),        \
 214                     __ret.u64, __new.u64[0]) && __ret.val == (old))     \
 215                         ;                                               \
 216                 break;                                                  \
 217         )                                                               \
 218         }                                                               \
 219         __ret.val;                                                      \
 220 })
 221
 222 #define cmpxchg_relaxed(...)    cmpxchg(__VA_ARGS__)
 223
 224 #define xchg(ptr, new) ({                                               \
 225         union {                                                         \
 226                 __typeof(*(ptr)) val;                                   \
 227                 u8 u8[0];                                               \
 228                 u16 u16[0];                                             \
 229                 u32 u32[0];                                             \
 230                 u64 u64[0];                                             \
 231         } __ret, __new = { .val = (new) };                              \
 232                                                                         \
 233         CTASSERT(                                                       \
 234             LINUXKPI_ATOMIC_8(sizeof(__ret.val) == 1 ||)                \
 235             LINUXKPI_ATOMIC_16(sizeof(__ret.val) == 2 ||)               \
 236             LINUXKPI_ATOMIC_64(sizeof(__ret.val) == 8 ||)               \
 237             sizeof(__ret.val) == 4);                                    \
 238                                                                         \
 239         switch (sizeof(__ret.val)) {                                    \
 240         LINUXKPI_ATOMIC_8(                                              \
 241         case 1:                                                         \
 242                 __ret.val = READ_ONCE(*ptr);                            \
 243                 while (!atomic_fcmpset_8((volatile u8 *)(ptr),          \
 244                     __ret.u8, __new.u8[0]))                             \
 245                         ;                                               \
 246                 break;                                                  \
 247         )                                                               \
 248         LINUXKPI_ATOMIC_16(                                             \
 249         case 2:                                                         \
 250                 __ret.val = READ_ONCE(*ptr);                            \
 251                 while (!atomic_fcmpset_16((volatile u16 *)(ptr),        \
 252                     __ret.u16, __new.u16[0]))                           \
 253                         ;                                               \
 254                 break;                                                  \
 255         )                                                               \
 256         case 4:                                                         \
 257                 __ret.u32[0] = atomic_swap_32((volatile u32 *)(ptr),    \
 258                     __new.u32[0]);                                      \
 259                 break;                                                  \
 260         LINUXKPI_ATOMIC_64(                                             \
 261         case 8:                                                         \
 262                 __ret.u64[0] = atomic_swap_64((volatile u64 *)(ptr),    \
 263                     __new.u64[0]);                                      \
 264                 break;                                                  \
 265         )                                                               \
 266         }                                                               \
 267         __ret.val;                                                      \
 268 })
 269
 270 static inline int
 271 atomic_dec_if_positive(atomic_t *v)
 272 {
 273         int retval;
 274         int old;
 275
 276         old = atomic_read(v);
 277         for (;;) {
 278                 retval = old - 1;
 279                 if (unlikely(retval < 0))
 280                         break;
 281                 if (likely(atomic_fcmpset_int(&v->counter, &old, retval)))
 282                         break;
 283         }
 284         return (retval);
 285 }
 286
 287 #define LINUX_ATOMIC_OP(op, c_op)                               \
 288 static inline void atomic_##op(int i, atomic_t *v)              \
 289 {                                                               \
 290         int c, old;                                             \
 291                                                                 \
 292         c = v->counter;                                         \
 293         while ((old = atomic_cmpxchg(v, c, c c_op i)) != c)     \
 294                 c = old;                                        \
 295 }
 296
 297 #define LINUX_ATOMIC_FETCH_OP(op, c_op)                         \
 298 static inline int atomic_fetch_##op(int i, atomic_t *v)         \
 299 {                                                               \
 300         int c, old;                                             \
 301                                                                 \
 302         c = v->counter;                                         \
 303         while ((old = atomic_cmpxchg(v, c, c c_op i)) != c)     \
 304                 c = old;                                        \
 305                                                                 \
 306         return (c);                                             \
 307 }
 308
 309 LINUX_ATOMIC_OP(or, |)
 310 LINUX_ATOMIC_OP(and, &)
 311 LINUX_ATOMIC_OP(andnot, &~)
 312 LINUX_ATOMIC_OP(xor, ^)
 313
 314 LINUX_ATOMIC_FETCH_OP(or, |)
 315 LINUX_ATOMIC_FETCH_OP(and, &)
 316 LINUX_ATOMIC_FETCH_OP(andnot, &~)
 317 LINUX_ATOMIC_FETCH_OP(xor, ^)
 318
 319 #endif                                  /* _ASM_ATOMIC_H_ */