sys/compat/linuxkpi/common/include/asm/atomic.h

   1 /*-
   2  * Copyright (c) 2010 Isilon Systems, Inc.
   3  * Copyright (c) 2010 iX Systems, Inc.
   4  * Copyright (c) 2010 Panasas, Inc.
   5  * Copyright (c) 2013-2018 Mellanox Technologies, Ltd.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice unmodified, this list of conditions, and the following
  13  *    disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  19  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  20  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  21  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  22  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  23  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  27  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28  *
  29  * $FreeBSD$
  30  */
  31
  32 #ifndef _ASM_ATOMIC_H_
  33 #define _ASM_ATOMIC_H_
  34
  35 #include <linux/compiler.h>
  36 #include <sys/types.h>
  37 #include <machine/atomic.h>
  38 #define ATOMIC_INIT(x)  { .counter = (x) }
  39
  40 typedef struct {
  41         volatile int counter;
  42 } atomic_t;
  43
  44 /*------------------------------------------------------------------------*
  45  *      32-bit atomic operations
  46  *------------------------------------------------------------------------*/
  47
  48 #define atomic_add(i, v)                atomic_add_return((i), (v))
  49 #define atomic_sub(i, v)                atomic_sub_return((i), (v))
  50 #define atomic_inc_return(v)            atomic_add_return(1, (v))
  51 #define atomic_add_negative(i, v)       (atomic_add_return((i), (v)) < 0)
  52 #define atomic_add_and_test(i, v)       (atomic_add_return((i), (v)) == 0)
  53 #define atomic_sub_and_test(i, v)       (atomic_sub_return((i), (v)) == 0)
  54 #define atomic_dec_and_test(v)          (atomic_sub_return(1, (v)) == 0)
  55 #define atomic_inc_and_test(v)          (atomic_add_return(1, (v)) == 0)
  56 #define atomic_dec_return(v)            atomic_sub_return(1, (v))
  57 #define atomic_inc_not_zero(v)          atomic_add_unless((v), 1, 0)
  58
  59 static inline int
  60 atomic_add_return(int i, atomic_t *v)
  61 {
  62         return i + atomic_fetchadd_int(&v->counter, i);
  63 }
  64
  65 static inline int
  66 atomic_sub_return(int i, atomic_t *v)
  67 {
  68         return atomic_fetchadd_int(&v->counter, -i) - i;
  69 }
  70
  71 static inline void
  72 atomic_set(atomic_t *v, int i)
  73 {
  74         WRITE_ONCE(v->counter, i);
  75 }
  76
  77 static inline void
  78 atomic_set_release(atomic_t *v, int i)
  79 {
  80         atomic_store_rel_int(&v->counter, i);
  81 }
  82
  83 static inline void
  84 atomic_set_mask(unsigned int mask, atomic_t *v)
  85 {
  86         atomic_set_int(&v->counter, mask);
  87 }
  88
  89 static inline int
  90 atomic_read(const atomic_t *v)
  91 {
  92         return READ_ONCE(v->counter);
  93 }
  94
  95 static inline int
  96 atomic_inc(atomic_t *v)
  97 {
  98         return atomic_fetchadd_int(&v->counter, 1) + 1;
  99 }
 100
 101 static inline int
 102 atomic_dec(atomic_t *v)
 103 {
 104         return atomic_fetchadd_int(&v->counter, -1) - 1;
 105 }
 106
 107 static inline int
 108 atomic_add_unless(atomic_t *v, int a, int u)
 109 {
 110         int c = atomic_read(v);
 111
 112         for (;;) {
 113                 if (unlikely(c == u))
 114                         break;
 115                 if (likely(atomic_fcmpset_int(&v->counter, &c, c + a)))
 116                         break;
 117         }
 118         return (c != u);
 119 }
 120
 121 static inline int
 122 atomic_fetch_add_unless(atomic_t *v, int a, int u)
 123 {
 124         int c = atomic_read(v);
 125
 126         for (;;) {
 127                 if (unlikely(c == u))
 128                         break;
 129                 if (likely(atomic_fcmpset_int(&v->counter, &c, c + a)))
 130                         break;
 131         }
 132         return (c);
 133 }
 134
 135 static inline void
 136 atomic_clear_mask(unsigned int mask, atomic_t *v)
 137 {
 138         atomic_clear_int(&v->counter, mask);
 139 }
 140
 141 static inline int
 142 atomic_xchg(atomic_t *v, int i)
 143 {
 144         return (atomic_swap_int(&v->counter, i));
 145 }
 146
 147 static inline int
 148 atomic_cmpxchg(atomic_t *v, int old, int new)
 149 {
 150         int ret = old;
 151
 152         for (;;) {
 153                 if (atomic_fcmpset_int(&v->counter, &ret, new))
 154                         break;
 155                 if (ret != old)
 156                         break;
 157         }
 158         return (ret);
 159 }
 160
 161 #if defined(__amd64__) || defined(__arm64__) || defined(__i386__)
 162 #define LINUXKPI_ATOMIC_8(...) __VA_ARGS__
 163 #define LINUXKPI_ATOMIC_16(...) __VA_ARGS__
 164 #else
 165 #define LINUXKPI_ATOMIC_8(...)
 166 #define LINUXKPI_ATOMIC_16(...)
 167 #endif
 168
 169 #if !(defined(i386) || (defined(__mips__) && !(defined(__mips_n32) ||   \
 170     defined(__mips_n64))) || (defined(__powerpc__) &&                   \
 171     !defined(__powerpc64__)))
 172 #define LINUXKPI_ATOMIC_64(...) __VA_ARGS__
 173 #else
 174 #define LINUXKPI_ATOMIC_64(...)
 175 #endif
 176
 177 #define cmpxchg(ptr, old, new) ({                                       \
 178         union {                                                         \
 179                 __typeof(*(ptr)) val;                                   \
 180                 u8 u8[0];                                               \
 181                 u16 u16[0];                                             \
 182                 u32 u32[0];                                             \
 183                 u64 u64[0];                                             \
 184         } __ret = { .val = (old) }, __new = { .val = (new) };           \
 185                                                                         \
 186         CTASSERT(                                                       \
 187             LINUXKPI_ATOMIC_8(sizeof(__ret.val) == 1 ||)                \
 188             LINUXKPI_ATOMIC_16(sizeof(__ret.val) == 2 ||)               \
 189             LINUXKPI_ATOMIC_64(sizeof(__ret.val) == 8 ||)               \
 190             sizeof(__ret.val) == 4);                                    \
 191                                                                         \
 192         switch (sizeof(__ret.val)) {                                    \
 193         LINUXKPI_ATOMIC_8(                                              \
 194         case 1:                                                         \
 195                 while (!atomic_fcmpset_8((volatile u8 *)(ptr),          \
 196                     __ret.u8, __new.u8[0]) && __ret.val == (old))       \
 197                         ;                                               \
 198                 break;                                                  \
 199         )                                                               \
 200         LINUXKPI_ATOMIC_16(                                             \
 201         case 2:                                                         \
 202                 while (!atomic_fcmpset_16((volatile u16 *)(ptr),        \
 203                     __ret.u16, __new.u16[0]) && __ret.val == (old))     \
 204                         ;                                               \
 205                 break;                                                  \
 206         )                                                               \
 207         case 4:                                                         \
 208                 while (!atomic_fcmpset_32((volatile u32 *)(ptr),        \
 209                     __ret.u32, __new.u32[0]) && __ret.val == (old))     \
 210                         ;                                               \
 211                 break;                                                  \
 212         LINUXKPI_ATOMIC_64(                                             \
 213         case 8:                                                         \
 214                 while (!atomic_fcmpset_64((volatile u64 *)(ptr),        \
 215                     __ret.u64, __new.u64[0]) && __ret.val == (old))     \
 216                         ;                                               \
 217                 break;                                                  \
 218         )                                                               \
 219         }                                                               \
 220         __ret.val;                                                      \
 221 })
 222
 223 #define cmpxchg_relaxed(...)    cmpxchg(__VA_ARGS__)
 224
 225 #define xchg(ptr, new) ({                                               \
 226         union {                                                         \
 227                 __typeof(*(ptr)) val;                                   \
 228                 u8 u8[0];                                               \
 229                 u16 u16[0];                                             \
 230                 u32 u32[0];                                             \
 231                 u64 u64[0];                                             \
 232         } __ret, __new = { .val = (new) };                              \
 233                                                                         \
 234         CTASSERT(                                                       \
 235             LINUXKPI_ATOMIC_8(sizeof(__ret.val) == 1 ||)                \
 236             LINUXKPI_ATOMIC_16(sizeof(__ret.val) == 2 ||)               \
 237             LINUXKPI_ATOMIC_64(sizeof(__ret.val) == 8 ||)               \
 238             sizeof(__ret.val) == 4);                                    \
 239                                                                         \
 240         switch (sizeof(__ret.val)) {                                    \
 241         LINUXKPI_ATOMIC_8(                                              \
 242         case 1:                                                         \
 243                 __ret.val = READ_ONCE(*ptr);                            \
 244                 while (!atomic_fcmpset_8((volatile u8 *)(ptr),          \
 245                     __ret.u8, __new.u8[0]))                             \
 246                         ;                                               \
 247                 break;                                                  \
 248         )                                                               \
 249         LINUXKPI_ATOMIC_16(                                             \
 250         case 2:                                                         \
 251                 __ret.val = READ_ONCE(*ptr);                            \
 252                 while (!atomic_fcmpset_16((volatile u16 *)(ptr),        \
 253                     __ret.u16, __new.u16[0]))                           \
 254                         ;                                               \
 255                 break;                                                  \
 256         )                                                               \
 257         case 4:                                                         \
 258                 __ret.u32[0] = atomic_swap_32((volatile u32 *)(ptr),    \
 259                     __new.u32[0]);                                      \
 260                 break;                                                  \
 261         LINUXKPI_ATOMIC_64(                                             \
 262         case 8:                                                         \
 263                 __ret.u64[0] = atomic_swap_64((volatile u64 *)(ptr),    \
 264                     __new.u64[0]);                                      \
 265                 break;                                                  \
 266         )                                                               \
 267         }                                                               \
 268         __ret.val;                                                      \
 269 })
 270
 271 static inline int
 272 atomic_dec_if_positive(atomic_t *v)
 273 {
 274         int retval;
 275         int old;
 276
 277         old = atomic_read(v);
 278         for (;;) {
 279                 retval = old - 1;
 280                 if (unlikely(retval < 0))
 281                         break;
 282                 if (likely(atomic_fcmpset_int(&v->counter, &old, retval)))
 283                         break;
 284         }
 285         return (retval);
 286 }
 287
 288 #define LINUX_ATOMIC_OP(op, c_op)                               \
 289 static inline void atomic_##op(int i, atomic_t *v)              \
 290 {                                                               \
 291         int c, old;                                             \
 292                                                                 \
 293         c = v->counter;                                         \
 294         while ((old = atomic_cmpxchg(v, c, c c_op i)) != c)     \
 295                 c = old;                                        \
 296 }
 297
 298 #define LINUX_ATOMIC_FETCH_OP(op, c_op)                         \
 299 static inline int atomic_fetch_##op(int i, atomic_t *v)         \
 300 {                                                               \
 301         int c, old;                                             \
 302                                                                 \
 303         c = v->counter;                                         \
 304         while ((old = atomic_cmpxchg(v, c, c c_op i)) != c)     \
 305                 c = old;                                        \
 306                                                                 \
 307         return (c);                                             \
 308 }
 309
 310 LINUX_ATOMIC_OP(or, |)
 311 LINUX_ATOMIC_OP(and, &)
 312 LINUX_ATOMIC_OP(andnot, &~)
 313 LINUX_ATOMIC_OP(xor, ^)
 314
 315 LINUX_ATOMIC_FETCH_OP(or, |)
 316 LINUX_ATOMIC_FETCH_OP(and, &)
 317 LINUX_ATOMIC_FETCH_OP(andnot, &~)
 318 LINUX_ATOMIC_FETCH_OP(xor, ^)
 319
 320 #endif                                  /* _ASM_ATOMIC_H_ */