sys/sparc64/include/atomic.h

   1 /*-
   2  * Copyright (c) 1998 Doug Rabson.
   3  * Copyright (c) 2001 Jake Burkholder.
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  *
  27  *      from: FreeBSD: src/sys/i386/include/atomic.h,v 1.20 2001/02/11
  28  * $FreeBSD$
  29  */
  30
  31 #ifndef _MACHINE_ATOMIC_H_
  32 #define _MACHINE_ATOMIC_H_
  33
  34 #include <machine/cpufunc.h>
  35
  36 #define mb()    __asm__ __volatile__ ("membar #MemIssue": : :"memory")
  37 #define wmb()   mb()
  38 #define rmb()   mb()
  39
  40 /* Userland needs different ASI's. */
  41 #ifdef _KERNEL
  42 #define __ASI_ATOMIC    ASI_N
  43 #else
  44 #define __ASI_ATOMIC    ASI_P
  45 #endif
  46
  47 /*
  48  * Various simple arithmetic on memory which is atomic in the presence
  49  * of interrupts and multiple processors.  See atomic(9) for details.
  50  * Note that efficient hardware support exists only for the 32 and 64
  51  * bit variants; the 8 and 16 bit versions are not provided and should
  52  * not be used in MI code.
  53  *
  54  * This implementation takes advantage of the fact that the sparc64
  55  * cas instruction is both a load and a store.  The loop is often coded
  56  * as follows:
  57  *
  58  *      do {
  59  *              expect = *p;
  60  *              new = expect + 1;
  61  *      } while (cas(p, expect, new) != expect);
  62  *
  63  * which performs an unnnecessary load on each iteration that the cas
  64  * operation fails.  Modified as follows:
  65  *
  66  *      expect = *p;
  67  *      for (;;) {
  68  *              new = expect + 1;
  69  *              result = cas(p, expect, new);
  70  *              if (result == expect)
  71  *                      break;
  72  *              expect = result;
  73  *      }
  74  *
  75  * the return value of cas is used to avoid the extra reload.
  76  *
  77  * We only include a memory barrier in the rel variants as in total store
  78  * order which we use for running the kernel and all of the userland atomic
  79  * loads and stores behave as if the were followed by a membar with a mask
  80  * of #LoadLoad | #LoadStore | #StoreStore.  In order to be also sufficient
  81  * for use of relaxed memory ordering, the atomic_cas() in the acq variants
  82  * additionally would have to be followed by a membar #LoadLoad | #LoadStore.
  83  * Due to the suggested assembly syntax of the membar operands containing a
  84  * # character, they cannot be used in macros.  The cmask and mmask bits thus
  85  * are hard coded in machine/cpufunc.h and used here through macros.
  86  * Hopefully the bit numbers won't change in the future.
  87  */
  88
  89 #define itype(sz)       uint ## sz ## _t
  90
  91 #define atomic_cas_32(p, e, s)  casa((p), (e), (s), __ASI_ATOMIC)
  92 #define atomic_cas_64(p, e, s)  casxa((p), (e), (s), __ASI_ATOMIC)
  93
  94 #define atomic_cas(p, e, s, sz)                                         \
  95         atomic_cas_ ## sz((p), (e), (s))
  96
  97 #define atomic_cas_acq(p, e, s, sz) ({                                  \
  98         itype(sz) v;                                                    \
  99         v = atomic_cas((p), (e), (s), sz);                              \
 100         __asm __volatile("" : : : "memory");                            \
 101         v;                                                              \
 102 })
 103
 104 #define atomic_cas_rel(p, e, s, sz) ({                                  \
 105         itype(sz) v;                                                    \
 106         membar(LoadStore | StoreStore);                                 \
 107         v = atomic_cas((p), (e), (s), sz);                              \
 108         v;                                                              \
 109 })
 110
 111 #define atomic_op(p, op, v, sz) ({                                      \
 112         itype(sz) e, r, s;                                              \
 113         for (e = *(volatile itype(sz) *)(p);; e = r) {                  \
 114                 s = e op (v);                                           \
 115                 r = atomic_cas_ ## sz((p), e, s);                       \
 116                 if (r == e)                                             \
 117                         break;                                          \
 118         }                                                               \
 119         e;                                                              \
 120 })
 121
 122 #define atomic_op_acq(p, op, v, sz) ({                                  \
 123         itype(sz) t;                                                    \
 124         t = atomic_op((p), op, (v), sz);                                \
 125         __asm __volatile("" : : : "memory");                            \
 126         t;                                                              \
 127 })
 128
 129 #define atomic_op_rel(p, op, v, sz) ({                                  \
 130         itype(sz) t;                                                    \
 131         membar(LoadStore | StoreStore);                                 \
 132         t = atomic_op((p), op, (v), sz);                                \
 133         t;                                                              \
 134 })
 135
 136 #define atomic_load(p, sz)                                              \
 137         atomic_cas((p), 0, 0, sz)
 138
 139 #define atomic_load_acq(p, sz) ({                                       \
 140         itype(sz) v;                                                    \
 141         v = atomic_load((p), sz);                                       \
 142         __asm __volatile("" : : : "memory");                            \
 143         v;                                                              \
 144 })
 145
 146 #define atomic_load_clear(p, sz) ({                                     \
 147         itype(sz) e, r;                                                 \
 148         for (e = *(volatile itype(sz) *)(p);; e = r) {                  \
 149                 r = atomic_cas((p), e, 0, sz);                          \
 150                 if (r == e)                                             \
 151                         break;                                          \
 152         }                                                               \
 153         e;                                                              \
 154 })
 155
 156 #define atomic_store(p, v, sz) do {                                     \
 157         itype(sz) e, r;                                                 \
 158         for (e = *(volatile itype(sz) *)(p);; e = r) {                  \
 159                 r = atomic_cas((p), e, (v), sz);                        \
 160                 if (r == e)                                             \
 161                         break;                                          \
 162         }                                                               \
 163 } while (0)
 164
 165 #define atomic_store_rel(p, v, sz) do {                                 \
 166         membar(LoadStore | StoreStore);                                 \
 167         atomic_store((p), (v), sz);                                     \
 168 } while (0)
 169
 170 #define ATOMIC_GEN(name, ptype, vtype, atype, sz)                       \
 171                                                                         \
 172 static __inline vtype                                                   \
 173 atomic_add_ ## name(volatile ptype p, atype v)                          \
 174 {                                                                       \
 175         return ((vtype)atomic_op((p), +, (v), sz));                     \
 176 }                                                                       \
 177 static __inline vtype                                                   \
 178 atomic_add_acq_ ## name(volatile ptype p, atype v)                      \
 179 {                                                                       \
 180         return ((vtype)atomic_op_acq((p), +, (v), sz));                 \
 181 }                                                                       \
 182 static __inline vtype                                                   \
 183 atomic_add_rel_ ## name(volatile ptype p, atype v)                      \
 184 {                                                                       \
 185         return ((vtype)atomic_op_rel((p), +, (v), sz));                 \
 186 }                                                                       \
 187                                                                         \
 188 static __inline vtype                                                   \
 189 atomic_clear_ ## name(volatile ptype p, atype v)                        \
 190 {                                                                       \
 191         return ((vtype)atomic_op((p), &, ~(v), sz));                    \
 192 }                                                                       \
 193 static __inline vtype                                                   \
 194 atomic_clear_acq_ ## name(volatile ptype p, atype v)                    \
 195 {                                                                       \
 196         return ((vtype)atomic_op_acq((p), &, ~(v), sz));                \
 197 }                                                                       \
 198 static __inline vtype                                                   \
 199 atomic_clear_rel_ ## name(volatile ptype p, atype v)                    \
 200 {                                                                       \
 201         return ((vtype)atomic_op_rel((p), &, ~(v), sz));                \
 202 }                                                                       \
 203                                                                         \
 204 static __inline int                                                     \
 205 atomic_cmpset_ ## name(volatile ptype p, vtype e, vtype s)              \
 206 {                                                                       \
 207         return (((vtype)atomic_cas((p), (e), (s), sz)) == (e));         \
 208 }                                                                       \
 209 static __inline int                                                     \
 210 atomic_cmpset_acq_ ## name(volatile ptype p, vtype e, vtype s)          \
 211 {                                                                       \
 212         return (((vtype)atomic_cas_acq((p), (e), (s), sz)) == (e));     \
 213 }                                                                       \
 214 static __inline int                                                     \
 215 atomic_cmpset_rel_ ## name(volatile ptype p, vtype e, vtype s)          \
 216 {                                                                       \
 217         return (((vtype)atomic_cas_rel((p), (e), (s), sz)) == (e));     \
 218 }                                                                       \
 219                                                                         \
 220 static __inline vtype                                                   \
 221 atomic_load_ ## name(volatile ptype p)                                  \
 222 {                                                                       \
 223         return ((vtype)atomic_cas((p), 0, 0, sz));                      \
 224 }                                                                       \
 225 static __inline vtype                                                   \
 226 atomic_load_acq_ ## name(volatile ptype p)                              \
 227 {                                                                       \
 228         return ((vtype)atomic_cas_acq((p), 0, 0, sz));                  \
 229 }                                                                       \
 230                                                                         \
 231 static __inline vtype                                                   \
 232 atomic_readandclear_ ## name(volatile ptype p)                          \
 233 {                                                                       \
 234         return ((vtype)atomic_load_clear((p), sz));                     \
 235 }                                                                       \
 236                                                                         \
 237 static __inline vtype                                                   \
 238 atomic_set_ ## name(volatile ptype p, atype v)                          \
 239 {                                                                       \
 240         return ((vtype)atomic_op((p), |, (v), sz));                     \
 241 }                                                                       \
 242 static __inline vtype                                                   \
 243 atomic_set_acq_ ## name(volatile ptype p, atype v)                      \
 244 {                                                                       \
 245         return ((vtype)atomic_op_acq((p), |, (v), sz));                 \
 246 }                                                                       \
 247 static __inline vtype                                                   \
 248 atomic_set_rel_ ## name(volatile ptype p, atype v)                      \
 249 {                                                                       \
 250         return ((vtype)atomic_op_rel((p), |, (v), sz));                 \
 251 }                                                                       \
 252                                                                         \
 253 static __inline vtype                                                   \
 254 atomic_subtract_ ## name(volatile ptype p, atype v)                     \
 255 {                                                                       \
 256         return ((vtype)atomic_op((p), -, (v), sz));                     \
 257 }                                                                       \
 258 static __inline vtype                                                   \
 259 atomic_subtract_acq_ ## name(volatile ptype p, atype v)                 \
 260 {                                                                       \
 261         return ((vtype)atomic_op_acq((p), -, (v), sz));                 \
 262 }                                                                       \
 263 static __inline vtype                                                   \
 264 atomic_subtract_rel_ ## name(volatile ptype p, atype v)                 \
 265 {                                                                       \
 266         return ((vtype)atomic_op_rel((p), -, (v), sz));                 \
 267 }                                                                       \
 268                                                                         \
 269 static __inline void                                                    \
 270 atomic_store_ ## name(volatile ptype p, vtype v)                        \
 271 {                                                                       \
 272         atomic_store((p), (v), sz);                                     \
 273 }                                                                       \
 274 static __inline void                                                    \
 275 atomic_store_rel_ ## name(volatile ptype p, vtype v)                    \
 276 {                                                                       \
 277         atomic_store_rel((p), (v), sz);                                 \
 278 }
 279
 280 ATOMIC_GEN(int, u_int *, u_int, u_int, 32);
 281 ATOMIC_GEN(32, uint32_t *, uint32_t, uint32_t, 32);
 282
 283 ATOMIC_GEN(long, u_long *, u_long, u_long, 64);
 284 ATOMIC_GEN(64, uint64_t *, uint64_t, uint64_t, 64);
 285
 286 ATOMIC_GEN(ptr, uintptr_t *, uintptr_t, uintptr_t, 64);
 287
 288 #define atomic_fetchadd_int     atomic_add_int
 289 #define atomic_fetchadd_32      atomic_add_32
 290 #define atomic_fetchadd_long    atomic_add_long
 291
 292 #undef ATOMIC_GEN
 293 #undef atomic_cas
 294 #undef atomic_cas_acq
 295 #undef atomic_cas_rel
 296 #undef atomic_op
 297 #undef atomic_op_acq
 298 #undef atomic_op_rel
 299 #undef atomic_load_acq
 300 #undef atomic_store_rel
 301 #undef atomic_load_clear
 302
 303 #endif /* !_MACHINE_ATOMIC_H_ */