sys/sparc64/include/atomic.h

   1 /*-
   2  * Copyright (c) 1998 Doug Rabson.
   3  * Copyright (c) 2001 Jake Burkholder.
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  *
  27  *      from: FreeBSD: src/sys/i386/include/atomic.h,v 1.20 2001/02/11
  28  * $FreeBSD$
  29  */
  30
  31 #ifndef _MACHINE_ATOMIC_H_
  32 #define _MACHINE_ATOMIC_H_
  33
  34 #include <machine/cpufunc.h>
  35
  36 #define mb()    __asm__ __volatile__ ("membar #MemIssue": : :"memory")
  37 #define wmb()   mb()
  38 #define rmb()   mb()
  39
  40 /* Userland needs different ASI's. */
  41 #ifdef _KERNEL
  42 #define __ASI_ATOMIC    ASI_N
  43 #else
  44 #define __ASI_ATOMIC    ASI_P
  45 #endif
  46
  47 /*
  48  * Various simple arithmetic on memory which is atomic in the presence
  49  * of interrupts and multiple processors.  See atomic(9) for details.
  50  * Note that efficient hardware support exists only for the 32 and 64
  51  * bit variants; the 8 and 16 bit versions are not provided and should
  52  * not be used in MI code.
  53  *
  54  * This implementation takes advantage of the fact that the sparc64
  55  * cas instruction is both a load and a store.  The loop is often coded
  56  * as follows:
  57  *
  58  *      do {
  59  *              expect = *p;
  60  *              new = expect + 1;
  61  *      } while (cas(p, expect, new) != expect);
  62  *
  63  * which performs an unnnecessary load on each iteration that the cas
  64  * operation fails.  Modified as follows:
  65  *
  66  *      expect = *p;
  67  *      for (;;) {
  68  *              new = expect + 1;
  69  *              result = cas(p, expect, new);
  70  *              if (result == expect)
  71  *                      break;
  72  *              expect = result;
  73  *      }
  74  *
  75  * the return value of cas is used to avoid the extra reload.
  76  *
  77  * We only include a memory barrier in the rel variants as in total store
  78  * order which we use for running the kernel and all of the userland atomic
  79  * loads and stores behave as if the were followed by a membar with a mask
  80  * of #LoadLoad | #LoadStore | #StoreStore.  In order to be also sufficient
  81  * for use of relaxed memory ordering, the atomic_cas() in the acq variants
  82  * additionally would have to be followed by a membar #LoadLoad | #LoadStore.
  83  * Due to the suggested assembly syntax of the membar operands containing a
  84  * # character, they cannot be used in macros.  The cmask and mmask bits thus
  85  * are hard coded in machine/cpufunc.h and used here through macros.
  86  * Hopefully the bit numbers won't change in the future.
  87  */
  88
  89 #define itype(sz)       uint ## sz ## _t
  90
  91 #define atomic_cas_32(p, e, s)  casa((p), (e), (s), __ASI_ATOMIC)
  92 #define atomic_cas_64(p, e, s)  casxa((p), (e), (s), __ASI_ATOMIC)
  93
  94 #define atomic_cas(p, e, s, sz)                                         \
  95         atomic_cas_ ## sz((p), (e), (s))
  96
  97 #define atomic_cas_acq(p, e, s, sz) ({                                  \
  98         itype(sz) v;                                                    \
  99         v = atomic_cas((p), (e), (s), sz);                              \
 100         __compiler_membar();                                            \
 101         v;                                                              \
 102 })
 103
 104 #define atomic_cas_rel(p, e, s, sz) ({                                  \
 105         itype(sz) v;                                                    \
 106         membar(LoadStore | StoreStore);                                 \
 107         v = atomic_cas((p), (e), (s), sz);                              \
 108         v;                                                              \
 109 })
 110
 111 #define atomic_op(p, op, v, sz) ({                                      \
 112         itype(sz) e, r, s;                                              \
 113         for (e = *(volatile itype(sz) *)(p);; e = r) {                  \
 114                 s = e op (v);                                           \
 115                 r = atomic_cas_ ## sz((p), e, s);                       \
 116                 if (r == e)                                             \
 117                         break;                                          \
 118         }                                                               \
 119         e;                                                              \
 120 })
 121
 122 #define atomic_op_acq(p, op, v, sz) ({                                  \
 123         itype(sz) t;                                                    \
 124         t = atomic_op((p), op, (v), sz);                                \
 125         __compiler_membar();                                            \
 126         t;                                                              \
 127 })
 128
 129 #define atomic_op_rel(p, op, v, sz) ({                                  \
 130         itype(sz) t;                                                    \
 131         membar(LoadStore | StoreStore);                                 \
 132         t = atomic_op((p), op, (v), sz);                                \
 133         t;                                                              \
 134 })
 135
 136 #define atomic_load_acq(p, sz) ({                                       \
 137         itype(sz) v;                                                    \
 138         v = atomic_cas((p), 0, 0, sz);                                  \
 139         __compiler_membar();                                            \
 140         v;                                                              \
 141 })
 142
 143 #define atomic_load_clear(p, sz) ({                                     \
 144         itype(sz) e, r;                                                 \
 145         for (e = *(volatile itype(sz) *)(p);; e = r) {                  \
 146                 r = atomic_cas((p), e, 0, sz);                          \
 147                 if (r == e)                                             \
 148                         break;                                          \
 149         }                                                               \
 150         e;                                                              \
 151 })
 152
 153 #define atomic_store_rel(p, v, sz) do {                                 \
 154         itype(sz) e, r;                                                 \
 155         membar(LoadStore | StoreStore);                                 \
 156         for (e = *(volatile itype(sz) *)(p);; e = r) {                  \
 157                 r = atomic_cas((p), e, (v), sz);                        \
 158                 if (r == e)                                             \
 159                         break;                                          \
 160         }                                                               \
 161 } while (0)
 162
 163 #define ATOMIC_GEN(name, ptype, vtype, atype, sz)                       \
 164                                                                         \
 165 static __inline vtype                                                   \
 166 atomic_add_ ## name(volatile ptype p, atype v)                          \
 167 {                                                                       \
 168         return ((vtype)atomic_op((p), +, (v), sz));                     \
 169 }                                                                       \
 170 static __inline vtype                                                   \
 171 atomic_add_acq_ ## name(volatile ptype p, atype v)                      \
 172 {                                                                       \
 173         return ((vtype)atomic_op_acq((p), +, (v), sz));                 \
 174 }                                                                       \
 175 static __inline vtype                                                   \
 176 atomic_add_rel_ ## name(volatile ptype p, atype v)                      \
 177 {                                                                       \
 178         return ((vtype)atomic_op_rel((p), +, (v), sz));                 \
 179 }                                                                       \
 180                                                                         \
 181 static __inline vtype                                                   \
 182 atomic_clear_ ## name(volatile ptype p, atype v)                        \
 183 {                                                                       \
 184         return ((vtype)atomic_op((p), &, ~(v), sz));                    \
 185 }                                                                       \
 186 static __inline vtype                                                   \
 187 atomic_clear_acq_ ## name(volatile ptype p, atype v)                    \
 188 {                                                                       \
 189         return ((vtype)atomic_op_acq((p), &, ~(v), sz));                \
 190 }                                                                       \
 191 static __inline vtype                                                   \
 192 atomic_clear_rel_ ## name(volatile ptype p, atype v)                    \
 193 {                                                                       \
 194         return ((vtype)atomic_op_rel((p), &, ~(v), sz));                \
 195 }                                                                       \
 196                                                                         \
 197 static __inline int                                                     \
 198 atomic_cmpset_ ## name(volatile ptype p, vtype e, vtype s)              \
 199 {                                                                       \
 200         return (((vtype)atomic_cas((p), (e), (s), sz)) == (e));         \
 201 }                                                                       \
 202 static __inline int                                                     \
 203 atomic_cmpset_acq_ ## name(volatile ptype p, vtype e, vtype s)          \
 204 {                                                                       \
 205         return (((vtype)atomic_cas_acq((p), (e), (s), sz)) == (e));     \
 206 }                                                                       \
 207 static __inline int                                                     \
 208 atomic_cmpset_rel_ ## name(volatile ptype p, vtype e, vtype s)          \
 209 {                                                                       \
 210         return (((vtype)atomic_cas_rel((p), (e), (s), sz)) == (e));     \
 211 }                                                                       \
 212                                                                         \
 213 static __inline vtype                                                   \
 214 atomic_load_ ## name(volatile ptype p)                                  \
 215 {                                                                       \
 216         return ((vtype)atomic_cas((p), 0, 0, sz));                      \
 217 }                                                                       \
 218 static __inline vtype                                                   \
 219 atomic_load_acq_ ## name(volatile ptype p)                              \
 220 {                                                                       \
 221         return ((vtype)atomic_cas_acq((p), 0, 0, sz));                  \
 222 }                                                                       \
 223                                                                         \
 224 static __inline vtype                                                   \
 225 atomic_readandclear_ ## name(volatile ptype p)                          \
 226 {                                                                       \
 227         return ((vtype)atomic_load_clear((p), sz));                     \
 228 }                                                                       \
 229                                                                         \
 230 static __inline vtype                                                   \
 231 atomic_set_ ## name(volatile ptype p, atype v)                          \
 232 {                                                                       \
 233         return ((vtype)atomic_op((p), |, (v), sz));                     \
 234 }                                                                       \
 235 static __inline vtype                                                   \
 236 atomic_set_acq_ ## name(volatile ptype p, atype v)                      \
 237 {                                                                       \
 238         return ((vtype)atomic_op_acq((p), |, (v), sz));                 \
 239 }                                                                       \
 240 static __inline vtype                                                   \
 241 atomic_set_rel_ ## name(volatile ptype p, atype v)                      \
 242 {                                                                       \
 243         return ((vtype)atomic_op_rel((p), |, (v), sz));                 \
 244 }                                                                       \
 245                                                                         \
 246 static __inline vtype                                                   \
 247 atomic_subtract_ ## name(volatile ptype p, atype v)                     \
 248 {                                                                       \
 249         return ((vtype)atomic_op((p), -, (v), sz));                     \
 250 }                                                                       \
 251 static __inline vtype                                                   \
 252 atomic_subtract_acq_ ## name(volatile ptype p, atype v)                 \
 253 {                                                                       \
 254         return ((vtype)atomic_op_acq((p), -, (v), sz));                 \
 255 }                                                                       \
 256 static __inline vtype                                                   \
 257 atomic_subtract_rel_ ## name(volatile ptype p, atype v)                 \
 258 {                                                                       \
 259         return ((vtype)atomic_op_rel((p), -, (v), sz));                 \
 260 }                                                                       \
 261                                                                         \
 262 static __inline void                                                    \
 263 atomic_store_rel_ ## name(volatile ptype p, vtype v)                    \
 264 {                                                                       \
 265         atomic_store_rel((p), (v), sz);                                 \
 266 }
 267
 268 ATOMIC_GEN(int, u_int *, u_int, u_int, 32);
 269 ATOMIC_GEN(32, uint32_t *, uint32_t, uint32_t, 32);
 270
 271 ATOMIC_GEN(long, u_long *, u_long, u_long, 64);
 272 ATOMIC_GEN(64, uint64_t *, uint64_t, uint64_t, 64);
 273
 274 ATOMIC_GEN(ptr, uintptr_t *, uintptr_t, uintptr_t, 64);
 275
 276 #define atomic_fetchadd_int     atomic_add_int
 277 #define atomic_fetchadd_32      atomic_add_32
 278 #define atomic_fetchadd_long    atomic_add_long
 279
 280 #undef ATOMIC_GEN
 281 #undef atomic_cas
 282 #undef atomic_cas_acq
 283 #undef atomic_cas_rel
 284 #undef atomic_op
 285 #undef atomic_op_acq
 286 #undef atomic_op_rel
 287 #undef atomic_load_acq
 288 #undef atomic_store_rel
 289 #undef atomic_load_clear
 290
 291 #endif /* !_MACHINE_ATOMIC_H_ */