sys/sun4v/include/atomic.h

   1 /*-
   2  * Copyright (c) 1998 Doug Rabson.
   3  * Copyright (c) 2001 Jake Burkholder.
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  *
  27  *      from: FreeBSD: src/sys/i386/include/atomic.h,v 1.20 2001/02/11
  28  * $FreeBSD$
  29  */
  30
  31 #ifndef _MACHINE_ATOMIC_H_
  32 #define _MACHINE_ATOMIC_H_
  33
  34 #include <machine/cpufunc.h>
  35
  36 /* Userland needs different ASI's. */
  37 #ifdef _KERNEL
  38 #define __ASI_ATOMIC    ASI_N
  39 #else
  40 #define __ASI_ATOMIC    ASI_P
  41 #endif
  42
  43 /*
  44  * Various simple arithmetic on memory which is atomic in the presence
  45  * of interrupts and multiple processors.  See atomic(9) for details.
  46  * Note that efficient hardware support exists only for the 32 and 64
  47  * bit variants; the 8 and 16 bit versions are not provided and should
  48  * not be used in MI code.
  49  *
  50  * This implementation takes advantage of the fact that the sparc64
  51  * cas instruction is both a load and a store.  The loop is often coded
  52  * as follows:
  53  *
  54  *      do {
  55  *              expect = *p;
  56  *              new = expect + 1;
  57  *      } while (cas(p, expect, new) != expect);
  58  *
  59  * which performs an unnnecessary load on each iteration that the cas
  60  * operation fails.  Modified as follows:
  61  *
  62  *      expect = *p;
  63  *      for (;;) {
  64  *              new = expect + 1;
  65  *              result = cas(p, expect, new);
  66  *              if (result == expect)
  67  *                      break;
  68  *              expect = result;
  69  *      }
  70  *
  71  * the return value of cas is used to avoid the extra reload.
  72  *
  73  * The memory barriers provided by the acq and rel variants are intended
  74  * to be sufficient for use of relaxed memory ordering.  Due to the
  75  * suggested assembly syntax of the membar operands containing a #
  76  * character, they cannot be used in macros.  The cmask and mmask bits
  77  * are hard coded in machine/cpufunc.h and used here through macros.
  78  * Hopefully sun will choose not to change the bit numbers.
  79  */
  80
  81 #define itype(sz)       uint ## sz ## _t
  82
  83 #define atomic_cas_32(p, e, s)  casa(p, e, s, __ASI_ATOMIC)
  84 #define atomic_cas_64(p, e, s)  casxa(p, e, s, __ASI_ATOMIC)
  85
  86 #define atomic_cas(p, e, s, sz)                                         \
  87         atomic_cas_ ## sz(p, e, s)
  88
  89 #define atomic_cas_acq(p, e, s, sz) ({                                  \
  90         itype(sz) v;                                                    \
  91         v = atomic_cas(p, e, s, sz);                                    \
  92         membar(LoadLoad | LoadStore);                                   \
  93         v;                                                              \
  94 })
  95
  96 #define atomic_cas_rel(p, e, s, sz) ({                                  \
  97         itype(sz) v;                                                    \
  98         membar(LoadStore | StoreStore);                                 \
  99         v = atomic_cas(p, e, s, sz);                                    \
 100         v;                                                              \
 101 })
 102
 103 #define atomic_op(p, op, v, sz) ({                                      \
 104         itype(sz) e, r, s;                                              \
 105         for (e = *(volatile itype(sz) *)p;; e = r) {                    \
 106                 s = e op v;                                             \
 107                 r = atomic_cas_ ## sz(p, e, s);                         \
 108                 if (r == e)                                             \
 109                         break;                                          \
 110         }                                                               \
 111         e;                                                              \
 112 })
 113
 114 #define atomic_op_acq(p, op, v, sz) ({                                  \
 115         itype(sz) t;                                                    \
 116         t = atomic_op(p, op, v, sz);                                    \
 117         membar(LoadLoad | LoadStore);                                   \
 118         t;                                                              \
 119 })
 120
 121 #define atomic_op_rel(p, op, v, sz) ({                                  \
 122         itype(sz) t;                                                    \
 123         membar(LoadStore | StoreStore);                                 \
 124         t = atomic_op(p, op, v, sz);                                    \
 125         t;                                                              \
 126 })
 127
 128 #define atomic_load(p, sz)                                              \
 129         atomic_cas(p, 0, 0, sz)
 130
 131 #define atomic_load_acq(p, sz) ({                                       \
 132         itype(sz) v;                                                    \
 133         v = atomic_load(p, sz);                                         \
 134         membar(LoadLoad | LoadStore);                                   \
 135         v;                                                              \
 136 })
 137
 138 #define atomic_load_clear(p, sz) ({                                     \
 139         itype(sz) e, r;                                                 \
 140         for (e = *(volatile itype(sz) *)p;; e = r) {                    \
 141                 r = atomic_cas(p, e, 0, sz);                            \
 142                 if (r == e)                                             \
 143                         break;                                          \
 144         }                                                               \
 145         e;                                                              \
 146 })
 147
 148 #define atomic_store(p, v, sz) do {                                     \
 149         itype(sz) e, r;                                                 \
 150         for (e = *(volatile itype(sz) *)p;; e = r) {                    \
 151                 r = atomic_cas(p, e, v, sz);                            \
 152                 if (r == e)                                             \
 153                         break;                                          \
 154         }                                                               \
 155 } while (0)
 156
 157 #define atomic_store_rel(p, v, sz) do {                                 \
 158         membar(LoadStore | StoreStore);                                 \
 159         atomic_store(p, v, sz);                                         \
 160 } while (0)
 161
 162 #define ATOMIC_GEN(name, ptype, vtype, atype, sz)                       \
 163                                                                         \
 164 static __inline vtype                                                   \
 165 atomic_add_ ## name(volatile ptype p, atype v)                          \
 166 {                                                                       \
 167         return ((vtype)atomic_op(p, +, v, sz));                         \
 168 }                                                                       \
 169 static __inline vtype                                                   \
 170 atomic_add_acq_ ## name(volatile ptype p, atype v)                      \
 171 {                                                                       \
 172         return ((vtype)atomic_op_acq(p, +, v, sz));                     \
 173 }                                                                       \
 174 static __inline vtype                                                   \
 175 atomic_add_rel_ ## name(volatile ptype p, atype v)                      \
 176 {                                                                       \
 177         return ((vtype)atomic_op_rel(p, +, v, sz));                     \
 178 }                                                                       \
 179                                                                         \
 180 static __inline vtype                                                   \
 181 atomic_clear_ ## name(volatile ptype p, atype v)                        \
 182 {                                                                       \
 183         return ((vtype)atomic_op(p, &, ~v, sz));                        \
 184 }                                                                       \
 185 static __inline vtype                                                   \
 186 atomic_clear_acq_ ## name(volatile ptype p, atype v)                    \
 187 {                                                                       \
 188         return ((vtype)atomic_op_acq(p, &, ~v, sz));                    \
 189 }                                                                       \
 190 static __inline vtype                                                   \
 191 atomic_clear_rel_ ## name(volatile ptype p, atype v)                    \
 192 {                                                                       \
 193         return ((vtype)atomic_op_rel(p, &, ~v, sz));                    \
 194 }                                                                       \
 195                                                                         \
 196 static __inline int                                                     \
 197 atomic_cmpset_ ## name(volatile ptype p, vtype e, vtype s)              \
 198 {                                                                       \
 199         return (((vtype)atomic_cas(p, e, s, sz)) == e);                 \
 200 }                                                                       \
 201 static __inline int                                                     \
 202 atomic_cmpset_acq_ ## name(volatile ptype p, vtype e, vtype s)          \
 203 {                                                                       \
 204         return (((vtype)atomic_cas_acq(p, e, s, sz)) == e);             \
 205 }                                                                       \
 206 static __inline int                                                     \
 207 atomic_cmpset_rel_ ## name(volatile ptype p, vtype e, vtype s)          \
 208 {                                                                       \
 209         return (((vtype)atomic_cas_rel(p, e, s, sz)) == e);             \
 210 }                                                                       \
 211                                                                         \
 212 static __inline vtype                                                   \
 213 atomic_load_ ## name(volatile ptype p)                                  \
 214 {                                                                       \
 215         return ((vtype)atomic_cas(p, 0, 0, sz));                        \
 216 }                                                                       \
 217 static __inline vtype                                                   \
 218 atomic_load_acq_ ## name(volatile ptype p)                              \
 219 {                                                                       \
 220         return ((vtype)atomic_cas_acq(p, 0, 0, sz));                    \
 221 }                                                                       \
 222                                                                         \
 223 static __inline vtype                                                   \
 224 atomic_readandclear_ ## name(volatile ptype p)                          \
 225 {                                                                       \
 226         return ((vtype)atomic_load_clear(p, sz));                       \
 227 }                                                                       \
 228                                                                         \
 229 static __inline vtype                                                   \
 230 atomic_set_ ## name(volatile ptype p, atype v)                          \
 231 {                                                                       \
 232         return ((vtype)atomic_op(p, |, v, sz));                         \
 233 }                                                                       \
 234 static __inline vtype                                                   \
 235 atomic_set_acq_ ## name(volatile ptype p, atype v)                      \
 236 {                                                                       \
 237         return ((vtype)atomic_op_acq(p, |, v, sz));                     \
 238 }                                                                       \
 239 static __inline vtype                                                   \
 240 atomic_set_rel_ ## name(volatile ptype p, atype v)                      \
 241 {                                                                       \
 242         return ((vtype)atomic_op_rel(p, |, v, sz));                     \
 243 }                                                                       \
 244                                                                         \
 245 static __inline vtype                                                   \
 246 atomic_subtract_ ## name(volatile ptype p, atype v)                     \
 247 {                                                                       \
 248         return ((vtype)atomic_op(p, -, v, sz));                         \
 249 }                                                                       \
 250 static __inline vtype                                                   \
 251 atomic_subtract_acq_ ## name(volatile ptype p, atype v)                 \
 252 {                                                                       \
 253         return ((vtype)atomic_op_acq(p, -, v, sz));                     \
 254 }                                                                       \
 255 static __inline vtype                                                   \
 256 atomic_subtract_rel_ ## name(volatile ptype p, atype v)                 \
 257 {                                                                       \
 258         return ((vtype)atomic_op_rel(p, -, v, sz));                     \
 259 }                                                                       \
 260                                                                         \
 261 static __inline void                                                    \
 262 atomic_store_ ## name(volatile ptype p, vtype v)                        \
 263 {                                                                       \
 264         atomic_store(p, v, sz);                                         \
 265 }                                                                       \
 266 static __inline void                                                    \
 267 atomic_store_rel_ ## name(volatile ptype p, vtype v)                    \
 268 {                                                                       \
 269         atomic_store_rel(p, v, sz);                                     \
 270 }
 271
 272 ATOMIC_GEN(int, u_int *, u_int, u_int, 32);
 273 ATOMIC_GEN(32, uint32_t *, uint32_t, uint32_t, 32);
 274
 275 ATOMIC_GEN(long, u_long *, u_long, u_long, 64);
 276 ATOMIC_GEN(64, uint64_t *, uint64_t, uint64_t, 64);
 277
 278 ATOMIC_GEN(ptr, uintptr_t *, uintptr_t, uintptr_t, 64);
 279
 280 #define atomic_fetchadd_int     atomic_add_int
 281 #define atomic_fetchadd_32      atomic_add_32
 282 #define atomic_fetchadd_long    atomic_add_long
 283
 284 #undef ATOMIC_GEN
 285 #undef atomic_cas
 286 #undef atomic_cas_acq
 287 #undef atomic_cas_rel
 288 #undef atomic_op
 289 #undef atomic_op_acq
 290 #undef atomic_op_rel
 291 #undef atomic_load_acq
 292 #undef atomic_store_rel
 293 #undef atomic_load_clear
 294
 295 #endif /* !_MACHINE_ATOMIC_H_ */