sys/sun4v/include/atomic.h

   1 /*-
   2  * Copyright (c) 1998 Doug Rabson.
   3  * Copyright (c) 2001 Jake Burkholder.
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  *
  27  *      from: FreeBSD: src/sys/i386/include/atomic.h,v 1.20 2001/02/11
  28  * $FreeBSD$
  29  */
  30
  31 #ifndef _MACHINE_ATOMIC_H_
  32 #define _MACHINE_ATOMIC_H_
  33
  34 #include <machine/cpufunc.h>
  35
  36 #define mb()    __asm__ __volatile__ ("membar #MemIssue": : :"memory")
  37 #define wmb()   mb()
  38 #define rmb()   mb()
  39
  40 /* Userland needs different ASI's. */
  41 #ifdef _KERNEL
  42 #define __ASI_ATOMIC    ASI_N
  43 #else
  44 #define __ASI_ATOMIC    ASI_P
  45 #endif
  46
  47 /*
  48  * Various simple arithmetic on memory which is atomic in the presence
  49  * of interrupts and multiple processors.  See atomic(9) for details.
  50  * Note that efficient hardware support exists only for the 32 and 64
  51  * bit variants; the 8 and 16 bit versions are not provided and should
  52  * not be used in MI code.
  53  *
  54  * This implementation takes advantage of the fact that the sparc64
  55  * cas instruction is both a load and a store.  The loop is often coded
  56  * as follows:
  57  *
  58  *      do {
  59  *              expect = *p;
  60  *              new = expect + 1;
  61  *      } while (cas(p, expect, new) != expect);
  62  *
  63  * which performs an unnnecessary load on each iteration that the cas
  64  * operation fails.  Modified as follows:
  65  *
  66  *      expect = *p;
  67  *      for (;;) {
  68  *              new = expect + 1;
  69  *              result = cas(p, expect, new);
  70  *              if (result == expect)
  71  *                      break;
  72  *              expect = result;
  73  *      }
  74  *
  75  * the return value of cas is used to avoid the extra reload.
  76  *
  77  * The memory barriers provided by the acq and rel variants are intended
  78  * to be sufficient for use of relaxed memory ordering.  Due to the
  79  * suggested assembly syntax of the membar operands containing a #
  80  * character, they cannot be used in macros.  The cmask and mmask bits
  81  * are hard coded in machine/cpufunc.h and used here through macros.
  82  * Hopefully sun will choose not to change the bit numbers.
  83  */
  84
  85 #define itype(sz)       uint ## sz ## _t
  86
  87 #define atomic_cas_32(p, e, s)  casa(p, e, s, __ASI_ATOMIC)
  88 #define atomic_cas_64(p, e, s)  casxa(p, e, s, __ASI_ATOMIC)
  89
  90 #define atomic_cas(p, e, s, sz)                                         \
  91         atomic_cas_ ## sz(p, e, s)
  92
  93 #define atomic_cas_acq(p, e, s, sz) ({                                  \
  94         itype(sz) v;                                                    \
  95         v = atomic_cas(p, e, s, sz);                                    \
  96         membar(LoadLoad | LoadStore);                                   \
  97         v;                                                              \
  98 })
  99
 100 #define atomic_cas_rel(p, e, s, sz) ({                                  \
 101         itype(sz) v;                                                    \
 102         membar(LoadStore | StoreStore);                                 \
 103         v = atomic_cas(p, e, s, sz);                                    \
 104         v;                                                              \
 105 })
 106
 107 #define atomic_op(p, op, v, sz) ({                                      \
 108         itype(sz) e, r, s;                                              \
 109         for (e = *(volatile itype(sz) *)p;; e = r) {                    \
 110                 s = e op v;                                             \
 111                 r = atomic_cas_ ## sz(p, e, s);                         \
 112                 if (r == e)                                             \
 113                         break;                                          \
 114         }                                                               \
 115         e;                                                              \
 116 })
 117
 118 #define atomic_op_acq(p, op, v, sz) ({                                  \
 119         itype(sz) t;                                                    \
 120         t = atomic_op(p, op, v, sz);                                    \
 121         membar(LoadLoad | LoadStore);                                   \
 122         t;                                                              \
 123 })
 124
 125 #define atomic_op_rel(p, op, v, sz) ({                                  \
 126         itype(sz) t;                                                    \
 127         membar(LoadStore | StoreStore);                                 \
 128         t = atomic_op(p, op, v, sz);                                    \
 129         t;                                                              \
 130 })
 131
 132 #define atomic_load(p, sz)                                              \
 133         atomic_cas(p, 0, 0, sz)
 134
 135 #define atomic_load_acq(p, sz) ({                                       \
 136         itype(sz) v;                                                    \
 137         v = atomic_load(p, sz);                                         \
 138         membar(LoadLoad | LoadStore);                                   \
 139         v;                                                              \
 140 })
 141
 142 #define atomic_load_clear(p, sz) ({                                     \
 143         itype(sz) e, r;                                                 \
 144         for (e = *(volatile itype(sz) *)p;; e = r) {                    \
 145                 r = atomic_cas(p, e, 0, sz);                            \
 146                 if (r == e)                                             \
 147                         break;                                          \
 148         }                                                               \
 149         e;                                                              \
 150 })
 151
 152 #define atomic_store(p, v, sz) do {                                     \
 153         itype(sz) e, r;                                                 \
 154         for (e = *(volatile itype(sz) *)p;; e = r) {                    \
 155                 r = atomic_cas(p, e, v, sz);                            \
 156                 if (r == e)                                             \
 157                         break;                                          \
 158         }                                                               \
 159 } while (0)
 160
 161 #define atomic_store_rel(p, v, sz) do {                                 \
 162         membar(LoadStore | StoreStore);                                 \
 163         atomic_store(p, v, sz);                                         \
 164 } while (0)
 165
 166 #define ATOMIC_GEN(name, ptype, vtype, atype, sz)                       \
 167                                                                         \
 168 static __inline vtype                                                   \
 169 atomic_add_ ## name(volatile ptype p, atype v)                          \
 170 {                                                                       \
 171         return ((vtype)atomic_op(p, +, v, sz));                         \
 172 }                                                                       \
 173 static __inline vtype                                                   \
 174 atomic_add_acq_ ## name(volatile ptype p, atype v)                      \
 175 {                                                                       \
 176         return ((vtype)atomic_op_acq(p, +, v, sz));                     \
 177 }                                                                       \
 178 static __inline vtype                                                   \
 179 atomic_add_rel_ ## name(volatile ptype p, atype v)                      \
 180 {                                                                       \
 181         return ((vtype)atomic_op_rel(p, +, v, sz));                     \
 182 }                                                                       \
 183                                                                         \
 184 static __inline vtype                                                   \
 185 atomic_clear_ ## name(volatile ptype p, atype v)                        \
 186 {                                                                       \
 187         return ((vtype)atomic_op(p, &, ~v, sz));                        \
 188 }                                                                       \
 189 static __inline vtype                                                   \
 190 atomic_clear_acq_ ## name(volatile ptype p, atype v)                    \
 191 {                                                                       \
 192         return ((vtype)atomic_op_acq(p, &, ~v, sz));                    \
 193 }                                                                       \
 194 static __inline vtype                                                   \
 195 atomic_clear_rel_ ## name(volatile ptype p, atype v)                    \
 196 {                                                                       \
 197         return ((vtype)atomic_op_rel(p, &, ~v, sz));                    \
 198 }                                                                       \
 199                                                                         \
 200 static __inline int                                                     \
 201 atomic_cmpset_ ## name(volatile ptype p, vtype e, vtype s)              \
 202 {                                                                       \
 203         return (((vtype)atomic_cas(p, e, s, sz)) == e);                 \
 204 }                                                                       \
 205 static __inline int                                                     \
 206 atomic_cmpset_acq_ ## name(volatile ptype p, vtype e, vtype s)          \
 207 {                                                                       \
 208         return (((vtype)atomic_cas_acq(p, e, s, sz)) == e);             \
 209 }                                                                       \
 210 static __inline int                                                     \
 211 atomic_cmpset_rel_ ## name(volatile ptype p, vtype e, vtype s)          \
 212 {                                                                       \
 213         return (((vtype)atomic_cas_rel(p, e, s, sz)) == e);             \
 214 }                                                                       \
 215                                                                         \
 216 static __inline vtype                                                   \
 217 atomic_load_ ## name(volatile ptype p)                                  \
 218 {                                                                       \
 219         return ((vtype)atomic_cas(p, 0, 0, sz));                        \
 220 }                                                                       \
 221 static __inline vtype                                                   \
 222 atomic_load_acq_ ## name(volatile ptype p)                              \
 223 {                                                                       \
 224         return ((vtype)atomic_cas_acq(p, 0, 0, sz));                    \
 225 }                                                                       \
 226                                                                         \
 227 static __inline vtype                                                   \
 228 atomic_readandclear_ ## name(volatile ptype p)                          \
 229 {                                                                       \
 230         return ((vtype)atomic_load_clear(p, sz));                       \
 231 }                                                                       \
 232                                                                         \
 233 static __inline vtype                                                   \
 234 atomic_set_ ## name(volatile ptype p, atype v)                          \
 235 {                                                                       \
 236         return ((vtype)atomic_op(p, |, v, sz));                         \
 237 }                                                                       \
 238 static __inline vtype                                                   \
 239 atomic_set_acq_ ## name(volatile ptype p, atype v)                      \
 240 {                                                                       \
 241         return ((vtype)atomic_op_acq(p, |, v, sz));                     \
 242 }                                                                       \
 243 static __inline vtype                                                   \
 244 atomic_set_rel_ ## name(volatile ptype p, atype v)                      \
 245 {                                                                       \
 246         return ((vtype)atomic_op_rel(p, |, v, sz));                     \
 247 }                                                                       \
 248                                                                         \
 249 static __inline vtype                                                   \
 250 atomic_subtract_ ## name(volatile ptype p, atype v)                     \
 251 {                                                                       \
 252         return ((vtype)atomic_op(p, -, v, sz));                         \
 253 }                                                                       \
 254 static __inline vtype                                                   \
 255 atomic_subtract_acq_ ## name(volatile ptype p, atype v)                 \
 256 {                                                                       \
 257         return ((vtype)atomic_op_acq(p, -, v, sz));                     \
 258 }                                                                       \
 259 static __inline vtype                                                   \
 260 atomic_subtract_rel_ ## name(volatile ptype p, atype v)                 \
 261 {                                                                       \
 262         return ((vtype)atomic_op_rel(p, -, v, sz));                     \
 263 }                                                                       \
 264                                                                         \
 265 static __inline void                                                    \
 266 atomic_store_ ## name(volatile ptype p, vtype v)                        \
 267 {                                                                       \
 268         atomic_store(p, v, sz);                                         \
 269 }                                                                       \
 270 static __inline void                                                    \
 271 atomic_store_rel_ ## name(volatile ptype p, vtype v)                    \
 272 {                                                                       \
 273         atomic_store_rel(p, v, sz);                                     \
 274 }
 275
 276 ATOMIC_GEN(int, u_int *, u_int, u_int, 32);
 277 ATOMIC_GEN(32, uint32_t *, uint32_t, uint32_t, 32);
 278
 279 ATOMIC_GEN(long, u_long *, u_long, u_long, 64);
 280 ATOMIC_GEN(64, uint64_t *, uint64_t, uint64_t, 64);
 281
 282 ATOMIC_GEN(ptr, uintptr_t *, uintptr_t, uintptr_t, 64);
 283
 284 #define atomic_fetchadd_int     atomic_add_int
 285 #define atomic_fetchadd_32      atomic_add_32
 286 #define atomic_fetchadd_long    atomic_add_long
 287
 288 #undef ATOMIC_GEN
 289 #undef atomic_cas
 290 #undef atomic_cas_acq
 291 #undef atomic_cas_rel
 292 #undef atomic_op
 293 #undef atomic_op_acq
 294 #undef atomic_op_rel
 295 #undef atomic_load_acq
 296 #undef atomic_store_rel
 297 #undef atomic_load_clear
 298
 299 #endif /* !_MACHINE_ATOMIC_H_ */