sys/sparc64/include/atomic.h

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 1998 Doug Rabson.
   5  * Copyright (c) 2001 Jake Burkholder.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  *
  29  *      from: FreeBSD: src/sys/i386/include/atomic.h,v 1.20 2001/02/11
  30  * $FreeBSD$
  31  */
  32
  33 #ifndef _MACHINE_ATOMIC_H_
  34 #define _MACHINE_ATOMIC_H_
  35
  36 #include <machine/cpufunc.h>
  37
  38 #define mb()    __asm__ __volatile__ ("membar #MemIssue": : :"memory")
  39 #define wmb()   mb()
  40 #define rmb()   mb()
  41
  42 #include <sys/atomic_common.h>
  43
  44 /* Userland needs different ASI's. */
  45 #ifdef _KERNEL
  46 #define __ASI_ATOMIC    ASI_N
  47 #else
  48 #define __ASI_ATOMIC    ASI_P
  49 #endif
  50
  51 /*
  52  * Various simple arithmetic on memory which is atomic in the presence
  53  * of interrupts and multiple processors.  See atomic(9) for details.
  54  * Note that efficient hardware support exists only for the 32 and 64
  55  * bit variants; the 8 and 16 bit versions are not provided and should
  56  * not be used in MI code.
  57  *
  58  * This implementation takes advantage of the fact that the sparc64
  59  * cas instruction is both a load and a store.  The loop is often coded
  60  * as follows:
  61  *
  62  *      do {
  63  *              expect = *p;
  64  *              new = expect + 1;
  65  *      } while (cas(p, expect, new) != expect);
  66  *
  67  * which performs an unnnecessary load on each iteration that the cas
  68  * operation fails.  Modified as follows:
  69  *
  70  *      expect = *p;
  71  *      for (;;) {
  72  *              new = expect + 1;
  73  *              result = cas(p, expect, new);
  74  *              if (result == expect)
  75  *                      break;
  76  *              expect = result;
  77  *      }
  78  *
  79  * the return value of cas is used to avoid the extra reload.
  80  *
  81  * We only include a memory barrier in the rel variants as in total store
  82  * order which we use for running the kernel and all of the userland atomic
  83  * loads and stores behave as if the were followed by a membar with a mask
  84  * of #LoadLoad | #LoadStore | #StoreStore.  In order to be also sufficient
  85  * for use of relaxed memory ordering, the atomic_cas() in the acq variants
  86  * additionally would have to be followed by a membar #LoadLoad | #LoadStore.
  87  * Due to the suggested assembly syntax of the membar operands containing a
  88  * # character, they cannot be used in macros.  The cmask and mmask bits thus
  89  * are hard coded in machine/cpufunc.h and used here through macros.
  90  * Hopefully the bit numbers won't change in the future.
  91  */
  92
  93 #define itype(sz)       uint ## sz ## _t
  94
  95 #define atomic_cas_32(p, e, s)  casa((p), (e), (s), __ASI_ATOMIC)
  96 #define atomic_cas_64(p, e, s)  casxa((p), (e), (s), __ASI_ATOMIC)
  97
  98 #define atomic_cas(p, e, s, sz)                                         \
  99         atomic_cas_ ## sz((p), (e), (s))
 100
 101 #define atomic_cas_acq(p, e, s, sz) ({                                  \
 102         itype(sz) v;                                                    \
 103         v = atomic_cas((p), (e), (s), sz);                              \
 104         __compiler_membar();                                            \
 105         v;                                                              \
 106 })
 107
 108 #define atomic_cas_rel(p, e, s, sz) ({                                  \
 109         itype(sz) v;                                                    \
 110         membar(LoadStore | StoreStore);                                 \
 111         v = atomic_cas((p), (e), (s), sz);                              \
 112         v;                                                              \
 113 })
 114
 115 #define atomic_op(p, op, v, sz) ({                                      \
 116         itype(sz) e, r, s;                                              \
 117         for (e = *(volatile itype(sz) *)(p);; e = r) {                  \
 118                 s = e op (v);                                           \
 119                 r = atomic_cas_ ## sz((p), e, s);                       \
 120                 if (r == e)                                             \
 121                         break;                                          \
 122         }                                                               \
 123         e;                                                              \
 124 })
 125
 126 #define atomic_op_acq(p, op, v, sz) ({                                  \
 127         itype(sz) t;                                                    \
 128         t = atomic_op((p), op, (v), sz);                                \
 129         __compiler_membar();                                            \
 130         t;                                                              \
 131 })
 132
 133 #define atomic_op_rel(p, op, v, sz) ({                                  \
 134         itype(sz) t;                                                    \
 135         membar(LoadStore | StoreStore);                                 \
 136         t = atomic_op((p), op, (v), sz);                                \
 137         t;                                                              \
 138 })
 139
 140 #define atomic_ld_acq(p, sz) ({                                         \
 141         itype(sz) v;                                                    \
 142         v = atomic_cas((p), 0, 0, sz);                                  \
 143         __compiler_membar();                                            \
 144         v;                                                              \
 145 })
 146
 147 #define atomic_ld_clear(p, sz) ({                                       \
 148         itype(sz) e, r;                                                 \
 149         for (e = *(volatile itype(sz) *)(p);; e = r) {                  \
 150                 r = atomic_cas((p), e, 0, sz);                          \
 151                 if (r == e)                                             \
 152                         break;                                          \
 153         }                                                               \
 154         e;                                                              \
 155 })
 156
 157 #define atomic_st(p, v, sz) do {                                        \
 158         itype(sz) e, r;                                                 \
 159         for (e = *(volatile itype(sz) *)(p);; e = r) {                  \
 160                 r = atomic_cas((p), e, (v), sz);                        \
 161                 if (r == e)                                             \
 162                         break;                                          \
 163         }                                                               \
 164 } while (0)
 165
 166 #define atomic_st_acq(p, v, sz) do {                                    \
 167         atomic_st((p), (v), sz);                                        \
 168         __compiler_membar();                                            \
 169 } while (0)
 170
 171 #define atomic_st_rel(p, v, sz) do {                                    \
 172         membar(LoadStore | StoreStore);                                 \
 173         atomic_st((p), (v), sz);                                        \
 174 } while (0)
 175
 176 #define ATOMIC_GEN(name, ptype, vtype, atype, sz)                       \
 177                                                                         \
 178 static __inline vtype                                                   \
 179 atomic_add_ ## name(volatile ptype p, atype v)                          \
 180 {                                                                       \
 181         return ((vtype)atomic_op((p), +, (v), sz));                     \
 182 }                                                                       \
 183 static __inline vtype                                                   \
 184 atomic_add_acq_ ## name(volatile ptype p, atype v)                      \
 185 {                                                                       \
 186         return ((vtype)atomic_op_acq((p), +, (v), sz));                 \
 187 }                                                                       \
 188 static __inline vtype                                                   \
 189 atomic_add_rel_ ## name(volatile ptype p, atype v)                      \
 190 {                                                                       \
 191         return ((vtype)atomic_op_rel((p), +, (v), sz));                 \
 192 }                                                                       \
 193                                                                         \
 194 static __inline vtype                                                   \
 195 atomic_clear_ ## name(volatile ptype p, atype v)                        \
 196 {                                                                       \
 197         return ((vtype)atomic_op((p), &, ~(v), sz));                    \
 198 }                                                                       \
 199 static __inline vtype                                                   \
 200 atomic_clear_acq_ ## name(volatile ptype p, atype v)                    \
 201 {                                                                       \
 202         return ((vtype)atomic_op_acq((p), &, ~(v), sz));                \
 203 }                                                                       \
 204 static __inline vtype                                                   \
 205 atomic_clear_rel_ ## name(volatile ptype p, atype v)                    \
 206 {                                                                       \
 207         return ((vtype)atomic_op_rel((p), &, ~(v), sz));                \
 208 }                                                                       \
 209                                                                         \
 210 static __inline int                                                     \
 211 atomic_cmpset_ ## name(volatile ptype p, vtype e, vtype s)              \
 212 {                                                                       \
 213         return (((vtype)atomic_cas((p), (e), (s), sz)) == (e));         \
 214 }                                                                       \
 215 static __inline int                                                     \
 216 atomic_cmpset_acq_ ## name(volatile ptype p, vtype e, vtype s)          \
 217 {                                                                       \
 218         return (((vtype)atomic_cas_acq((p), (e), (s), sz)) == (e));     \
 219 }                                                                       \
 220 static __inline int                                                     \
 221 atomic_cmpset_rel_ ## name(volatile ptype p, vtype e, vtype s)          \
 222 {                                                                       \
 223         return (((vtype)atomic_cas_rel((p), (e), (s), sz)) == (e));     \
 224 }                                                                       \
 225                                                                         \
 226 static __inline int                                                     \
 227 atomic_fcmpset_ ## name(volatile ptype p, vtype *ep, vtype s)           \
 228 {                                                                       \
 229         vtype t;                                                        \
 230                                                                         \
 231         t = (vtype)atomic_cas((p), (*ep), (s), sz);                     \
 232         if (t == (*ep))                                                 \
 233                 return (1);                                             \
 234         *ep = t;                                                        \
 235         return (0);                                                     \
 236 }                                                                       \
 237 static __inline int                                                     \
 238 atomic_fcmpset_acq_ ## name(volatile ptype p, vtype *ep, vtype s)       \
 239 {                                                                       \
 240         vtype t;                                                        \
 241                                                                         \
 242         t = (vtype)atomic_cas_acq((p), (*ep), (s), sz);                 \
 243         if (t == (*ep))                                                 \
 244                 return (1);                                             \
 245         *ep = t;                                                        \
 246         return (0);                                                     \
 247 }                                                                       \
 248 static __inline int                                                     \
 249 atomic_fcmpset_rel_ ## name(volatile ptype p, vtype *ep, vtype s)       \
 250 {                                                                       \
 251         vtype t;                                                        \
 252                                                                         \
 253         t = (vtype)atomic_cas_rel((p), (*ep), (s), sz);                 \
 254         if (t == (*ep))                                                 \
 255                 return (1);                                             \
 256         *ep = t;                                                        \
 257         return (0);                                                     \
 258 }                                                                       \
 259                                                                         \
 260 static __inline vtype                                                   \
 261 atomic_load_acq_ ## name(volatile ptype p)                              \
 262 {                                                                       \
 263         return ((vtype)atomic_cas_acq((p), 0, 0, sz));                  \
 264 }                                                                       \
 265                                                                         \
 266 static __inline vtype                                                   \
 267 atomic_readandclear_ ## name(volatile ptype p)                          \
 268 {                                                                       \
 269         return ((vtype)atomic_ld_clear((p), sz));                       \
 270 }                                                                       \
 271                                                                         \
 272 static __inline vtype                                                   \
 273 atomic_set_ ## name(volatile ptype p, atype v)                          \
 274 {                                                                       \
 275         return ((vtype)atomic_op((p), |, (v), sz));                     \
 276 }                                                                       \
 277 static __inline vtype                                                   \
 278 atomic_set_acq_ ## name(volatile ptype p, atype v)                      \
 279 {                                                                       \
 280         return ((vtype)atomic_op_acq((p), |, (v), sz));                 \
 281 }                                                                       \
 282 static __inline vtype                                                   \
 283 atomic_set_rel_ ## name(volatile ptype p, atype v)                      \
 284 {                                                                       \
 285         return ((vtype)atomic_op_rel((p), |, (v), sz));                 \
 286 }                                                                       \
 287                                                                         \
 288 static __inline vtype                                                   \
 289 atomic_subtract_ ## name(volatile ptype p, atype v)                     \
 290 {                                                                       \
 291         return ((vtype)atomic_op((p), -, (v), sz));                     \
 292 }                                                                       \
 293 static __inline vtype                                                   \
 294 atomic_subtract_acq_ ## name(volatile ptype p, atype v)                 \
 295 {                                                                       \
 296         return ((vtype)atomic_op_acq((p), -, (v), sz));                 \
 297 }                                                                       \
 298 static __inline vtype                                                   \
 299 atomic_subtract_rel_ ## name(volatile ptype p, atype v)                 \
 300 {                                                                       \
 301         return ((vtype)atomic_op_rel((p), -, (v), sz));                 \
 302 }                                                                       \
 303                                                                         \
 304 static __inline void                                                    \
 305 atomic_store_acq_ ## name(volatile ptype p, vtype v)                    \
 306 {                                                                       \
 307         atomic_st_acq((p), (v), sz);                                    \
 308 }                                                                       \
 309 static __inline void                                                    \
 310 atomic_store_rel_ ## name(volatile ptype p, vtype v)                    \
 311 {                                                                       \
 312         atomic_st_rel((p), (v), sz);                                    \
 313 }
 314
 315 static __inline void
 316 atomic_thread_fence_acq(void)
 317 {
 318
 319         __compiler_membar();
 320 }
 321
 322 static __inline void
 323 atomic_thread_fence_rel(void)
 324 {
 325
 326         __compiler_membar();
 327 }
 328
 329 static __inline void
 330 atomic_thread_fence_acq_rel(void)
 331 {
 332
 333         __compiler_membar();
 334 }
 335
 336 static __inline void
 337 atomic_thread_fence_seq_cst(void)
 338 {
 339
 340         membar(LoadLoad | LoadStore | StoreStore | StoreLoad);
 341 }
 342
 343
 344 ATOMIC_GEN(int, u_int *, u_int, u_int, 32);
 345 ATOMIC_GEN(32, uint32_t *, uint32_t, uint32_t, 32);
 346
 347 ATOMIC_GEN(long, u_long *, u_long, u_long, 64);
 348 ATOMIC_GEN(64, uint64_t *, uint64_t, uint64_t, 64);
 349
 350 ATOMIC_GEN(ptr, uintptr_t *, uintptr_t, uintptr_t, 64);
 351
 352 #define atomic_fetchadd_int     atomic_add_int
 353 #define atomic_fetchadd_32      atomic_add_32
 354 #define atomic_fetchadd_long    atomic_add_long
 355 #define atomic_fetchadd_64      atomic_add_64
 356
 357 #undef ATOMIC_GEN
 358 #undef atomic_cas
 359 #undef atomic_cas_acq
 360 #undef atomic_cas_rel
 361 #undef atomic_op
 362 #undef atomic_op_acq
 363 #undef atomic_op_rel
 364 #undef atomic_ld_acq
 365 #undef atomic_ld_clear
 366 #undef atomic_st
 367 #undef atomic_st_acq
 368 #undef atomic_st_rel
 369
 370 #endif /* !_MACHINE_ATOMIC_H_ */