sys/sparc64/include/atomic.h

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 1998 Doug Rabson.
   5  * Copyright (c) 2001 Jake Burkholder.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  *
  29  *      from: FreeBSD: src/sys/i386/include/atomic.h,v 1.20 2001/02/11
  30  * $FreeBSD$
  31  */
  32
  33 #ifndef _MACHINE_ATOMIC_H_
  34 #define _MACHINE_ATOMIC_H_
  35
  36 #include <machine/cpufunc.h>
  37
  38 #define mb()    __asm__ __volatile__ ("membar #MemIssue": : :"memory")
  39 #define wmb()   mb()
  40 #define rmb()   mb()
  41
  42 #include <sys/atomic_common.h>
  43
  44 /* Userland needs different ASI's. */
  45 #ifdef _KERNEL
  46 #define __ASI_ATOMIC    ASI_N
  47 #else
  48 #define __ASI_ATOMIC    ASI_P
  49 #endif
  50
  51 /*
  52  * Various simple arithmetic on memory which is atomic in the presence
  53  * of interrupts and multiple processors.  See atomic(9) for details.
  54  * Note that efficient hardware support exists only for the 32 and 64
  55  * bit variants; the 8 and 16 bit versions are not provided and should
  56  * not be used in MI code.
  57  *
  58  * This implementation takes advantage of the fact that the sparc64
  59  * cas instruction is both a load and a store.  The loop is often coded
  60  * as follows:
  61  *
  62  *      do {
  63  *              expect = *p;
  64  *              new = expect + 1;
  65  *      } while (cas(p, expect, new) != expect);
  66  *
  67  * which performs an unnnecessary load on each iteration that the cas
  68  * operation fails.  Modified as follows:
  69  *
  70  *      expect = *p;
  71  *      for (;;) {
  72  *              new = expect + 1;
  73  *              result = cas(p, expect, new);
  74  *              if (result == expect)
  75  *                      break;
  76  *              expect = result;
  77  *      }
  78  *
  79  * the return value of cas is used to avoid the extra reload.
  80  *
  81  * We only include a memory barrier in the rel variants as in total store
  82  * order which we use for running the kernel and all of the userland atomic
  83  * loads and stores behave as if the were followed by a membar with a mask
  84  * of #LoadLoad | #LoadStore | #StoreStore.  In order to be also sufficient
  85  * for use of relaxed memory ordering, the atomic_cas() in the acq variants
  86  * additionally would have to be followed by a membar #LoadLoad | #LoadStore.
  87  * Due to the suggested assembly syntax of the membar operands containing a
  88  * # character, they cannot be used in macros.  The cmask and mmask bits thus
  89  * are hard coded in machine/cpufunc.h and used here through macros.
  90  * Hopefully the bit numbers won't change in the future.
  91  */
  92
  93 #define itype(sz)       uint ## sz ## _t
  94
  95 #define atomic_cas_32(p, e, s)  casa((p), (e), (s), __ASI_ATOMIC)
  96 #define atomic_cas_64(p, e, s)  casxa((p), (e), (s), __ASI_ATOMIC)
  97
  98 #define atomic_cas(p, e, s, sz)                                         \
  99         atomic_cas_ ## sz((p), (e), (s))
 100
 101 #define atomic_cas_acq(p, e, s, sz) ({                                  \
 102         itype(sz) v;                                                    \
 103         v = atomic_cas((p), (e), (s), sz);                              \
 104         __compiler_membar();                                            \
 105         v;                                                              \
 106 })
 107
 108 #define atomic_cas_rel(p, e, s, sz) ({                                  \
 109         itype(sz) v;                                                    \
 110         membar(LoadStore | StoreStore);                                 \
 111         v = atomic_cas((p), (e), (s), sz);                              \
 112         v;                                                              \
 113 })
 114
 115 #define atomic_op(p, op, v, sz) ({                                      \
 116         itype(sz) e, r, s;                                              \
 117         for (e = *(volatile itype(sz) *)(p);; e = r) {                  \
 118                 s = e op (v);                                           \
 119                 r = atomic_cas_ ## sz((p), e, s);                       \
 120                 if (r == e)                                             \
 121                         break;                                          \
 122         }                                                               \
 123         e;                                                              \
 124 })
 125
 126 #define atomic_op_acq(p, op, v, sz) ({                                  \
 127         itype(sz) t;                                                    \
 128         t = atomic_op((p), op, (v), sz);                                \
 129         __compiler_membar();                                            \
 130         t;                                                              \
 131 })
 132
 133 #define atomic_op_rel(p, op, v, sz) ({                                  \
 134         itype(sz) t;                                                    \
 135         membar(LoadStore | StoreStore);                                 \
 136         t = atomic_op((p), op, (v), sz);                                \
 137         t;                                                              \
 138 })
 139
 140 #define atomic_ld_acq(p, sz) ({                                         \
 141         itype(sz) v;                                                    \
 142         v = atomic_cas((p), 0, 0, sz);                                  \
 143         __compiler_membar();                                            \
 144         v;                                                              \
 145 })
 146
 147 #define atomic_ld_clear(p, sz) ({                                       \
 148         itype(sz) e, r;                                                 \
 149         for (e = *(volatile itype(sz) *)(p);; e = r) {                  \
 150                 r = atomic_cas((p), e, 0, sz);                          \
 151                 if (r == e)                                             \
 152                         break;                                          \
 153         }                                                               \
 154         e;                                                              \
 155 })
 156
 157 #define atomic_st(p, v, sz) ({                                          \
 158         itype(sz) e, r;                                                 \
 159         for (e = *(volatile itype(sz) *)(p);; e = r) {                  \
 160                 r = atomic_cas((p), e, (v), sz);                        \
 161                 if (r == e)                                             \
 162                         break;                                          \
 163         }                                                               \
 164         e;                                                              \
 165 })
 166
 167 #define atomic_st_acq(p, v, sz) do {                                    \
 168         atomic_st((p), (v), sz);                                        \
 169         __compiler_membar();                                            \
 170 } while (0)
 171
 172 #define atomic_st_rel(p, v, sz) do {                                    \
 173         membar(LoadStore | StoreStore);                                 \
 174         atomic_st((p), (v), sz);                                        \
 175 } while (0)
 176
 177 #define ATOMIC_GEN(name, ptype, vtype, atype, sz)                       \
 178                                                                         \
 179 static __inline vtype                                                   \
 180 atomic_add_ ## name(volatile ptype p, atype v)                          \
 181 {                                                                       \
 182         return ((vtype)atomic_op((p), +, (v), sz));                     \
 183 }                                                                       \
 184 static __inline vtype                                                   \
 185 atomic_add_acq_ ## name(volatile ptype p, atype v)                      \
 186 {                                                                       \
 187         return ((vtype)atomic_op_acq((p), +, (v), sz));                 \
 188 }                                                                       \
 189 static __inline vtype                                                   \
 190 atomic_add_rel_ ## name(volatile ptype p, atype v)                      \
 191 {                                                                       \
 192         return ((vtype)atomic_op_rel((p), +, (v), sz));                 \
 193 }                                                                       \
 194                                                                         \
 195 static __inline vtype                                                   \
 196 atomic_clear_ ## name(volatile ptype p, atype v)                        \
 197 {                                                                       \
 198         return ((vtype)atomic_op((p), &, ~(v), sz));                    \
 199 }                                                                       \
 200 static __inline vtype                                                   \
 201 atomic_clear_acq_ ## name(volatile ptype p, atype v)                    \
 202 {                                                                       \
 203         return ((vtype)atomic_op_acq((p), &, ~(v), sz));                \
 204 }                                                                       \
 205 static __inline vtype                                                   \
 206 atomic_clear_rel_ ## name(volatile ptype p, atype v)                    \
 207 {                                                                       \
 208         return ((vtype)atomic_op_rel((p), &, ~(v), sz));                \
 209 }                                                                       \
 210                                                                         \
 211 static __inline int                                                     \
 212 atomic_cmpset_ ## name(volatile ptype p, vtype e, vtype s)              \
 213 {                                                                       \
 214         return (((vtype)atomic_cas((p), (e), (s), sz)) == (e));         \
 215 }                                                                       \
 216 static __inline int                                                     \
 217 atomic_cmpset_acq_ ## name(volatile ptype p, vtype e, vtype s)          \
 218 {                                                                       \
 219         return (((vtype)atomic_cas_acq((p), (e), (s), sz)) == (e));     \
 220 }                                                                       \
 221 static __inline int                                                     \
 222 atomic_cmpset_rel_ ## name(volatile ptype p, vtype e, vtype s)          \
 223 {                                                                       \
 224         return (((vtype)atomic_cas_rel((p), (e), (s), sz)) == (e));     \
 225 }                                                                       \
 226                                                                         \
 227 static __inline int                                                     \
 228 atomic_fcmpset_ ## name(volatile ptype p, vtype *ep, vtype s)           \
 229 {                                                                       \
 230         vtype t;                                                        \
 231                                                                         \
 232         t = (vtype)atomic_cas((p), (*ep), (s), sz);                     \
 233         if (t == (*ep))                                                 \
 234                 return (1);                                             \
 235         *ep = t;                                                        \
 236         return (0);                                                     \
 237 }                                                                       \
 238 static __inline int                                                     \
 239 atomic_fcmpset_acq_ ## name(volatile ptype p, vtype *ep, vtype s)       \
 240 {                                                                       \
 241         vtype t;                                                        \
 242                                                                         \
 243         t = (vtype)atomic_cas_acq((p), (*ep), (s), sz);                 \
 244         if (t == (*ep))                                                 \
 245                 return (1);                                             \
 246         *ep = t;                                                        \
 247         return (0);                                                     \
 248 }                                                                       \
 249 static __inline int                                                     \
 250 atomic_fcmpset_rel_ ## name(volatile ptype p, vtype *ep, vtype s)       \
 251 {                                                                       \
 252         vtype t;                                                        \
 253                                                                         \
 254         t = (vtype)atomic_cas_rel((p), (*ep), (s), sz);                 \
 255         if (t == (*ep))                                                 \
 256                 return (1);                                             \
 257         *ep = t;                                                        \
 258         return (0);                                                     \
 259 }                                                                       \
 260                                                                         \
 261 static __inline vtype                                                   \
 262 atomic_load_acq_ ## name(volatile ptype p)                              \
 263 {                                                                       \
 264         return ((vtype)atomic_cas_acq((p), 0, 0, sz));                  \
 265 }                                                                       \
 266                                                                         \
 267 static __inline vtype                                                   \
 268 atomic_readandclear_ ## name(volatile ptype p)                          \
 269 {                                                                       \
 270         return ((vtype)atomic_ld_clear((p), sz));                       \
 271 }                                                                       \
 272                                                                         \
 273 static __inline vtype                                                   \
 274 atomic_set_ ## name(volatile ptype p, atype v)                          \
 275 {                                                                       \
 276         return ((vtype)atomic_op((p), |, (v), sz));                     \
 277 }                                                                       \
 278 static __inline vtype                                                   \
 279 atomic_set_acq_ ## name(volatile ptype p, atype v)                      \
 280 {                                                                       \
 281         return ((vtype)atomic_op_acq((p), |, (v), sz));                 \
 282 }                                                                       \
 283 static __inline vtype                                                   \
 284 atomic_set_rel_ ## name(volatile ptype p, atype v)                      \
 285 {                                                                       \
 286         return ((vtype)atomic_op_rel((p), |, (v), sz));                 \
 287 }                                                                       \
 288                                                                         \
 289 static __inline vtype                                                   \
 290 atomic_subtract_ ## name(volatile ptype p, atype v)                     \
 291 {                                                                       \
 292         return ((vtype)atomic_op((p), -, (v), sz));                     \
 293 }                                                                       \
 294 static __inline vtype                                                   \
 295 atomic_subtract_acq_ ## name(volatile ptype p, atype v)                 \
 296 {                                                                       \
 297         return ((vtype)atomic_op_acq((p), -, (v), sz));                 \
 298 }                                                                       \
 299 static __inline vtype                                                   \
 300 atomic_subtract_rel_ ## name(volatile ptype p, atype v)                 \
 301 {                                                                       \
 302         return ((vtype)atomic_op_rel((p), -, (v), sz));                 \
 303 }                                                                       \
 304                                                                         \
 305 static __inline void                                                    \
 306 atomic_store_acq_ ## name(volatile ptype p, vtype v)                    \
 307 {                                                                       \
 308         atomic_st_acq((p), (v), sz);                                    \
 309 }                                                                       \
 310 static __inline void                                                    \
 311 atomic_store_rel_ ## name(volatile ptype p, vtype v)                    \
 312 {                                                                       \
 313         atomic_st_rel((p), (v), sz);                                    \
 314 }                                                                       \
 315                                                                         \
 316 static __inline vtype                                                   \
 317 atomic_swap_ ## name(volatile ptype p, vtype v)                         \
 318 {                                                                       \
 319         return ((vtype)atomic_st((p), (v), sz));                        \
 320 }
 321
 322 static __inline void
 323 atomic_thread_fence_acq(void)
 324 {
 325
 326         __compiler_membar();
 327 }
 328
 329 static __inline void
 330 atomic_thread_fence_rel(void)
 331 {
 332
 333         __compiler_membar();
 334 }
 335
 336 static __inline void
 337 atomic_thread_fence_acq_rel(void)
 338 {
 339
 340         __compiler_membar();
 341 }
 342
 343 static __inline void
 344 atomic_thread_fence_seq_cst(void)
 345 {
 346
 347         membar(LoadLoad | LoadStore | StoreStore | StoreLoad);
 348 }
 349
 350
 351 ATOMIC_GEN(int, u_int *, u_int, u_int, 32);
 352 ATOMIC_GEN(32, uint32_t *, uint32_t, uint32_t, 32);
 353
 354 ATOMIC_GEN(long, u_long *, u_long, u_long, 64);
 355 ATOMIC_GEN(64, uint64_t *, uint64_t, uint64_t, 64);
 356
 357 ATOMIC_GEN(ptr, uintptr_t *, uintptr_t, uintptr_t, 64);
 358
 359 #define atomic_fetchadd_int     atomic_add_int
 360 #define atomic_fetchadd_32      atomic_add_32
 361 #define atomic_fetchadd_long    atomic_add_long
 362 #define atomic_fetchadd_64      atomic_add_64
 363
 364 #undef ATOMIC_GEN
 365 #undef atomic_cas
 366 #undef atomic_cas_acq
 367 #undef atomic_cas_rel
 368 #undef atomic_op
 369 #undef atomic_op_acq
 370 #undef atomic_op_rel
 371 #undef atomic_ld_acq
 372 #undef atomic_ld_clear
 373 #undef atomic_st
 374 #undef atomic_st_acq
 375 #undef atomic_st_rel
 376
 377 #endif /* !_MACHINE_ATOMIC_H_ */