sys/powerpc/include/atomic.h

   1 /*-
   2  * Copyright (c) 2008 Marcel Moolenaar
   3  * Copyright (c) 2001 Benno Rice
   4  * Copyright (c) 2001 David E. O'Brien
   5  * Copyright (c) 1998 Doug Rabson
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  *
  29  * $FreeBSD$
  30  */
  31
  32 #ifndef _MACHINE_ATOMIC_H_
  33 #define _MACHINE_ATOMIC_H_
  34
  35 #ifndef _SYS_CDEFS_H_
  36 #error this file needs sys/cdefs.h as a prerequisite
  37 #endif
  38
  39 #define __ATOMIC_BARRIER                                        \
  40     __asm __volatile("sync" : : : "memory")
  41
  42 #define mb()    __ATOMIC_BARRIER
  43 #define wmb()   mb()
  44 #define rmb()   mb()
  45
  46 /*
  47  * atomic_add(p, v)
  48  * { *p += v; }
  49  */
  50
  51 #define __ATOMIC_ADD_8(p, v, t)                                 \
  52     8-bit atomic_add not implemented
  53
  54 #define __ATOMIC_ADD_16(p, v, t)                                \
  55     16-bit atomic_add not implemented
  56
  57 #define __ATOMIC_ADD_32(p, v, t)                                \
  58     __asm __volatile(                                           \
  59         "1:     lwarx   %0, 0, %2\n"                            \
  60         "       add     %0, %3, %0\n"                           \
  61         "       stwcx.  %0, 0, %2\n"                            \
  62         "       bne-    1b\n"                                   \
  63         : "=&r" (t), "=m" (*p)                                  \
  64         : "r" (p), "r" (v), "m" (*p)                            \
  65         : "cc", "memory")                                       \
  66     /* __ATOMIC_ADD_32 */
  67
  68 #define __ATOMIC_ADD_64(p, v, t)                                \
  69     64-bit atomic_add not implemented
  70
  71 #define _ATOMIC_ADD(width, suffix, type)                        \
  72     static __inline void                                        \
  73     atomic_add_##suffix(volatile type *p, type v) {             \
  74         type t;                                                 \
  75         __ATOMIC_ADD_##width(p, v, t);                          \
  76     }                                                           \
  77                                                                 \
  78     static __inline void                                        \
  79     atomic_add_acq_##suffix(volatile type *p, type v) {         \
  80         type t;                                                 \
  81         __ATOMIC_ADD_##width(p, v, t);                          \
  82         __ATOMIC_BARRIER;                                       \
  83     }                                                           \
  84                                                                 \
  85     static __inline void                                        \
  86     atomic_add_rel_##suffix(volatile type *p, type v) {         \
  87         type t;                                                 \
  88         __ATOMIC_BARRIER;                                       \
  89         __ATOMIC_ADD_##width(p, v, t);                          \
  90     }                                                           \
  91     /* _ATOMIC_ADD */
  92
  93 #if 0
  94 _ATOMIC_ADD(8, 8, uint8_t)
  95 _ATOMIC_ADD(8, char, u_char)
  96 _ATOMIC_ADD(16, 16, uint16_t)
  97 _ATOMIC_ADD(16, short, u_short)
  98 #endif
  99 _ATOMIC_ADD(32, 32, uint32_t)
 100 _ATOMIC_ADD(32, int, u_int)
 101 _ATOMIC_ADD(32, long, u_long)
 102 _ATOMIC_ADD(32, ptr, uintptr_t)
 103 #if 0
 104 _ATOMIC_ADD(64, 64, uint64_t)
 105 _ATOMIC_ADD(64, long_long, u_long_long)
 106 #endif
 107
 108 #undef _ATOMIC_ADD
 109 #undef __ATOMIC_ADD_64
 110 #undef __ATOMIC_ADD_32
 111 #undef __ATOMIC_ADD_16
 112 #undef __ATOMIC_ADD_8
 113
 114 /*
 115  * atomic_clear(p, v)
 116  * { *p &= ~v; }
 117  */
 118
 119 #define __ATOMIC_CLEAR_8(p, v, t)                               \
 120     8-bit atomic_clear not implemented
 121
 122 #define __ATOMIC_CLEAR_16(p, v, t)                              \
 123     16-bit atomic_clear not implemented
 124
 125 #define __ATOMIC_CLEAR_32(p, v, t)                              \
 126     __asm __volatile(                                           \
 127         "1:     lwarx   %0, 0, %2\n"                            \
 128         "       andc    %0, %0, %3\n"                           \
 129         "       stwcx.  %0, 0, %2\n"                            \
 130         "       bne-    1b\n"                                   \
 131         : "=&r" (t), "=m" (*p)                                  \
 132         : "r" (p), "r" (v), "m" (*p)                            \
 133         : "cc", "memory")                                       \
 134     /* __ATOMIC_CLEAR_32 */
 135
 136 #define __ATOMIC_CLEAR_64(p, v, t)                              \
 137     64-bit atomic_clear not implemented
 138
 139 #define _ATOMIC_CLEAR(width, suffix, type)                      \
 140     static __inline void                                        \
 141     atomic_clear_##suffix(volatile type *p, type v) {           \
 142         type t;                                                 \
 143         __ATOMIC_CLEAR_##width(p, v, t);                        \
 144     }                                                           \
 145                                                                 \
 146     static __inline void                                        \
 147     atomic_clear_acq_##suffix(volatile type *p, type v) {       \
 148         type t;                                                 \
 149         __ATOMIC_CLEAR_##width(p, v, t);                        \
 150         __ATOMIC_BARRIER;                                       \
 151     }                                                           \
 152                                                                 \
 153     static __inline void                                        \
 154     atomic_clear_rel_##suffix(volatile type *p, type v) {       \
 155         type t;                                                 \
 156         __ATOMIC_BARRIER;                                       \
 157         __ATOMIC_CLEAR_##width(p, v, t);                        \
 158     }                                                           \
 159     /* _ATOMIC_CLEAR */
 160
 161 #if 0
 162 _ATOMIC_CLEAR(8, 8, uint8_t)
 163 _ATOMIC_CLEAR(8, char, u_char)
 164 _ATOMIC_CLEAR(16, 16, uint16_t)
 165 _ATOMIC_CLEAR(16, short, u_short)
 166 #endif
 167 _ATOMIC_CLEAR(32, 32, uint32_t)
 168 _ATOMIC_CLEAR(32, int, u_int)
 169 _ATOMIC_CLEAR(32, long, u_long)
 170 _ATOMIC_CLEAR(32, ptr, uintptr_t)
 171 #if 0
 172 _ATOMIC_CLEAR(64, 64, uint64_t)
 173 _ATOMIC_CLEAR(64, long_long, u_long_long)
 174 #endif
 175
 176 #undef _ATOMIC_CLEAR
 177 #undef __ATOMIC_CLEAR_64
 178 #undef __ATOMIC_CLEAR_32
 179 #undef __ATOMIC_CLEAR_16
 180 #undef __ATOMIC_CLEAR_8
 181
 182 /*
 183  * atomic_cmpset(p, o, n)
 184  */
 185 /* TODO -- see below */
 186
 187 /*
 188  * atomic_load_acq(p)
 189  */
 190 /* TODO -- see below */
 191
 192 /*
 193  * atomic_readandclear(p)
 194  */
 195 /* TODO -- see below */
 196
 197 /*
 198  * atomic_set(p, v)
 199  * { *p |= v; }
 200  */
 201
 202 #define __ATOMIC_SET_8(p, v, t)                                 \
 203     8-bit atomic_set not implemented
 204
 205 #define __ATOMIC_SET_16(p, v, t)                                \
 206     16-bit atomic_set not implemented
 207
 208 #define __ATOMIC_SET_32(p, v, t)                                \
 209     __asm __volatile(                                           \
 210         "1:     lwarx   %0, 0, %2\n"                            \
 211         "       or      %0, %3, %0\n"                           \
 212         "       stwcx.  %0, 0, %2\n"                            \
 213         "       bne-    1b\n"                                   \
 214         : "=&r" (t), "=m" (*p)                                  \
 215         : "r" (p), "r" (v), "m" (*p)                            \
 216         : "cc", "memory")                                       \
 217     /* __ATOMIC_SET_32 */
 218
 219 #define __ATOMIC_SET_64(p, v, t)                                \
 220     64-bit atomic_set not implemented
 221
 222 #define _ATOMIC_SET(width, suffix, type)                        \
 223     static __inline void                                        \
 224     atomic_set_##suffix(volatile type *p, type v) {             \
 225         type t;                                                 \
 226         __ATOMIC_SET_##width(p, v, t);                          \
 227     }                                                           \
 228                                                                 \
 229     static __inline void                                        \
 230     atomic_set_acq_##suffix(volatile type *p, type v) {         \
 231         type t;                                                 \
 232         __ATOMIC_SET_##width(p, v, t);                          \
 233         __ATOMIC_BARRIER;                                       \
 234     }                                                           \
 235                                                                 \
 236     static __inline void                                        \
 237     atomic_set_rel_##suffix(volatile type *p, type v) {         \
 238         type t;                                                 \
 239         __ATOMIC_BARRIER;                                       \
 240         __ATOMIC_SET_##width(p, v, t);                          \
 241     }                                                           \
 242     /* _ATOMIC_SET */
 243
 244 #if 0
 245 _ATOMIC_SET(8, 8, uint8_t)
 246 _ATOMIC_SET(8, char, u_char)
 247 _ATOMIC_SET(16, 16, uint16_t)
 248 _ATOMIC_SET(16, short, u_short)
 249 #endif
 250 _ATOMIC_SET(32, 32, uint32_t)
 251 _ATOMIC_SET(32, int, u_int)
 252 _ATOMIC_SET(32, long, u_long)
 253 _ATOMIC_SET(32, ptr, uintptr_t)
 254 #if 0
 255 _ATOMIC_SET(64, 64, uint64_t)
 256 _ATOMIC_SET(64, long_long, u_long_long)
 257 #endif
 258
 259 #undef _ATOMIC_SET
 260 #undef __ATOMIC_SET_64
 261 #undef __ATOMIC_SET_32
 262 #undef __ATOMIC_SET_16
 263 #undef __ATOMIC_SET_8
 264
 265 /*
 266  * atomic_subtract(p, v)
 267  * { *p -= v; }
 268  */
 269
 270 #define __ATOMIC_SUBTRACT_8(p, v, t)                            \
 271     8-bit atomic_subtract not implemented
 272
 273 #define __ATOMIC_SUBTRACT_16(p, v, t)                           \
 274     16-bit atomic_subtract not implemented
 275
 276 #define __ATOMIC_SUBTRACT_32(p, v, t)                           \
 277     __asm __volatile(                                           \
 278         "1:     lwarx   %0, 0, %2\n"                            \
 279         "       subf    %0, %3, %0\n"                           \
 280         "       stwcx.  %0, 0, %2\n"                            \
 281         "       bne-    1b\n"                                   \
 282         : "=&r" (t), "=m" (*p)                                  \
 283         : "r" (p), "r" (v), "m" (*p)                            \
 284         : "cc", "memory")                                       \
 285     /* __ATOMIC_SUBTRACT_32 */
 286
 287 #define __ATOMIC_SUBTRACT_64(p, v, t)                           \
 288     64-bit atomic_subtract not implemented
 289
 290 #define _ATOMIC_SUBTRACT(width, suffix, type)                   \
 291     static __inline void                                        \
 292     atomic_subtract_##suffix(volatile type *p, type v) {        \
 293         type t;                                                 \
 294         __ATOMIC_SUBTRACT_##width(p, v, t);                     \
 295     }                                                           \
 296                                                                 \
 297     static __inline void                                        \
 298     atomic_subtract_acq_##suffix(volatile type *p, type v) {    \
 299         type t;                                                 \
 300         __ATOMIC_SUBTRACT_##width(p, v, t);                     \
 301         __ATOMIC_BARRIER;                                       \
 302     }                                                           \
 303                                                                 \
 304     static __inline void                                        \
 305     atomic_subtract_rel_##suffix(volatile type *p, type v) {    \
 306         type t;                                                 \
 307         __ATOMIC_BARRIER;                                       \
 308         __ATOMIC_SUBTRACT_##width(p, v, t);                     \
 309     }                                                           \
 310     /* _ATOMIC_SUBTRACT */
 311
 312 #if 0
 313 _ATOMIC_SUBTRACT(8, 8, uint8_t)
 314 _ATOMIC_SUBTRACT(8, char, u_char)
 315 _ATOMIC_SUBTRACT(16, 16, uint16_t)
 316 _ATOMIC_SUBTRACT(16, short, u_short)
 317 #endif
 318 _ATOMIC_SUBTRACT(32, 32, uint32_t)
 319 _ATOMIC_SUBTRACT(32, int, u_int)
 320 _ATOMIC_SUBTRACT(32, long, u_long)
 321 _ATOMIC_SUBTRACT(32, ptr, uintptr_t)
 322 #if 0
 323 _ATOMIC_SUBTRACT(64, 64, uint64_t)
 324 _ATOMIC_SUBTRACT(64, long_long, u_long_long)
 325 #endif
 326
 327 #undef _ATOMIC_SUBTRACT
 328 #undef __ATOMIC_SUBTRACT_64
 329 #undef __ATOMIC_SUBTRACT_32
 330 #undef __ATOMIC_SUBTRACT_16
 331 #undef __ATOMIC_SUBTRACT_8
 332
 333 /*
 334  * atomic_store_rel(p, v)
 335  */
 336 /* TODO -- see below */
 337
 338 /*
 339  * Old/original implementations that still need revisiting.
 340  */
 341
 342 static __inline uint32_t
 343 atomic_readandclear_32(volatile uint32_t *addr)
 344 {
 345         uint32_t result,temp;
 346
 347 #ifdef __GNUCLIKE_ASM
 348         __asm __volatile (
 349                 "\tsync\n"                      /* drain writes */
 350                 "1:\tlwarx %0, 0, %3\n\t"       /* load old value */
 351                 "li %1, 0\n\t"                  /* load new value */
 352                 "stwcx. %1, 0, %3\n\t"          /* attempt to store */
 353                 "bne- 1b\n\t"                   /* spin if failed */
 354                 : "=&r"(result), "=&r"(temp), "=m" (*addr)
 355                 : "r" (addr), "m" (*addr)
 356                 : "cc", "memory");
 357 #endif
 358
 359         return (result);
 360 }
 361
 362 #define atomic_readandclear_int         atomic_readandclear_32
 363 #define atomic_readandclear_long        atomic_readandclear_32
 364 #define atomic_readandclear_ptr         atomic_readandclear_32
 365
 366 /*
 367  * We assume that a = b will do atomic loads and stores.
 368  */
 369 #define ATOMIC_STORE_LOAD(TYPE, WIDTH)                          \
 370 static __inline u_##TYPE                                        \
 371 atomic_load_acq_##WIDTH(volatile u_##TYPE *p)                   \
 372 {                                                               \
 373         u_##TYPE v;                                             \
 374                                                                 \
 375         v = *p;                                                 \
 376         __ATOMIC_BARRIER;                                       \
 377         return (v);                                             \
 378 }                                                               \
 379                                                                 \
 380 static __inline void                                            \
 381 atomic_store_rel_##WIDTH(volatile u_##TYPE *p, u_##TYPE v)      \
 382 {                                                               \
 383         __ATOMIC_BARRIER;                                       \
 384         *p = v;                                                 \
 385 }                                                               \
 386                                                                 \
 387 static __inline u_##TYPE                                        \
 388 atomic_load_acq_##TYPE(volatile u_##TYPE *p)                    \
 389 {                                                               \
 390         u_##TYPE v;                                             \
 391                                                                 \
 392         v = *p;                                                 \
 393         __ATOMIC_BARRIER;                                       \
 394         return (v);                                             \
 395 }                                                               \
 396                                                                 \
 397 static __inline void                                            \
 398 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)       \
 399 {                                                               \
 400         __ATOMIC_BARRIER;                                       \
 401         *p = v;                                                 \
 402 }
 403
 404 ATOMIC_STORE_LOAD(char,         8)
 405 ATOMIC_STORE_LOAD(short,        16)
 406 ATOMIC_STORE_LOAD(int,          32)
 407
 408 #define atomic_load_acq_long    atomic_load_acq_32
 409 #define atomic_store_rel_long   atomic_store_rel_32
 410 #define atomic_load_acq_ptr     atomic_load_acq_32
 411 #define atomic_store_rel_ptr    atomic_store_rel_32
 412
 413 #undef ATOMIC_STORE_LOAD
 414
 415 /*
 416  * Atomically compare the value stored at *p with cmpval and if the
 417  * two values are equal, update the value of *p with newval. Returns
 418  * zero if the compare failed, nonzero otherwise.
 419  */
 420 static __inline uint32_t
 421 atomic_cmpset_32(volatile uint32_t* p, uint32_t cmpval, uint32_t newval)
 422 {
 423         uint32_t        ret;
 424
 425 #ifdef __GNUCLIKE_ASM
 426         __asm __volatile (
 427                 "1:\tlwarx %0, 0, %2\n\t"       /* load old value */
 428                 "cmplw %3, %0\n\t"              /* compare */
 429                 "bne 2f\n\t"                    /* exit if not equal */
 430                 "stwcx. %4, 0, %2\n\t"          /* attempt to store */
 431                 "bne- 1b\n\t"                   /* spin if failed */
 432                 "li %0, 1\n\t"                  /* success - retval = 1 */
 433                 "b 3f\n\t"                      /* we've succeeded */
 434                 "2:\n\t"
 435                 "stwcx. %0, 0, %2\n\t"          /* clear reservation (74xx) */
 436                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 437                 "3:\n\t"
 438                 : "=&r" (ret), "=m" (*p)
 439                 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
 440                 : "cc", "memory");
 441 #endif
 442
 443         return (ret);
 444 }
 445
 446 static __inline u_long
 447 atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval)
 448 {
 449         uint32_t        ret;
 450
 451 #ifdef __GNUCLIKE_ASM
 452         __asm __volatile (
 453                 "1:\tlwarx %0, 0, %2\n\t"       /* load old value */
 454                 "cmplw %3, %0\n\t"              /* compare */
 455                 "bne 2f\n\t"                    /* exit if not equal */
 456                 "stwcx. %4, 0, %2\n\t"          /* attempt to store */
 457                 "bne- 1b\n\t"                   /* spin if failed */
 458                 "li %0, 1\n\t"                  /* success - retval = 1 */
 459                 "b 3f\n\t"                      /* we've succeeded */
 460                 "2:\n\t"
 461                 "stwcx. %0, 0, %2\n\t"          /* clear reservation (74xx) */
 462                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 463                 "3:\n\t"
 464                 : "=&r" (ret), "=m" (*p)
 465                 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
 466                 : "cc", "memory");
 467 #endif
 468
 469         return (ret);
 470 }
 471
 472 #define atomic_cmpset_int       atomic_cmpset_32
 473
 474 #define atomic_cmpset_ptr(dst, old, new)        \
 475     atomic_cmpset_32((volatile u_int *)(dst), (u_int)(old), (u_int)(new))
 476
 477 static __inline uint32_t
 478 atomic_cmpset_acq_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)
 479 {
 480         int retval;
 481
 482         retval = atomic_cmpset_32(p, cmpval, newval);
 483         __ATOMIC_BARRIER;
 484         return (retval);
 485 }
 486
 487 static __inline uint32_t
 488 atomic_cmpset_rel_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)
 489 {
 490         __ATOMIC_BARRIER;
 491         return (atomic_cmpset_32(p, cmpval, newval));
 492 }
 493
 494 static __inline u_long
 495 atomic_cmpset_acq_long(volatile u_long *p, u_long cmpval, u_long newval)
 496 {
 497         int retval;
 498
 499         retval = atomic_cmpset_long(p, cmpval, newval);
 500         __ATOMIC_BARRIER;
 501         return (retval);
 502 }
 503
 504 static __inline uint32_t
 505 atomic_cmpset_rel_long(volatile u_long *p, u_long cmpval, u_long newval)
 506 {
 507         __ATOMIC_BARRIER;
 508         return (atomic_cmpset_long(p, cmpval, newval));
 509 }
 510
 511 #define atomic_cmpset_acq_int   atomic_cmpset_acq_32
 512 #define atomic_cmpset_rel_int   atomic_cmpset_rel_32
 513
 514 #define atomic_cmpset_acq_ptr(dst, old, new)    \
 515     atomic_cmpset_acq_32((volatile u_int *)(dst), (u_int)(old), (u_int)(new))
 516 #define atomic_cmpset_rel_ptr(dst, old, new)    \
 517     atomic_cmpset_rel_32((volatile u_int *)(dst), (u_int)(old), (u_int)(new))
 518
 519 static __inline uint32_t
 520 atomic_fetchadd_32(volatile uint32_t *p, uint32_t v)
 521 {
 522         uint32_t value;
 523
 524         do {
 525                 value = *p;
 526         } while (!atomic_cmpset_32(p, value, value + v));
 527         return (value);
 528 }
 529
 530 #define atomic_fetchadd_int     atomic_fetchadd_32
 531 #define atomic_fetchadd_long(p, v)      \
 532     (u_long)atomic_fetchadd_32((volatile u_int *)(p), (u_int)(v))
 533
 534 #endif /* ! _MACHINE_ATOMIC_H_ */