sys/powerpc/include/atomic.h

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2008 Marcel Moolenaar
   5  * Copyright (c) 2001 Benno Rice
   6  * Copyright (c) 2001 David E. O'Brien
   7  * Copyright (c) 1998 Doug Rabson
   8  * All rights reserved.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  29  * SUCH DAMAGE.
  30  *
  31  * $FreeBSD$
  32  */
  33
  34 #ifndef _MACHINE_ATOMIC_H_
  35 #define _MACHINE_ATOMIC_H_
  36
  37 #ifndef _SYS_CDEFS_H_
  38 #error this file needs sys/cdefs.h as a prerequisite
  39 #endif
  40
  41 /*
  42  * The __ATOMIC_REL/ACQ() macros provide memory barriers only in conjunction
  43  * with the atomic lXarx/stXcx. sequences below. They are not exposed outside
  44  * of this file. See also Appendix B.2 of Book II of the architecture manual.
  45  *
  46  * Note that not all Book-E processors accept the light-weight sync variant.
  47  * In particular, early models of E500 cores are known to wedge. Bank on all
  48  * 64-bit capable CPUs to accept lwsync properly and pressimize 32-bit CPUs
  49  * to use the heavier-weight sync.
  50  */
  51
  52 #ifdef __powerpc64__
  53 #define mb()            __asm __volatile("sync" : : : "memory")
  54 #define rmb()           __asm __volatile("lwsync" : : : "memory")
  55 #define wmb()           __asm __volatile("lwsync" : : : "memory")
  56 #define __ATOMIC_REL()  __asm __volatile("lwsync" : : : "memory")
  57 #define __ATOMIC_ACQ()  __asm __volatile("isync" : : : "memory")
  58 #else
  59 #define mb()            __asm __volatile("sync" : : : "memory")
  60 #define rmb()           __asm __volatile("sync" : : : "memory")
  61 #define wmb()           __asm __volatile("sync" : : : "memory")
  62 #define __ATOMIC_REL()  __asm __volatile("sync" : : : "memory")
  63 #define __ATOMIC_ACQ()  __asm __volatile("isync" : : : "memory")
  64 #endif
  65
  66 static __inline void
  67 powerpc_lwsync(void)
  68 {
  69
  70 #ifdef __powerpc64__
  71         __asm __volatile("lwsync" : : : "memory");
  72 #else
  73         __asm __volatile("sync" : : : "memory");
  74 #endif
  75 }
  76
  77 /*
  78  * atomic_add(p, v)
  79  * { *p += v; }
  80  */
  81
  82 #define __atomic_add_int(p, v, t)                               \
  83     __asm __volatile(                                           \
  84         "1:     lwarx   %0, 0, %2\n"                            \
  85         "       add     %0, %3, %0\n"                           \
  86         "       stwcx.  %0, 0, %2\n"                            \
  87         "       bne-    1b\n"                                   \
  88         : "=&r" (t), "=m" (*p)                                  \
  89         : "r" (p), "r" (v), "m" (*p)                            \
  90         : "cr0", "memory")                                      \
  91     /* __atomic_add_int */
  92
  93 #ifdef __powerpc64__
  94 #define __atomic_add_long(p, v, t)                              \
  95     __asm __volatile(                                           \
  96         "1:     ldarx   %0, 0, %2\n"                            \
  97         "       add     %0, %3, %0\n"                           \
  98         "       stdcx.  %0, 0, %2\n"                            \
  99         "       bne-    1b\n"                                   \
 100         : "=&r" (t), "=m" (*p)                                  \
 101         : "r" (p), "r" (v), "m" (*p)                            \
 102         : "cr0", "memory")                                      \
 103     /* __atomic_add_long */
 104 #else
 105 #define __atomic_add_long(p, v, t)                              \
 106     __asm __volatile(                                           \
 107         "1:     lwarx   %0, 0, %2\n"                            \
 108         "       add     %0, %3, %0\n"                           \
 109         "       stwcx.  %0, 0, %2\n"                            \
 110         "       bne-    1b\n"                                   \
 111         : "=&r" (t), "=m" (*p)                                  \
 112         : "r" (p), "r" (v), "m" (*p)                            \
 113         : "cr0", "memory")                                      \
 114     /* __atomic_add_long */
 115 #endif
 116
 117 #define _ATOMIC_ADD(type)                                       \
 118     static __inline void                                        \
 119     atomic_add_##type(volatile u_##type *p, u_##type v) {       \
 120         u_##type t;                                             \
 121         __atomic_add_##type(p, v, t);                           \
 122     }                                                           \
 123                                                                 \
 124     static __inline void                                        \
 125     atomic_add_acq_##type(volatile u_##type *p, u_##type v) {   \
 126         u_##type t;                                             \
 127         __atomic_add_##type(p, v, t);                           \
 128         __ATOMIC_ACQ();                                         \
 129     }                                                           \
 130                                                                 \
 131     static __inline void                                        \
 132     atomic_add_rel_##type(volatile u_##type *p, u_##type v) {   \
 133         u_##type t;                                             \
 134         __ATOMIC_REL();                                         \
 135         __atomic_add_##type(p, v, t);                           \
 136     }                                                           \
 137     /* _ATOMIC_ADD */
 138
 139 _ATOMIC_ADD(int)
 140 _ATOMIC_ADD(long)
 141
 142 #define atomic_add_32           atomic_add_int
 143 #define atomic_add_acq_32       atomic_add_acq_int
 144 #define atomic_add_rel_32       atomic_add_rel_int
 145
 146 #ifdef __powerpc64__
 147 #define atomic_add_64           atomic_add_long
 148 #define atomic_add_acq_64       atomic_add_acq_long
 149 #define atomic_add_rel_64       atomic_add_rel_long
 150
 151 #define atomic_add_ptr          atomic_add_long
 152 #define atomic_add_acq_ptr      atomic_add_acq_long
 153 #define atomic_add_rel_ptr      atomic_add_rel_long
 154 #else
 155 #define atomic_add_ptr          atomic_add_int
 156 #define atomic_add_acq_ptr      atomic_add_acq_int
 157 #define atomic_add_rel_ptr      atomic_add_rel_int
 158 #endif
 159 #undef _ATOMIC_ADD
 160 #undef __atomic_add_long
 161 #undef __atomic_add_int
 162
 163 /*
 164  * atomic_clear(p, v)
 165  * { *p &= ~v; }
 166  */
 167
 168 #define __atomic_clear_int(p, v, t)                             \
 169     __asm __volatile(                                           \
 170         "1:     lwarx   %0, 0, %2\n"                            \
 171         "       andc    %0, %0, %3\n"                           \
 172         "       stwcx.  %0, 0, %2\n"                            \
 173         "       bne-    1b\n"                                   \
 174         : "=&r" (t), "=m" (*p)                                  \
 175         : "r" (p), "r" (v), "m" (*p)                            \
 176         : "cr0", "memory")                                      \
 177     /* __atomic_clear_int */
 178
 179 #ifdef __powerpc64__
 180 #define __atomic_clear_long(p, v, t)                            \
 181     __asm __volatile(                                           \
 182         "1:     ldarx   %0, 0, %2\n"                            \
 183         "       andc    %0, %0, %3\n"                           \
 184         "       stdcx.  %0, 0, %2\n"                            \
 185         "       bne-    1b\n"                                   \
 186         : "=&r" (t), "=m" (*p)                                  \
 187         : "r" (p), "r" (v), "m" (*p)                            \
 188         : "cr0", "memory")                                      \
 189     /* __atomic_clear_long */
 190 #else
 191 #define __atomic_clear_long(p, v, t)                            \
 192     __asm __volatile(                                           \
 193         "1:     lwarx   %0, 0, %2\n"                            \
 194         "       andc    %0, %0, %3\n"                           \
 195         "       stwcx.  %0, 0, %2\n"                            \
 196         "       bne-    1b\n"                                   \
 197         : "=&r" (t), "=m" (*p)                                  \
 198         : "r" (p), "r" (v), "m" (*p)                            \
 199         : "cr0", "memory")                                      \
 200     /* __atomic_clear_long */
 201 #endif
 202
 203 #define _ATOMIC_CLEAR(type)                                     \
 204     static __inline void                                        \
 205     atomic_clear_##type(volatile u_##type *p, u_##type v) {     \
 206         u_##type t;                                             \
 207         __atomic_clear_##type(p, v, t);                         \
 208     }                                                           \
 209                                                                 \
 210     static __inline void                                        \
 211     atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \
 212         u_##type t;                                             \
 213         __atomic_clear_##type(p, v, t);                         \
 214         __ATOMIC_ACQ();                                         \
 215     }                                                           \
 216                                                                 \
 217     static __inline void                                        \
 218     atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \
 219         u_##type t;                                             \
 220         __ATOMIC_REL();                                         \
 221         __atomic_clear_##type(p, v, t);                         \
 222     }                                                           \
 223     /* _ATOMIC_CLEAR */
 224
 225
 226 _ATOMIC_CLEAR(int)
 227 _ATOMIC_CLEAR(long)
 228
 229 #define atomic_clear_32         atomic_clear_int
 230 #define atomic_clear_acq_32     atomic_clear_acq_int
 231 #define atomic_clear_rel_32     atomic_clear_rel_int
 232
 233 #ifdef __powerpc64__
 234 #define atomic_clear_64         atomic_clear_long
 235 #define atomic_clear_acq_64     atomic_clear_acq_long
 236 #define atomic_clear_rel_64     atomic_clear_rel_long
 237
 238 #define atomic_clear_ptr        atomic_clear_long
 239 #define atomic_clear_acq_ptr    atomic_clear_acq_long
 240 #define atomic_clear_rel_ptr    atomic_clear_rel_long
 241 #else
 242 #define atomic_clear_ptr        atomic_clear_int
 243 #define atomic_clear_acq_ptr    atomic_clear_acq_int
 244 #define atomic_clear_rel_ptr    atomic_clear_rel_int
 245 #endif
 246 #undef _ATOMIC_CLEAR
 247 #undef __atomic_clear_long
 248 #undef __atomic_clear_int
 249
 250 /*
 251  * atomic_cmpset(p, o, n)
 252  */
 253 /* TODO -- see below */
 254
 255 /*
 256  * atomic_load_acq(p)
 257  */
 258 /* TODO -- see below */
 259
 260 /*
 261  * atomic_readandclear(p)
 262  */
 263 /* TODO -- see below */
 264
 265 /*
 266  * atomic_set(p, v)
 267  * { *p |= v; }
 268  */
 269
 270 #define __atomic_set_int(p, v, t)                               \
 271     __asm __volatile(                                           \
 272         "1:     lwarx   %0, 0, %2\n"                            \
 273         "       or      %0, %3, %0\n"                           \
 274         "       stwcx.  %0, 0, %2\n"                            \
 275         "       bne-    1b\n"                                   \
 276         : "=&r" (t), "=m" (*p)                                  \
 277         : "r" (p), "r" (v), "m" (*p)                            \
 278         : "cr0", "memory")                                      \
 279     /* __atomic_set_int */
 280
 281 #ifdef __powerpc64__
 282 #define __atomic_set_long(p, v, t)                              \
 283     __asm __volatile(                                           \
 284         "1:     ldarx   %0, 0, %2\n"                            \
 285         "       or      %0, %3, %0\n"                           \
 286         "       stdcx.  %0, 0, %2\n"                            \
 287         "       bne-    1b\n"                                   \
 288         : "=&r" (t), "=m" (*p)                                  \
 289         : "r" (p), "r" (v), "m" (*p)                            \
 290         : "cr0", "memory")                                      \
 291     /* __atomic_set_long */
 292 #else
 293 #define __atomic_set_long(p, v, t)                              \
 294     __asm __volatile(                                           \
 295         "1:     lwarx   %0, 0, %2\n"                            \
 296         "       or      %0, %3, %0\n"                           \
 297         "       stwcx.  %0, 0, %2\n"                            \
 298         "       bne-    1b\n"                                   \
 299         : "=&r" (t), "=m" (*p)                                  \
 300         : "r" (p), "r" (v), "m" (*p)                            \
 301         : "cr0", "memory")                                      \
 302     /* __atomic_set_long */
 303 #endif
 304
 305 #define _ATOMIC_SET(type)                                       \
 306     static __inline void                                        \
 307     atomic_set_##type(volatile u_##type *p, u_##type v) {       \
 308         u_##type t;                                             \
 309         __atomic_set_##type(p, v, t);                           \
 310     }                                                           \
 311                                                                 \
 312     static __inline void                                        \
 313     atomic_set_acq_##type(volatile u_##type *p, u_##type v) {   \
 314         u_##type t;                                             \
 315         __atomic_set_##type(p, v, t);                           \
 316         __ATOMIC_ACQ();                                         \
 317     }                                                           \
 318                                                                 \
 319     static __inline void                                        \
 320     atomic_set_rel_##type(volatile u_##type *p, u_##type v) {   \
 321         u_##type t;                                             \
 322         __ATOMIC_REL();                                         \
 323         __atomic_set_##type(p, v, t);                           \
 324     }                                                           \
 325     /* _ATOMIC_SET */
 326
 327 _ATOMIC_SET(int)
 328 _ATOMIC_SET(long)
 329
 330 #define atomic_set_32           atomic_set_int
 331 #define atomic_set_acq_32       atomic_set_acq_int
 332 #define atomic_set_rel_32       atomic_set_rel_int
 333
 334 #ifdef __powerpc64__
 335 #define atomic_set_64           atomic_set_long
 336 #define atomic_set_acq_64       atomic_set_acq_long
 337 #define atomic_set_rel_64       atomic_set_rel_long
 338
 339 #define atomic_set_ptr          atomic_set_long
 340 #define atomic_set_acq_ptr      atomic_set_acq_long
 341 #define atomic_set_rel_ptr      atomic_set_rel_long
 342 #else
 343 #define atomic_set_ptr          atomic_set_int
 344 #define atomic_set_acq_ptr      atomic_set_acq_int
 345 #define atomic_set_rel_ptr      atomic_set_rel_int
 346 #endif
 347 #undef _ATOMIC_SET
 348 #undef __atomic_set_long
 349 #undef __atomic_set_int
 350
 351 /*
 352  * atomic_subtract(p, v)
 353  * { *p -= v; }
 354  */
 355
 356 #define __atomic_subtract_int(p, v, t)                          \
 357     __asm __volatile(                                           \
 358         "1:     lwarx   %0, 0, %2\n"                            \
 359         "       subf    %0, %3, %0\n"                           \
 360         "       stwcx.  %0, 0, %2\n"                            \
 361         "       bne-    1b\n"                                   \
 362         : "=&r" (t), "=m" (*p)                                  \
 363         : "r" (p), "r" (v), "m" (*p)                            \
 364         : "cr0", "memory")                                      \
 365     /* __atomic_subtract_int */
 366
 367 #ifdef __powerpc64__
 368 #define __atomic_subtract_long(p, v, t)                         \
 369     __asm __volatile(                                           \
 370         "1:     ldarx   %0, 0, %2\n"                            \
 371         "       subf    %0, %3, %0\n"                           \
 372         "       stdcx.  %0, 0, %2\n"                            \
 373         "       bne-    1b\n"                                   \
 374         : "=&r" (t), "=m" (*p)                                  \
 375         : "r" (p), "r" (v), "m" (*p)                            \
 376         : "cr0", "memory")                                      \
 377     /* __atomic_subtract_long */
 378 #else
 379 #define __atomic_subtract_long(p, v, t)                         \
 380     __asm __volatile(                                           \
 381         "1:     lwarx   %0, 0, %2\n"                            \
 382         "       subf    %0, %3, %0\n"                           \
 383         "       stwcx.  %0, 0, %2\n"                            \
 384         "       bne-    1b\n"                                   \
 385         : "=&r" (t), "=m" (*p)                                  \
 386         : "r" (p), "r" (v), "m" (*p)                            \
 387         : "cr0", "memory")                                      \
 388     /* __atomic_subtract_long */
 389 #endif
 390
 391 #define _ATOMIC_SUBTRACT(type)                                          \
 392     static __inline void                                                \
 393     atomic_subtract_##type(volatile u_##type *p, u_##type v) {          \
 394         u_##type t;                                                     \
 395         __atomic_subtract_##type(p, v, t);                              \
 396     }                                                                   \
 397                                                                         \
 398     static __inline void                                                \
 399     atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) {      \
 400         u_##type t;                                                     \
 401         __atomic_subtract_##type(p, v, t);                              \
 402         __ATOMIC_ACQ();                                                 \
 403     }                                                                   \
 404                                                                         \
 405     static __inline void                                                \
 406     atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) {      \
 407         u_##type t;                                                     \
 408         __ATOMIC_REL();                                                 \
 409         __atomic_subtract_##type(p, v, t);                              \
 410     }                                                                   \
 411     /* _ATOMIC_SUBTRACT */
 412
 413 _ATOMIC_SUBTRACT(int)
 414 _ATOMIC_SUBTRACT(long)
 415
 416 #define atomic_subtract_32      atomic_subtract_int
 417 #define atomic_subtract_acq_32  atomic_subtract_acq_int
 418 #define atomic_subtract_rel_32  atomic_subtract_rel_int
 419
 420 #ifdef __powerpc64__
 421 #define atomic_subtract_64      atomic_subtract_long
 422 #define atomic_subtract_acq_64  atomic_subract_acq_long
 423 #define atomic_subtract_rel_64  atomic_subtract_rel_long
 424
 425 #define atomic_subtract_ptr     atomic_subtract_long
 426 #define atomic_subtract_acq_ptr atomic_subtract_acq_long
 427 #define atomic_subtract_rel_ptr atomic_subtract_rel_long
 428 #else
 429 #define atomic_subtract_ptr     atomic_subtract_int
 430 #define atomic_subtract_acq_ptr atomic_subtract_acq_int
 431 #define atomic_subtract_rel_ptr atomic_subtract_rel_int
 432 #endif
 433 #undef _ATOMIC_SUBTRACT
 434 #undef __atomic_subtract_long
 435 #undef __atomic_subtract_int
 436
 437 /*
 438  * atomic_store_rel(p, v)
 439  */
 440 /* TODO -- see below */
 441
 442 /*
 443  * Old/original implementations that still need revisiting.
 444  */
 445
 446 static __inline u_int
 447 atomic_readandclear_int(volatile u_int *addr)
 448 {
 449         u_int result,temp;
 450
 451         __asm __volatile (
 452                 "\tsync\n"                      /* drain writes */
 453                 "1:\tlwarx %0, 0, %3\n\t"       /* load old value */
 454                 "li %1, 0\n\t"                  /* load new value */
 455                 "stwcx. %1, 0, %3\n\t"          /* attempt to store */
 456                 "bne- 1b\n\t"                   /* spin if failed */
 457                 : "=&r"(result), "=&r"(temp), "=m" (*addr)
 458                 : "r" (addr), "m" (*addr)
 459                 : "cr0", "memory");
 460
 461         return (result);
 462 }
 463
 464 #ifdef __powerpc64__
 465 static __inline u_long
 466 atomic_readandclear_long(volatile u_long *addr)
 467 {
 468         u_long result,temp;
 469
 470         __asm __volatile (
 471                 "\tsync\n"                      /* drain writes */
 472                 "1:\tldarx %0, 0, %3\n\t"       /* load old value */
 473                 "li %1, 0\n\t"                  /* load new value */
 474                 "stdcx. %1, 0, %3\n\t"          /* attempt to store */
 475                 "bne- 1b\n\t"                   /* spin if failed */
 476                 : "=&r"(result), "=&r"(temp), "=m" (*addr)
 477                 : "r" (addr), "m" (*addr)
 478                 : "cr0", "memory");
 479
 480         return (result);
 481 }
 482 #endif
 483
 484 #define atomic_readandclear_32          atomic_readandclear_int
 485
 486 #ifdef __powerpc64__
 487 #define atomic_readandclear_64          atomic_readandclear_long
 488
 489 #define atomic_readandclear_ptr         atomic_readandclear_long
 490 #else
 491 static __inline u_long
 492 atomic_readandclear_long(volatile u_long *addr)
 493 {
 494
 495         return ((u_long)atomic_readandclear_int((volatile u_int *)addr));
 496 }
 497
 498 #define atomic_readandclear_ptr         atomic_readandclear_int
 499 #endif
 500
 501 /*
 502  * We assume that a = b will do atomic loads and stores.
 503  */
 504 #define ATOMIC_STORE_LOAD(TYPE)                                 \
 505 static __inline u_##TYPE                                        \
 506 atomic_load_acq_##TYPE(volatile u_##TYPE *p)                    \
 507 {                                                               \
 508         u_##TYPE v;                                             \
 509                                                                 \
 510         v = *p;                                                 \
 511         mb();                                                   \
 512         return (v);                                             \
 513 }                                                               \
 514                                                                 \
 515 static __inline void                                            \
 516 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)       \
 517 {                                                               \
 518                                                                 \
 519         powerpc_lwsync();                                       \
 520         *p = v;                                                 \
 521 }
 522
 523 ATOMIC_STORE_LOAD(int)
 524
 525 #define atomic_load_acq_32      atomic_load_acq_int
 526 #define atomic_store_rel_32     atomic_store_rel_int
 527
 528 #ifdef __powerpc64__
 529 ATOMIC_STORE_LOAD(long)
 530
 531 #define atomic_load_acq_64      atomic_load_acq_long
 532 #define atomic_store_rel_64     atomic_store_rel_long
 533
 534 #define atomic_load_acq_ptr     atomic_load_acq_long
 535 #define atomic_store_rel_ptr    atomic_store_rel_long
 536 #else
 537 static __inline u_long
 538 atomic_load_acq_long(volatile u_long *addr)
 539 {
 540
 541         return ((u_long)atomic_load_acq_int((volatile u_int *)addr));
 542 }
 543
 544 static __inline void
 545 atomic_store_rel_long(volatile u_long *addr, u_long val)
 546 {
 547
 548         atomic_store_rel_int((volatile u_int *)addr, (u_int)val);
 549 }
 550
 551 #define atomic_load_acq_ptr     atomic_load_acq_int
 552 #define atomic_store_rel_ptr    atomic_store_rel_int
 553 #endif
 554 #undef ATOMIC_STORE_LOAD
 555
 556 /*
 557  * Atomically compare the value stored at *p with cmpval and if the
 558  * two values are equal, update the value of *p with newval. Returns
 559  * zero if the compare failed, nonzero otherwise.
 560  */
 561 static __inline int
 562 atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval)
 563 {
 564         int     ret;
 565
 566         __asm __volatile (
 567                 "1:\tlwarx %0, 0, %2\n\t"       /* load old value */
 568                 "cmplw %3, %0\n\t"              /* compare */
 569                 "bne 2f\n\t"                    /* exit if not equal */
 570                 "stwcx. %4, 0, %2\n\t"          /* attempt to store */
 571                 "bne- 1b\n\t"                   /* spin if failed */
 572                 "li %0, 1\n\t"                  /* success - retval = 1 */
 573                 "b 3f\n\t"                      /* we've succeeded */
 574                 "2:\n\t"
 575                 "stwcx. %0, 0, %2\n\t"          /* clear reservation (74xx) */
 576                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 577                 "3:\n\t"
 578                 : "=&r" (ret), "=m" (*p)
 579                 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
 580                 : "cr0", "memory");
 581
 582         return (ret);
 583 }
 584 static __inline int
 585 atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval)
 586 {
 587         int ret;
 588
 589         __asm __volatile (
 590             #ifdef __powerpc64__
 591                 "1:\tldarx %0, 0, %2\n\t"       /* load old value */
 592                 "cmpld %3, %0\n\t"              /* compare */
 593                 "bne 2f\n\t"                    /* exit if not equal */
 594                 "stdcx. %4, 0, %2\n\t"          /* attempt to store */
 595             #else
 596                 "1:\tlwarx %0, 0, %2\n\t"       /* load old value */
 597                 "cmplw %3, %0\n\t"              /* compare */
 598                 "bne 2f\n\t"                    /* exit if not equal */
 599                 "stwcx. %4, 0, %2\n\t"          /* attempt to store */
 600             #endif
 601                 "bne- 1b\n\t"                   /* spin if failed */
 602                 "li %0, 1\n\t"                  /* success - retval = 1 */
 603                 "b 3f\n\t"                      /* we've succeeded */
 604                 "2:\n\t"
 605             #ifdef __powerpc64__
 606                 "stdcx. %0, 0, %2\n\t"          /* clear reservation (74xx) */
 607             #else
 608                 "stwcx. %0, 0, %2\n\t"          /* clear reservation (74xx) */
 609             #endif
 610                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 611                 "3:\n\t"
 612                 : "=&r" (ret), "=m" (*p)
 613                 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
 614                 : "cr0", "memory");
 615
 616         return (ret);
 617 }
 618
 619 static __inline int
 620 atomic_cmpset_acq_int(volatile u_int *p, u_int cmpval, u_int newval)
 621 {
 622         int retval;
 623
 624         retval = atomic_cmpset_int(p, cmpval, newval);
 625         __ATOMIC_ACQ();
 626         return (retval);
 627 }
 628
 629 static __inline int
 630 atomic_cmpset_rel_int(volatile u_int *p, u_int cmpval, u_int newval)
 631 {
 632         __ATOMIC_REL();
 633         return (atomic_cmpset_int(p, cmpval, newval));
 634 }
 635
 636 static __inline int
 637 atomic_cmpset_acq_long(volatile u_long *p, u_long cmpval, u_long newval)
 638 {
 639         u_long retval;
 640
 641         retval = atomic_cmpset_long(p, cmpval, newval);
 642         __ATOMIC_ACQ();
 643         return (retval);
 644 }
 645
 646 static __inline int
 647 atomic_cmpset_rel_long(volatile u_long *p, u_long cmpval, u_long newval)
 648 {
 649         __ATOMIC_REL();
 650         return (atomic_cmpset_long(p, cmpval, newval));
 651 }
 652
 653 #define atomic_cmpset_32        atomic_cmpset_int
 654 #define atomic_cmpset_acq_32    atomic_cmpset_acq_int
 655 #define atomic_cmpset_rel_32    atomic_cmpset_rel_int
 656
 657 #ifdef __powerpc64__
 658 #define atomic_cmpset_64        atomic_cmpset_long
 659 #define atomic_cmpset_acq_64    atomic_cmpset_acq_long
 660 #define atomic_cmpset_rel_64    atomic_cmpset_rel_long
 661
 662 #define atomic_cmpset_ptr       atomic_cmpset_long
 663 #define atomic_cmpset_acq_ptr   atomic_cmpset_acq_long
 664 #define atomic_cmpset_rel_ptr   atomic_cmpset_rel_long
 665 #else
 666 #define atomic_cmpset_ptr       atomic_cmpset_int
 667 #define atomic_cmpset_acq_ptr   atomic_cmpset_acq_int
 668 #define atomic_cmpset_rel_ptr   atomic_cmpset_rel_int
 669 #endif
 670
 671 /*
 672  * Atomically compare the value stored at *p with *cmpval and if the
 673  * two values are equal, update the value of *p with newval. Returns
 674  * zero if the compare failed and sets *cmpval to the read value from *p,
 675  * nonzero otherwise.
 676  */
 677 static __inline int
 678 atomic_fcmpset_int(volatile u_int *p, u_int *cmpval, u_int newval)
 679 {
 680         int     ret;
 681
 682         __asm __volatile (
 683                 "lwarx %0, 0, %3\n\t"   /* load old value */
 684                 "cmplw %4, %0\n\t"              /* compare */
 685                 "bne 1f\n\t"                    /* exit if not equal */
 686                 "stwcx. %5, 0, %3\n\t"          /* attempt to store */
 687                 "bne- 1f\n\t"                   /* exit if failed */
 688                 "li %0, 1\n\t"                  /* success - retval = 1 */
 689                 "b 2f\n\t"                      /* we've succeeded */
 690                 "1:\n\t"
 691                 "stwcx. %0, 0, %3\n\t"          /* clear reservation (74xx) */
 692                 "stwx %0, 0, %7\n\t"
 693                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 694                 "2:\n\t"
 695                 : "=&r" (ret), "=m" (*p), "=m" (*cmpval)
 696                 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval)
 697                 : "cr0", "memory");
 698
 699         return (ret);
 700 }
 701 static __inline int
 702 atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval)
 703 {
 704         int ret;
 705
 706         __asm __volatile (
 707             #ifdef __powerpc64__
 708                 "ldarx %0, 0, %3\n\t"   /* load old value */
 709                 "cmpld %4, %0\n\t"              /* compare */
 710                 "bne 1f\n\t"                    /* exit if not equal */
 711                 "stdcx. %5, 0, %3\n\t"          /* attempt to store */
 712             #else
 713                 "lwarx %0, 0, %3\n\t"   /* load old value */
 714                 "cmplw %4, %0\n\t"              /* compare */
 715                 "bne 1f\n\t"                    /* exit if not equal */
 716                 "stwcx. %5, 0, %3\n\t"          /* attempt to store */
 717             #endif
 718                 "bne- 1f\n\t"                   /* exit if failed */
 719                 "li %0, 1\n\t"                  /* success - retval = 1 */
 720                 "b 2f\n\t"                      /* we've succeeded */
 721                 "1:\n\t"
 722             #ifdef __powerpc64__
 723                 "stdcx. %0, 0, %3\n\t"          /* clear reservation (74xx) */
 724                 "stdx %0, 0, %7\n\t"
 725             #else
 726                 "stwcx. %0, 0, %3\n\t"          /* clear reservation (74xx) */
 727                 "stwx %0, 0, %7\n\t"
 728             #endif
 729                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 730                 "2:\n\t"
 731                 : "=&r" (ret), "=m" (*p), "=m" (*cmpval)
 732                 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval)
 733                 : "cr0", "memory");
 734
 735         return (ret);
 736 }
 737
 738 static __inline int
 739 atomic_fcmpset_acq_int(volatile u_int *p, u_int *cmpval, u_int newval)
 740 {
 741         int retval;
 742
 743         retval = atomic_fcmpset_int(p, cmpval, newval);
 744         __ATOMIC_ACQ();
 745         return (retval);
 746 }
 747
 748 static __inline int
 749 atomic_fcmpset_rel_int(volatile u_int *p, u_int *cmpval, u_int newval)
 750 {
 751         __ATOMIC_REL();
 752         return (atomic_fcmpset_int(p, cmpval, newval));
 753 }
 754
 755 static __inline int
 756 atomic_fcmpset_acq_long(volatile u_long *p, u_long *cmpval, u_long newval)
 757 {
 758         u_long retval;
 759
 760         retval = atomic_fcmpset_long(p, cmpval, newval);
 761         __ATOMIC_ACQ();
 762         return (retval);
 763 }
 764
 765 static __inline int
 766 atomic_fcmpset_rel_long(volatile u_long *p, u_long *cmpval, u_long newval)
 767 {
 768         __ATOMIC_REL();
 769         return (atomic_fcmpset_long(p, cmpval, newval));
 770 }
 771
 772 #define atomic_fcmpset_32       atomic_fcmpset_int
 773 #define atomic_fcmpset_acq_32   atomic_fcmpset_acq_int
 774 #define atomic_fcmpset_rel_32   atomic_fcmpset_rel_int
 775
 776 #ifdef __powerpc64__
 777 #define atomic_fcmpset_64       atomic_fcmpset_long
 778 #define atomic_fcmpset_acq_64   atomic_fcmpset_acq_long
 779 #define atomic_fcmpset_rel_64   atomic_fcmpset_rel_long
 780
 781 #define atomic_fcmpset_ptr      atomic_fcmpset_long
 782 #define atomic_fcmpset_acq_ptr  atomic_fcmpset_acq_long
 783 #define atomic_fcmpset_rel_ptr  atomic_fcmpset_rel_long
 784 #else
 785 #define atomic_fcmpset_ptr      atomic_fcmpset_int
 786 #define atomic_fcmpset_acq_ptr  atomic_fcmpset_acq_int
 787 #define atomic_fcmpset_rel_ptr  atomic_fcmpset_rel_int
 788 #endif
 789
 790 static __inline u_int
 791 atomic_fetchadd_int(volatile u_int *p, u_int v)
 792 {
 793         u_int value;
 794
 795         do {
 796                 value = *p;
 797         } while (!atomic_cmpset_int(p, value, value + v));
 798         return (value);
 799 }
 800
 801 static __inline u_long
 802 atomic_fetchadd_long(volatile u_long *p, u_long v)
 803 {
 804         u_long value;
 805
 806         do {
 807                 value = *p;
 808         } while (!atomic_cmpset_long(p, value, value + v));
 809         return (value);
 810 }
 811
 812 static __inline u_int
 813 atomic_swap_32(volatile u_int *p, u_int v)
 814 {
 815         u_int prev;
 816
 817         __asm __volatile(
 818         "1:     lwarx   %0,0,%2\n"
 819         "       stwcx.  %3,0,%2\n"
 820         "       bne-    1b\n"
 821         : "=&r" (prev), "+m" (*(volatile u_int *)p)
 822         : "r" (p), "r" (v)
 823         : "cr0", "memory");
 824
 825         return (prev);
 826 }
 827
 828 #ifdef __powerpc64__
 829 static __inline u_long
 830 atomic_swap_64(volatile u_long *p, u_long v)
 831 {
 832         u_long prev;
 833
 834         __asm __volatile(
 835         "1:     ldarx   %0,0,%2\n"
 836         "       stdcx.  %3,0,%2\n"
 837         "       bne-    1b\n"
 838         : "=&r" (prev), "+m" (*(volatile u_long *)p)
 839         : "r" (p), "r" (v)
 840         : "cr0", "memory");
 841
 842         return (prev);
 843 }
 844 #endif
 845
 846 #define atomic_fetchadd_32      atomic_fetchadd_int
 847 #define atomic_swap_int         atomic_swap_32
 848
 849 #ifdef __powerpc64__
 850 #define atomic_fetchadd_64      atomic_fetchadd_long
 851 #define atomic_swap_long        atomic_swap_64
 852 #define atomic_swap_ptr         atomic_swap_64
 853 #endif
 854
 855 #undef __ATOMIC_REL
 856 #undef __ATOMIC_ACQ
 857
 858 static __inline void
 859 atomic_thread_fence_acq(void)
 860 {
 861
 862         powerpc_lwsync();
 863 }
 864
 865 static __inline void
 866 atomic_thread_fence_rel(void)
 867 {
 868
 869         powerpc_lwsync();
 870 }
 871
 872 static __inline void
 873 atomic_thread_fence_acq_rel(void)
 874 {
 875
 876         powerpc_lwsync();
 877 }
 878
 879 static __inline void
 880 atomic_thread_fence_seq_cst(void)
 881 {
 882
 883         __asm __volatile("sync" : : : "memory");
 884 }
 885
 886 #endif /* ! _MACHINE_ATOMIC_H_ */