sys/powerpc/include/atomic.h

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2008 Marcel Moolenaar
   5  * Copyright (c) 2001 Benno Rice
   6  * Copyright (c) 2001 David E. O'Brien
   7  * Copyright (c) 1998 Doug Rabson
   8  * All rights reserved.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  29  * SUCH DAMAGE.
  30  *
  31  * $FreeBSD$
  32  */
  33
  34 #ifndef _MACHINE_ATOMIC_H_
  35 #define _MACHINE_ATOMIC_H_
  36
  37 #ifndef _SYS_CDEFS_H_
  38 #error this file needs sys/cdefs.h as a prerequisite
  39 #endif
  40
  41 #include <sys/atomic_common.h>
  42
  43 /*
  44  * The __ATOMIC_REL/ACQ() macros provide memory barriers only in conjunction
  45  * with the atomic lXarx/stXcx. sequences below. They are not exposed outside
  46  * of this file. See also Appendix B.2 of Book II of the architecture manual.
  47  *
  48  * Note that not all Book-E processors accept the light-weight sync variant.
  49  * In particular, early models of E500 cores are known to wedge. Bank on all
  50  * 64-bit capable CPUs to accept lwsync properly and pressimize 32-bit CPUs
  51  * to use the heavier-weight sync.
  52  */
  53
  54 #ifdef __powerpc64__
  55 #define mb()            __asm __volatile("sync" : : : "memory")
  56 #define rmb()           __asm __volatile("lwsync" : : : "memory")
  57 #define wmb()           __asm __volatile("lwsync" : : : "memory")
  58 #define __ATOMIC_REL()  __asm __volatile("lwsync" : : : "memory")
  59 #define __ATOMIC_ACQ()  __asm __volatile("isync" : : : "memory")
  60 #else
  61 #define mb()            __asm __volatile("sync" : : : "memory")
  62 #define rmb()           __asm __volatile("sync" : : : "memory")
  63 #define wmb()           __asm __volatile("sync" : : : "memory")
  64 #define __ATOMIC_REL()  __asm __volatile("sync" : : : "memory")
  65 #define __ATOMIC_ACQ()  __asm __volatile("isync" : : : "memory")
  66 #endif
  67
  68 static __inline void
  69 powerpc_lwsync(void)
  70 {
  71
  72 #ifdef __powerpc64__
  73         __asm __volatile("lwsync" : : : "memory");
  74 #else
  75         __asm __volatile("sync" : : : "memory");
  76 #endif
  77 }
  78
  79 /*
  80  * atomic_add(p, v)
  81  * { *p += v; }
  82  */
  83
  84 #define __atomic_add_int(p, v, t)                               \
  85     __asm __volatile(                                           \
  86         "1:     lwarx   %0, 0, %2\n"                            \
  87         "       add     %0, %3, %0\n"                           \
  88         "       stwcx.  %0, 0, %2\n"                            \
  89         "       bne-    1b\n"                                   \
  90         : "=&r" (t), "=m" (*p)                                  \
  91         : "r" (p), "r" (v), "m" (*p)                            \
  92         : "cr0", "memory")                                      \
  93     /* __atomic_add_int */
  94
  95 #ifdef __powerpc64__
  96 #define __atomic_add_long(p, v, t)                              \
  97     __asm __volatile(                                           \
  98         "1:     ldarx   %0, 0, %2\n"                            \
  99         "       add     %0, %3, %0\n"                           \
 100         "       stdcx.  %0, 0, %2\n"                            \
 101         "       bne-    1b\n"                                   \
 102         : "=&r" (t), "=m" (*p)                                  \
 103         : "r" (p), "r" (v), "m" (*p)                            \
 104         : "cr0", "memory")                                      \
 105     /* __atomic_add_long */
 106 #else
 107 #define __atomic_add_long(p, v, t)                              \
 108     __asm __volatile(                                           \
 109         "1:     lwarx   %0, 0, %2\n"                            \
 110         "       add     %0, %3, %0\n"                           \
 111         "       stwcx.  %0, 0, %2\n"                            \
 112         "       bne-    1b\n"                                   \
 113         : "=&r" (t), "=m" (*p)                                  \
 114         : "r" (p), "r" (v), "m" (*p)                            \
 115         : "cr0", "memory")                                      \
 116     /* __atomic_add_long */
 117 #endif
 118
 119 #define _ATOMIC_ADD(type)                                       \
 120     static __inline void                                        \
 121     atomic_add_##type(volatile u_##type *p, u_##type v) {       \
 122         u_##type t;                                             \
 123         __atomic_add_##type(p, v, t);                           \
 124     }                                                           \
 125                                                                 \
 126     static __inline void                                        \
 127     atomic_add_acq_##type(volatile u_##type *p, u_##type v) {   \
 128         u_##type t;                                             \
 129         __atomic_add_##type(p, v, t);                           \
 130         __ATOMIC_ACQ();                                         \
 131     }                                                           \
 132                                                                 \
 133     static __inline void                                        \
 134     atomic_add_rel_##type(volatile u_##type *p, u_##type v) {   \
 135         u_##type t;                                             \
 136         __ATOMIC_REL();                                         \
 137         __atomic_add_##type(p, v, t);                           \
 138     }                                                           \
 139     /* _ATOMIC_ADD */
 140
 141 _ATOMIC_ADD(int)
 142 _ATOMIC_ADD(long)
 143
 144 #define atomic_add_32           atomic_add_int
 145 #define atomic_add_acq_32       atomic_add_acq_int
 146 #define atomic_add_rel_32       atomic_add_rel_int
 147
 148 #ifdef __powerpc64__
 149 #define atomic_add_64           atomic_add_long
 150 #define atomic_add_acq_64       atomic_add_acq_long
 151 #define atomic_add_rel_64       atomic_add_rel_long
 152
 153 #define atomic_add_ptr          atomic_add_long
 154 #define atomic_add_acq_ptr      atomic_add_acq_long
 155 #define atomic_add_rel_ptr      atomic_add_rel_long
 156 #else
 157 #define atomic_add_ptr          atomic_add_int
 158 #define atomic_add_acq_ptr      atomic_add_acq_int
 159 #define atomic_add_rel_ptr      atomic_add_rel_int
 160 #endif
 161 #undef _ATOMIC_ADD
 162 #undef __atomic_add_long
 163 #undef __atomic_add_int
 164
 165 /*
 166  * atomic_clear(p, v)
 167  * { *p &= ~v; }
 168  */
 169
 170 #define __atomic_clear_int(p, v, t)                             \
 171     __asm __volatile(                                           \
 172         "1:     lwarx   %0, 0, %2\n"                            \
 173         "       andc    %0, %0, %3\n"                           \
 174         "       stwcx.  %0, 0, %2\n"                            \
 175         "       bne-    1b\n"                                   \
 176         : "=&r" (t), "=m" (*p)                                  \
 177         : "r" (p), "r" (v), "m" (*p)                            \
 178         : "cr0", "memory")                                      \
 179     /* __atomic_clear_int */
 180
 181 #ifdef __powerpc64__
 182 #define __atomic_clear_long(p, v, t)                            \
 183     __asm __volatile(                                           \
 184         "1:     ldarx   %0, 0, %2\n"                            \
 185         "       andc    %0, %0, %3\n"                           \
 186         "       stdcx.  %0, 0, %2\n"                            \
 187         "       bne-    1b\n"                                   \
 188         : "=&r" (t), "=m" (*p)                                  \
 189         : "r" (p), "r" (v), "m" (*p)                            \
 190         : "cr0", "memory")                                      \
 191     /* __atomic_clear_long */
 192 #else
 193 #define __atomic_clear_long(p, v, t)                            \
 194     __asm __volatile(                                           \
 195         "1:     lwarx   %0, 0, %2\n"                            \
 196         "       andc    %0, %0, %3\n"                           \
 197         "       stwcx.  %0, 0, %2\n"                            \
 198         "       bne-    1b\n"                                   \
 199         : "=&r" (t), "=m" (*p)                                  \
 200         : "r" (p), "r" (v), "m" (*p)                            \
 201         : "cr0", "memory")                                      \
 202     /* __atomic_clear_long */
 203 #endif
 204
 205 #define _ATOMIC_CLEAR(type)                                     \
 206     static __inline void                                        \
 207     atomic_clear_##type(volatile u_##type *p, u_##type v) {     \
 208         u_##type t;                                             \
 209         __atomic_clear_##type(p, v, t);                         \
 210     }                                                           \
 211                                                                 \
 212     static __inline void                                        \
 213     atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \
 214         u_##type t;                                             \
 215         __atomic_clear_##type(p, v, t);                         \
 216         __ATOMIC_ACQ();                                         \
 217     }                                                           \
 218                                                                 \
 219     static __inline void                                        \
 220     atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \
 221         u_##type t;                                             \
 222         __ATOMIC_REL();                                         \
 223         __atomic_clear_##type(p, v, t);                         \
 224     }                                                           \
 225     /* _ATOMIC_CLEAR */
 226
 227
 228 _ATOMIC_CLEAR(int)
 229 _ATOMIC_CLEAR(long)
 230
 231 #define atomic_clear_32         atomic_clear_int
 232 #define atomic_clear_acq_32     atomic_clear_acq_int
 233 #define atomic_clear_rel_32     atomic_clear_rel_int
 234
 235 #ifdef __powerpc64__
 236 #define atomic_clear_64         atomic_clear_long
 237 #define atomic_clear_acq_64     atomic_clear_acq_long
 238 #define atomic_clear_rel_64     atomic_clear_rel_long
 239
 240 #define atomic_clear_ptr        atomic_clear_long
 241 #define atomic_clear_acq_ptr    atomic_clear_acq_long
 242 #define atomic_clear_rel_ptr    atomic_clear_rel_long
 243 #else
 244 #define atomic_clear_ptr        atomic_clear_int
 245 #define atomic_clear_acq_ptr    atomic_clear_acq_int
 246 #define atomic_clear_rel_ptr    atomic_clear_rel_int
 247 #endif
 248 #undef _ATOMIC_CLEAR
 249 #undef __atomic_clear_long
 250 #undef __atomic_clear_int
 251
 252 /*
 253  * atomic_cmpset(p, o, n)
 254  */
 255 /* TODO -- see below */
 256
 257 /*
 258  * atomic_load_acq(p)
 259  */
 260 /* TODO -- see below */
 261
 262 /*
 263  * atomic_readandclear(p)
 264  */
 265 /* TODO -- see below */
 266
 267 /*
 268  * atomic_set(p, v)
 269  * { *p |= v; }
 270  */
 271
 272 #define __atomic_set_int(p, v, t)                               \
 273     __asm __volatile(                                           \
 274         "1:     lwarx   %0, 0, %2\n"                            \
 275         "       or      %0, %3, %0\n"                           \
 276         "       stwcx.  %0, 0, %2\n"                            \
 277         "       bne-    1b\n"                                   \
 278         : "=&r" (t), "=m" (*p)                                  \
 279         : "r" (p), "r" (v), "m" (*p)                            \
 280         : "cr0", "memory")                                      \
 281     /* __atomic_set_int */
 282
 283 #ifdef __powerpc64__
 284 #define __atomic_set_long(p, v, t)                              \
 285     __asm __volatile(                                           \
 286         "1:     ldarx   %0, 0, %2\n"                            \
 287         "       or      %0, %3, %0\n"                           \
 288         "       stdcx.  %0, 0, %2\n"                            \
 289         "       bne-    1b\n"                                   \
 290         : "=&r" (t), "=m" (*p)                                  \
 291         : "r" (p), "r" (v), "m" (*p)                            \
 292         : "cr0", "memory")                                      \
 293     /* __atomic_set_long */
 294 #else
 295 #define __atomic_set_long(p, v, t)                              \
 296     __asm __volatile(                                           \
 297         "1:     lwarx   %0, 0, %2\n"                            \
 298         "       or      %0, %3, %0\n"                           \
 299         "       stwcx.  %0, 0, %2\n"                            \
 300         "       bne-    1b\n"                                   \
 301         : "=&r" (t), "=m" (*p)                                  \
 302         : "r" (p), "r" (v), "m" (*p)                            \
 303         : "cr0", "memory")                                      \
 304     /* __atomic_set_long */
 305 #endif
 306
 307 #define _ATOMIC_SET(type)                                       \
 308     static __inline void                                        \
 309     atomic_set_##type(volatile u_##type *p, u_##type v) {       \
 310         u_##type t;                                             \
 311         __atomic_set_##type(p, v, t);                           \
 312     }                                                           \
 313                                                                 \
 314     static __inline void                                        \
 315     atomic_set_acq_##type(volatile u_##type *p, u_##type v) {   \
 316         u_##type t;                                             \
 317         __atomic_set_##type(p, v, t);                           \
 318         __ATOMIC_ACQ();                                         \
 319     }                                                           \
 320                                                                 \
 321     static __inline void                                        \
 322     atomic_set_rel_##type(volatile u_##type *p, u_##type v) {   \
 323         u_##type t;                                             \
 324         __ATOMIC_REL();                                         \
 325         __atomic_set_##type(p, v, t);                           \
 326     }                                                           \
 327     /* _ATOMIC_SET */
 328
 329 _ATOMIC_SET(int)
 330 _ATOMIC_SET(long)
 331
 332 #define atomic_set_32           atomic_set_int
 333 #define atomic_set_acq_32       atomic_set_acq_int
 334 #define atomic_set_rel_32       atomic_set_rel_int
 335
 336 #ifdef __powerpc64__
 337 #define atomic_set_64           atomic_set_long
 338 #define atomic_set_acq_64       atomic_set_acq_long
 339 #define atomic_set_rel_64       atomic_set_rel_long
 340
 341 #define atomic_set_ptr          atomic_set_long
 342 #define atomic_set_acq_ptr      atomic_set_acq_long
 343 #define atomic_set_rel_ptr      atomic_set_rel_long
 344 #else
 345 #define atomic_set_ptr          atomic_set_int
 346 #define atomic_set_acq_ptr      atomic_set_acq_int
 347 #define atomic_set_rel_ptr      atomic_set_rel_int
 348 #endif
 349 #undef _ATOMIC_SET
 350 #undef __atomic_set_long
 351 #undef __atomic_set_int
 352
 353 /*
 354  * atomic_subtract(p, v)
 355  * { *p -= v; }
 356  */
 357
 358 #define __atomic_subtract_int(p, v, t)                          \
 359     __asm __volatile(                                           \
 360         "1:     lwarx   %0, 0, %2\n"                            \
 361         "       subf    %0, %3, %0\n"                           \
 362         "       stwcx.  %0, 0, %2\n"                            \
 363         "       bne-    1b\n"                                   \
 364         : "=&r" (t), "=m" (*p)                                  \
 365         : "r" (p), "r" (v), "m" (*p)                            \
 366         : "cr0", "memory")                                      \
 367     /* __atomic_subtract_int */
 368
 369 #ifdef __powerpc64__
 370 #define __atomic_subtract_long(p, v, t)                         \
 371     __asm __volatile(                                           \
 372         "1:     ldarx   %0, 0, %2\n"                            \
 373         "       subf    %0, %3, %0\n"                           \
 374         "       stdcx.  %0, 0, %2\n"                            \
 375         "       bne-    1b\n"                                   \
 376         : "=&r" (t), "=m" (*p)                                  \
 377         : "r" (p), "r" (v), "m" (*p)                            \
 378         : "cr0", "memory")                                      \
 379     /* __atomic_subtract_long */
 380 #else
 381 #define __atomic_subtract_long(p, v, t)                         \
 382     __asm __volatile(                                           \
 383         "1:     lwarx   %0, 0, %2\n"                            \
 384         "       subf    %0, %3, %0\n"                           \
 385         "       stwcx.  %0, 0, %2\n"                            \
 386         "       bne-    1b\n"                                   \
 387         : "=&r" (t), "=m" (*p)                                  \
 388         : "r" (p), "r" (v), "m" (*p)                            \
 389         : "cr0", "memory")                                      \
 390     /* __atomic_subtract_long */
 391 #endif
 392
 393 #define _ATOMIC_SUBTRACT(type)                                          \
 394     static __inline void                                                \
 395     atomic_subtract_##type(volatile u_##type *p, u_##type v) {          \
 396         u_##type t;                                                     \
 397         __atomic_subtract_##type(p, v, t);                              \
 398     }                                                                   \
 399                                                                         \
 400     static __inline void                                                \
 401     atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) {      \
 402         u_##type t;                                                     \
 403         __atomic_subtract_##type(p, v, t);                              \
 404         __ATOMIC_ACQ();                                                 \
 405     }                                                                   \
 406                                                                         \
 407     static __inline void                                                \
 408     atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) {      \
 409         u_##type t;                                                     \
 410         __ATOMIC_REL();                                                 \
 411         __atomic_subtract_##type(p, v, t);                              \
 412     }                                                                   \
 413     /* _ATOMIC_SUBTRACT */
 414
 415 _ATOMIC_SUBTRACT(int)
 416 _ATOMIC_SUBTRACT(long)
 417
 418 #define atomic_subtract_32      atomic_subtract_int
 419 #define atomic_subtract_acq_32  atomic_subtract_acq_int
 420 #define atomic_subtract_rel_32  atomic_subtract_rel_int
 421
 422 #ifdef __powerpc64__
 423 #define atomic_subtract_64      atomic_subtract_long
 424 #define atomic_subtract_acq_64  atomic_subract_acq_long
 425 #define atomic_subtract_rel_64  atomic_subtract_rel_long
 426
 427 #define atomic_subtract_ptr     atomic_subtract_long
 428 #define atomic_subtract_acq_ptr atomic_subtract_acq_long
 429 #define atomic_subtract_rel_ptr atomic_subtract_rel_long
 430 #else
 431 #define atomic_subtract_ptr     atomic_subtract_int
 432 #define atomic_subtract_acq_ptr atomic_subtract_acq_int
 433 #define atomic_subtract_rel_ptr atomic_subtract_rel_int
 434 #endif
 435 #undef _ATOMIC_SUBTRACT
 436 #undef __atomic_subtract_long
 437 #undef __atomic_subtract_int
 438
 439 /*
 440  * atomic_store_rel(p, v)
 441  */
 442 /* TODO -- see below */
 443
 444 /*
 445  * Old/original implementations that still need revisiting.
 446  */
 447
 448 static __inline u_int
 449 atomic_readandclear_int(volatile u_int *addr)
 450 {
 451         u_int result,temp;
 452
 453         __asm __volatile (
 454                 "\tsync\n"                      /* drain writes */
 455                 "1:\tlwarx %0, 0, %3\n\t"       /* load old value */
 456                 "li %1, 0\n\t"                  /* load new value */
 457                 "stwcx. %1, 0, %3\n\t"          /* attempt to store */
 458                 "bne- 1b\n\t"                   /* spin if failed */
 459                 : "=&r"(result), "=&r"(temp), "=m" (*addr)
 460                 : "r" (addr), "m" (*addr)
 461                 : "cr0", "memory");
 462
 463         return (result);
 464 }
 465
 466 #ifdef __powerpc64__
 467 static __inline u_long
 468 atomic_readandclear_long(volatile u_long *addr)
 469 {
 470         u_long result,temp;
 471
 472         __asm __volatile (
 473                 "\tsync\n"                      /* drain writes */
 474                 "1:\tldarx %0, 0, %3\n\t"       /* load old value */
 475                 "li %1, 0\n\t"                  /* load new value */
 476                 "stdcx. %1, 0, %3\n\t"          /* attempt to store */
 477                 "bne- 1b\n\t"                   /* spin if failed */
 478                 : "=&r"(result), "=&r"(temp), "=m" (*addr)
 479                 : "r" (addr), "m" (*addr)
 480                 : "cr0", "memory");
 481
 482         return (result);
 483 }
 484 #endif
 485
 486 #define atomic_readandclear_32          atomic_readandclear_int
 487
 488 #ifdef __powerpc64__
 489 #define atomic_readandclear_64          atomic_readandclear_long
 490
 491 #define atomic_readandclear_ptr         atomic_readandclear_long
 492 #else
 493 static __inline u_long
 494 atomic_readandclear_long(volatile u_long *addr)
 495 {
 496
 497         return ((u_long)atomic_readandclear_int((volatile u_int *)addr));
 498 }
 499
 500 #define atomic_readandclear_ptr         atomic_readandclear_int
 501 #endif
 502
 503 /*
 504  * We assume that a = b will do atomic loads and stores.
 505  */
 506 #define ATOMIC_STORE_LOAD(TYPE)                                 \
 507 static __inline u_##TYPE                                        \
 508 atomic_load_acq_##TYPE(volatile u_##TYPE *p)                    \
 509 {                                                               \
 510         u_##TYPE v;                                             \
 511                                                                 \
 512         v = *p;                                                 \
 513         powerpc_lwsync();                                       \
 514         return (v);                                             \
 515 }                                                               \
 516                                                                 \
 517 static __inline void                                            \
 518 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)       \
 519 {                                                               \
 520                                                                 \
 521         powerpc_lwsync();                                       \
 522         *p = v;                                                 \
 523 }
 524
 525 ATOMIC_STORE_LOAD(int)
 526
 527 #define atomic_load_acq_32      atomic_load_acq_int
 528 #define atomic_store_rel_32     atomic_store_rel_int
 529
 530 #ifdef __powerpc64__
 531 ATOMIC_STORE_LOAD(long)
 532
 533 #define atomic_load_acq_64      atomic_load_acq_long
 534 #define atomic_store_rel_64     atomic_store_rel_long
 535
 536 #define atomic_load_acq_ptr     atomic_load_acq_long
 537 #define atomic_store_rel_ptr    atomic_store_rel_long
 538 #else
 539 static __inline u_long
 540 atomic_load_acq_long(volatile u_long *addr)
 541 {
 542
 543         return ((u_long)atomic_load_acq_int((volatile u_int *)addr));
 544 }
 545
 546 static __inline void
 547 atomic_store_rel_long(volatile u_long *addr, u_long val)
 548 {
 549
 550         atomic_store_rel_int((volatile u_int *)addr, (u_int)val);
 551 }
 552
 553 #define atomic_load_acq_ptr     atomic_load_acq_int
 554 #define atomic_store_rel_ptr    atomic_store_rel_int
 555 #endif
 556 #undef ATOMIC_STORE_LOAD
 557
 558 /*
 559  * Atomically compare the value stored at *p with cmpval and if the
 560  * two values are equal, update the value of *p with newval. Returns
 561  * zero if the compare failed, nonzero otherwise.
 562  */
 563 static __inline int
 564 atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval)
 565 {
 566         int     ret;
 567
 568         __asm __volatile (
 569                 "1:\tlwarx %0, 0, %2\n\t"       /* load old value */
 570                 "cmplw %3, %0\n\t"              /* compare */
 571                 "bne 2f\n\t"                    /* exit if not equal */
 572                 "stwcx. %4, 0, %2\n\t"          /* attempt to store */
 573                 "bne- 1b\n\t"                   /* spin if failed */
 574                 "li %0, 1\n\t"                  /* success - retval = 1 */
 575                 "b 3f\n\t"                      /* we've succeeded */
 576                 "2:\n\t"
 577                 "stwcx. %0, 0, %2\n\t"          /* clear reservation (74xx) */
 578                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 579                 "3:\n\t"
 580                 : "=&r" (ret), "=m" (*p)
 581                 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
 582                 : "cr0", "memory");
 583
 584         return (ret);
 585 }
 586 static __inline int
 587 atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval)
 588 {
 589         int ret;
 590
 591         __asm __volatile (
 592             #ifdef __powerpc64__
 593                 "1:\tldarx %0, 0, %2\n\t"       /* load old value */
 594                 "cmpld %3, %0\n\t"              /* compare */
 595                 "bne 2f\n\t"                    /* exit if not equal */
 596                 "stdcx. %4, 0, %2\n\t"          /* attempt to store */
 597             #else
 598                 "1:\tlwarx %0, 0, %2\n\t"       /* load old value */
 599                 "cmplw %3, %0\n\t"              /* compare */
 600                 "bne 2f\n\t"                    /* exit if not equal */
 601                 "stwcx. %4, 0, %2\n\t"          /* attempt to store */
 602             #endif
 603                 "bne- 1b\n\t"                   /* spin if failed */
 604                 "li %0, 1\n\t"                  /* success - retval = 1 */
 605                 "b 3f\n\t"                      /* we've succeeded */
 606                 "2:\n\t"
 607             #ifdef __powerpc64__
 608                 "stdcx. %0, 0, %2\n\t"          /* clear reservation (74xx) */
 609             #else
 610                 "stwcx. %0, 0, %2\n\t"          /* clear reservation (74xx) */
 611             #endif
 612                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 613                 "3:\n\t"
 614                 : "=&r" (ret), "=m" (*p)
 615                 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
 616                 : "cr0", "memory");
 617
 618         return (ret);
 619 }
 620
 621 static __inline int
 622 atomic_cmpset_acq_int(volatile u_int *p, u_int cmpval, u_int newval)
 623 {
 624         int retval;
 625
 626         retval = atomic_cmpset_int(p, cmpval, newval);
 627         __ATOMIC_ACQ();
 628         return (retval);
 629 }
 630
 631 static __inline int
 632 atomic_cmpset_rel_int(volatile u_int *p, u_int cmpval, u_int newval)
 633 {
 634         __ATOMIC_REL();
 635         return (atomic_cmpset_int(p, cmpval, newval));
 636 }
 637
 638 static __inline int
 639 atomic_cmpset_acq_long(volatile u_long *p, u_long cmpval, u_long newval)
 640 {
 641         u_long retval;
 642
 643         retval = atomic_cmpset_long(p, cmpval, newval);
 644         __ATOMIC_ACQ();
 645         return (retval);
 646 }
 647
 648 static __inline int
 649 atomic_cmpset_rel_long(volatile u_long *p, u_long cmpval, u_long newval)
 650 {
 651         __ATOMIC_REL();
 652         return (atomic_cmpset_long(p, cmpval, newval));
 653 }
 654
 655 #define atomic_cmpset_32        atomic_cmpset_int
 656 #define atomic_cmpset_acq_32    atomic_cmpset_acq_int
 657 #define atomic_cmpset_rel_32    atomic_cmpset_rel_int
 658
 659 #ifdef __powerpc64__
 660 #define atomic_cmpset_64        atomic_cmpset_long
 661 #define atomic_cmpset_acq_64    atomic_cmpset_acq_long
 662 #define atomic_cmpset_rel_64    atomic_cmpset_rel_long
 663
 664 #define atomic_cmpset_ptr       atomic_cmpset_long
 665 #define atomic_cmpset_acq_ptr   atomic_cmpset_acq_long
 666 #define atomic_cmpset_rel_ptr   atomic_cmpset_rel_long
 667 #else
 668 #define atomic_cmpset_ptr       atomic_cmpset_int
 669 #define atomic_cmpset_acq_ptr   atomic_cmpset_acq_int
 670 #define atomic_cmpset_rel_ptr   atomic_cmpset_rel_int
 671 #endif
 672
 673 /*
 674  * Atomically compare the value stored at *p with *cmpval and if the
 675  * two values are equal, update the value of *p with newval. Returns
 676  * zero if the compare failed and sets *cmpval to the read value from *p,
 677  * nonzero otherwise.
 678  */
 679 static __inline int
 680 atomic_fcmpset_int(volatile u_int *p, u_int *cmpval, u_int newval)
 681 {
 682         int     ret;
 683
 684         __asm __volatile (
 685                 "lwarx %0, 0, %3\n\t"   /* load old value */
 686                 "cmplw %4, %0\n\t"              /* compare */
 687                 "bne 1f\n\t"                    /* exit if not equal */
 688                 "stwcx. %5, 0, %3\n\t"          /* attempt to store */
 689                 "bne- 1f\n\t"                   /* exit if failed */
 690                 "li %0, 1\n\t"                  /* success - retval = 1 */
 691                 "b 2f\n\t"                      /* we've succeeded */
 692                 "1:\n\t"
 693                 "stwcx. %0, 0, %3\n\t"          /* clear reservation (74xx) */
 694                 "stwx %0, 0, %7\n\t"
 695                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 696                 "2:\n\t"
 697                 : "=&r" (ret), "=m" (*p), "=m" (*cmpval)
 698                 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval)
 699                 : "cr0", "memory");
 700
 701         return (ret);
 702 }
 703 static __inline int
 704 atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval)
 705 {
 706         int ret;
 707
 708         __asm __volatile (
 709             #ifdef __powerpc64__
 710                 "ldarx %0, 0, %3\n\t"   /* load old value */
 711                 "cmpld %4, %0\n\t"              /* compare */
 712                 "bne 1f\n\t"                    /* exit if not equal */
 713                 "stdcx. %5, 0, %3\n\t"          /* attempt to store */
 714             #else
 715                 "lwarx %0, 0, %3\n\t"   /* load old value */
 716                 "cmplw %4, %0\n\t"              /* compare */
 717                 "bne 1f\n\t"                    /* exit if not equal */
 718                 "stwcx. %5, 0, %3\n\t"          /* attempt to store */
 719             #endif
 720                 "bne- 1f\n\t"                   /* exit if failed */
 721                 "li %0, 1\n\t"                  /* success - retval = 1 */
 722                 "b 2f\n\t"                      /* we've succeeded */
 723                 "1:\n\t"
 724             #ifdef __powerpc64__
 725                 "stdcx. %0, 0, %3\n\t"          /* clear reservation (74xx) */
 726                 "stdx %0, 0, %7\n\t"
 727             #else
 728                 "stwcx. %0, 0, %3\n\t"          /* clear reservation (74xx) */
 729                 "stwx %0, 0, %7\n\t"
 730             #endif
 731                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 732                 "2:\n\t"
 733                 : "=&r" (ret), "=m" (*p), "=m" (*cmpval)
 734                 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval)
 735                 : "cr0", "memory");
 736
 737         return (ret);
 738 }
 739
 740 static __inline int
 741 atomic_fcmpset_acq_int(volatile u_int *p, u_int *cmpval, u_int newval)
 742 {
 743         int retval;
 744
 745         retval = atomic_fcmpset_int(p, cmpval, newval);
 746         __ATOMIC_ACQ();
 747         return (retval);
 748 }
 749
 750 static __inline int
 751 atomic_fcmpset_rel_int(volatile u_int *p, u_int *cmpval, u_int newval)
 752 {
 753         __ATOMIC_REL();
 754         return (atomic_fcmpset_int(p, cmpval, newval));
 755 }
 756
 757 static __inline int
 758 atomic_fcmpset_acq_long(volatile u_long *p, u_long *cmpval, u_long newval)
 759 {
 760         u_long retval;
 761
 762         retval = atomic_fcmpset_long(p, cmpval, newval);
 763         __ATOMIC_ACQ();
 764         return (retval);
 765 }
 766
 767 static __inline int
 768 atomic_fcmpset_rel_long(volatile u_long *p, u_long *cmpval, u_long newval)
 769 {
 770         __ATOMIC_REL();
 771         return (atomic_fcmpset_long(p, cmpval, newval));
 772 }
 773
 774 #define atomic_fcmpset_32       atomic_fcmpset_int
 775 #define atomic_fcmpset_acq_32   atomic_fcmpset_acq_int
 776 #define atomic_fcmpset_rel_32   atomic_fcmpset_rel_int
 777
 778 #ifdef __powerpc64__
 779 #define atomic_fcmpset_64       atomic_fcmpset_long
 780 #define atomic_fcmpset_acq_64   atomic_fcmpset_acq_long
 781 #define atomic_fcmpset_rel_64   atomic_fcmpset_rel_long
 782
 783 #define atomic_fcmpset_ptr      atomic_fcmpset_long
 784 #define atomic_fcmpset_acq_ptr  atomic_fcmpset_acq_long
 785 #define atomic_fcmpset_rel_ptr  atomic_fcmpset_rel_long
 786 #else
 787 #define atomic_fcmpset_ptr      atomic_fcmpset_int
 788 #define atomic_fcmpset_acq_ptr  atomic_fcmpset_acq_int
 789 #define atomic_fcmpset_rel_ptr  atomic_fcmpset_rel_int
 790 #endif
 791
 792 static __inline u_int
 793 atomic_fetchadd_int(volatile u_int *p, u_int v)
 794 {
 795         u_int value;
 796
 797         do {
 798                 value = *p;
 799         } while (!atomic_cmpset_int(p, value, value + v));
 800         return (value);
 801 }
 802
 803 static __inline u_long
 804 atomic_fetchadd_long(volatile u_long *p, u_long v)
 805 {
 806         u_long value;
 807
 808         do {
 809                 value = *p;
 810         } while (!atomic_cmpset_long(p, value, value + v));
 811         return (value);
 812 }
 813
 814 static __inline u_int
 815 atomic_swap_32(volatile u_int *p, u_int v)
 816 {
 817         u_int prev;
 818
 819         __asm __volatile(
 820         "1:     lwarx   %0,0,%2\n"
 821         "       stwcx.  %3,0,%2\n"
 822         "       bne-    1b\n"
 823         : "=&r" (prev), "+m" (*(volatile u_int *)p)
 824         : "r" (p), "r" (v)
 825         : "cr0", "memory");
 826
 827         return (prev);
 828 }
 829
 830 #ifdef __powerpc64__
 831 static __inline u_long
 832 atomic_swap_64(volatile u_long *p, u_long v)
 833 {
 834         u_long prev;
 835
 836         __asm __volatile(
 837         "1:     ldarx   %0,0,%2\n"
 838         "       stdcx.  %3,0,%2\n"
 839         "       bne-    1b\n"
 840         : "=&r" (prev), "+m" (*(volatile u_long *)p)
 841         : "r" (p), "r" (v)
 842         : "cr0", "memory");
 843
 844         return (prev);
 845 }
 846 #endif
 847
 848 #define atomic_fetchadd_32      atomic_fetchadd_int
 849 #define atomic_swap_int         atomic_swap_32
 850
 851 #ifdef __powerpc64__
 852 #define atomic_fetchadd_64      atomic_fetchadd_long
 853 #define atomic_swap_long        atomic_swap_64
 854 #define atomic_swap_ptr         atomic_swap_64
 855 #else
 856 #define atomic_swap_long(p,v)   atomic_swap_32((volatile u_int *)(p), v)
 857 #define atomic_swap_ptr(p,v)    atomic_swap_32((volatile u_int *)(p), v)
 858 #endif
 859
 860 #undef __ATOMIC_REL
 861 #undef __ATOMIC_ACQ
 862
 863 static __inline void
 864 atomic_thread_fence_acq(void)
 865 {
 866
 867         powerpc_lwsync();
 868 }
 869
 870 static __inline void
 871 atomic_thread_fence_rel(void)
 872 {
 873
 874         powerpc_lwsync();
 875 }
 876
 877 static __inline void
 878 atomic_thread_fence_acq_rel(void)
 879 {
 880
 881         powerpc_lwsync();
 882 }
 883
 884 static __inline void
 885 atomic_thread_fence_seq_cst(void)
 886 {
 887
 888         __asm __volatile("sync" : : : "memory");
 889 }
 890
 891 #endif /* ! _MACHINE_ATOMIC_H_ */