sys/powerpc/include/atomic.h

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2008 Marcel Moolenaar
   5  * Copyright (c) 2001 Benno Rice
   6  * Copyright (c) 2001 David E. O'Brien
   7  * Copyright (c) 1998 Doug Rabson
   8  * All rights reserved.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  29  * SUCH DAMAGE.
  30  *
  31  * $FreeBSD$
  32  */
  33
  34 #ifndef _MACHINE_ATOMIC_H_
  35 #define _MACHINE_ATOMIC_H_
  36
  37 #ifndef _SYS_CDEFS_H_
  38 #error this file needs sys/cdefs.h as a prerequisite
  39 #endif
  40
  41 #include <sys/atomic_common.h>
  42
  43 #ifndef __powerpc64__
  44 #include <sys/_atomic64e.h>
  45 #endif
  46
  47 /*
  48  * The __ATOMIC_REL/ACQ() macros provide memory barriers only in conjunction
  49  * with the atomic lXarx/stXcx. sequences below. They are not exposed outside
  50  * of this file. See also Appendix B.2 of Book II of the architecture manual.
  51  *
  52  * Note that not all Book-E processors accept the light-weight sync variant.
  53  * In particular, early models of E500 cores are known to wedge. Bank on all
  54  * 64-bit capable CPUs to accept lwsync properly and pressimize 32-bit CPUs
  55  * to use the heavier-weight sync.
  56  */
  57
  58 #ifdef __powerpc64__
  59 #define mb()            __asm __volatile("sync" : : : "memory")
  60 #define rmb()           __asm __volatile("lwsync" : : : "memory")
  61 #define wmb()           __asm __volatile("lwsync" : : : "memory")
  62 #define __ATOMIC_REL()  __asm __volatile("lwsync" : : : "memory")
  63 #define __ATOMIC_ACQ()  __asm __volatile("isync" : : : "memory")
  64 #else
  65 #define mb()            __asm __volatile("sync" : : : "memory")
  66 #define rmb()           __asm __volatile("sync" : : : "memory")
  67 #define wmb()           __asm __volatile("sync" : : : "memory")
  68 #define __ATOMIC_REL()  __asm __volatile("sync" : : : "memory")
  69 #define __ATOMIC_ACQ()  __asm __volatile("isync" : : : "memory")
  70 #endif
  71
  72 static __inline void
  73 powerpc_lwsync(void)
  74 {
  75
  76 #ifdef __powerpc64__
  77         __asm __volatile("lwsync" : : : "memory");
  78 #else
  79         __asm __volatile("sync" : : : "memory");
  80 #endif
  81 }
  82
  83 /*
  84  * atomic_add(p, v)
  85  * { *p += v; }
  86  */
  87
  88 #define __atomic_add_int(p, v, t)                               \
  89     __asm __volatile(                                           \
  90         "1:     lwarx   %0, 0, %2\n"                            \
  91         "       add     %0, %3, %0\n"                           \
  92         "       stwcx.  %0, 0, %2\n"                            \
  93         "       bne-    1b\n"                                   \
  94         : "=&r" (t), "=m" (*p)                                  \
  95         : "r" (p), "r" (v), "m" (*p)                            \
  96         : "cr0", "memory")                                      \
  97     /* __atomic_add_int */
  98
  99 #ifdef __powerpc64__
 100 #define __atomic_add_long(p, v, t)                              \
 101     __asm __volatile(                                           \
 102         "1:     ldarx   %0, 0, %2\n"                            \
 103         "       add     %0, %3, %0\n"                           \
 104         "       stdcx.  %0, 0, %2\n"                            \
 105         "       bne-    1b\n"                                   \
 106         : "=&r" (t), "=m" (*p)                                  \
 107         : "r" (p), "r" (v), "m" (*p)                            \
 108         : "cr0", "memory")                                      \
 109     /* __atomic_add_long */
 110 #else
 111 #define __atomic_add_long(p, v, t)                              \
 112     __asm __volatile(                                           \
 113         "1:     lwarx   %0, 0, %2\n"                            \
 114         "       add     %0, %3, %0\n"                           \
 115         "       stwcx.  %0, 0, %2\n"                            \
 116         "       bne-    1b\n"                                   \
 117         : "=&r" (t), "=m" (*p)                                  \
 118         : "r" (p), "r" (v), "m" (*p)                            \
 119         : "cr0", "memory")                                      \
 120     /* __atomic_add_long */
 121 #endif
 122
 123 #define _ATOMIC_ADD(type)                                       \
 124     static __inline void                                        \
 125     atomic_add_##type(volatile u_##type *p, u_##type v) {       \
 126         u_##type t;                                             \
 127         __atomic_add_##type(p, v, t);                           \
 128     }                                                           \
 129                                                                 \
 130     static __inline void                                        \
 131     atomic_add_acq_##type(volatile u_##type *p, u_##type v) {   \
 132         u_##type t;                                             \
 133         __atomic_add_##type(p, v, t);                           \
 134         __ATOMIC_ACQ();                                         \
 135     }                                                           \
 136                                                                 \
 137     static __inline void                                        \
 138     atomic_add_rel_##type(volatile u_##type *p, u_##type v) {   \
 139         u_##type t;                                             \
 140         __ATOMIC_REL();                                         \
 141         __atomic_add_##type(p, v, t);                           \
 142     }                                                           \
 143     /* _ATOMIC_ADD */
 144
 145 _ATOMIC_ADD(int)
 146 _ATOMIC_ADD(long)
 147
 148 #define atomic_add_32           atomic_add_int
 149 #define atomic_add_acq_32       atomic_add_acq_int
 150 #define atomic_add_rel_32       atomic_add_rel_int
 151
 152 #ifdef __powerpc64__
 153 #define atomic_add_64           atomic_add_long
 154 #define atomic_add_acq_64       atomic_add_acq_long
 155 #define atomic_add_rel_64       atomic_add_rel_long
 156
 157 #define atomic_add_ptr          atomic_add_long
 158 #define atomic_add_acq_ptr      atomic_add_acq_long
 159 #define atomic_add_rel_ptr      atomic_add_rel_long
 160 #else
 161 #define atomic_add_ptr          atomic_add_int
 162 #define atomic_add_acq_ptr      atomic_add_acq_int
 163 #define atomic_add_rel_ptr      atomic_add_rel_int
 164 #endif
 165 #undef _ATOMIC_ADD
 166 #undef __atomic_add_long
 167 #undef __atomic_add_int
 168
 169 /*
 170  * atomic_clear(p, v)
 171  * { *p &= ~v; }
 172  */
 173
 174 #define __atomic_clear_int(p, v, t)                             \
 175     __asm __volatile(                                           \
 176         "1:     lwarx   %0, 0, %2\n"                            \
 177         "       andc    %0, %0, %3\n"                           \
 178         "       stwcx.  %0, 0, %2\n"                            \
 179         "       bne-    1b\n"                                   \
 180         : "=&r" (t), "=m" (*p)                                  \
 181         : "r" (p), "r" (v), "m" (*p)                            \
 182         : "cr0", "memory")                                      \
 183     /* __atomic_clear_int */
 184
 185 #ifdef __powerpc64__
 186 #define __atomic_clear_long(p, v, t)                            \
 187     __asm __volatile(                                           \
 188         "1:     ldarx   %0, 0, %2\n"                            \
 189         "       andc    %0, %0, %3\n"                           \
 190         "       stdcx.  %0, 0, %2\n"                            \
 191         "       bne-    1b\n"                                   \
 192         : "=&r" (t), "=m" (*p)                                  \
 193         : "r" (p), "r" (v), "m" (*p)                            \
 194         : "cr0", "memory")                                      \
 195     /* __atomic_clear_long */
 196 #else
 197 #define __atomic_clear_long(p, v, t)                            \
 198     __asm __volatile(                                           \
 199         "1:     lwarx   %0, 0, %2\n"                            \
 200         "       andc    %0, %0, %3\n"                           \
 201         "       stwcx.  %0, 0, %2\n"                            \
 202         "       bne-    1b\n"                                   \
 203         : "=&r" (t), "=m" (*p)                                  \
 204         : "r" (p), "r" (v), "m" (*p)                            \
 205         : "cr0", "memory")                                      \
 206     /* __atomic_clear_long */
 207 #endif
 208
 209 #define _ATOMIC_CLEAR(type)                                     \
 210     static __inline void                                        \
 211     atomic_clear_##type(volatile u_##type *p, u_##type v) {     \
 212         u_##type t;                                             \
 213         __atomic_clear_##type(p, v, t);                         \
 214     }                                                           \
 215                                                                 \
 216     static __inline void                                        \
 217     atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \
 218         u_##type t;                                             \
 219         __atomic_clear_##type(p, v, t);                         \
 220         __ATOMIC_ACQ();                                         \
 221     }                                                           \
 222                                                                 \
 223     static __inline void                                        \
 224     atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \
 225         u_##type t;                                             \
 226         __ATOMIC_REL();                                         \
 227         __atomic_clear_##type(p, v, t);                         \
 228     }                                                           \
 229     /* _ATOMIC_CLEAR */
 230
 231
 232 _ATOMIC_CLEAR(int)
 233 _ATOMIC_CLEAR(long)
 234
 235 #define atomic_clear_32         atomic_clear_int
 236 #define atomic_clear_acq_32     atomic_clear_acq_int
 237 #define atomic_clear_rel_32     atomic_clear_rel_int
 238
 239 #ifdef __powerpc64__
 240 #define atomic_clear_64         atomic_clear_long
 241 #define atomic_clear_acq_64     atomic_clear_acq_long
 242 #define atomic_clear_rel_64     atomic_clear_rel_long
 243
 244 #define atomic_clear_ptr        atomic_clear_long
 245 #define atomic_clear_acq_ptr    atomic_clear_acq_long
 246 #define atomic_clear_rel_ptr    atomic_clear_rel_long
 247 #else
 248 #define atomic_clear_ptr        atomic_clear_int
 249 #define atomic_clear_acq_ptr    atomic_clear_acq_int
 250 #define atomic_clear_rel_ptr    atomic_clear_rel_int
 251 #endif
 252 #undef _ATOMIC_CLEAR
 253 #undef __atomic_clear_long
 254 #undef __atomic_clear_int
 255
 256 /*
 257  * atomic_cmpset(p, o, n)
 258  */
 259 /* TODO -- see below */
 260
 261 /*
 262  * atomic_load_acq(p)
 263  */
 264 /* TODO -- see below */
 265
 266 /*
 267  * atomic_readandclear(p)
 268  */
 269 /* TODO -- see below */
 270
 271 /*
 272  * atomic_set(p, v)
 273  * { *p |= v; }
 274  */
 275
 276 #define __atomic_set_int(p, v, t)                               \
 277     __asm __volatile(                                           \
 278         "1:     lwarx   %0, 0, %2\n"                            \
 279         "       or      %0, %3, %0\n"                           \
 280         "       stwcx.  %0, 0, %2\n"                            \
 281         "       bne-    1b\n"                                   \
 282         : "=&r" (t), "=m" (*p)                                  \
 283         : "r" (p), "r" (v), "m" (*p)                            \
 284         : "cr0", "memory")                                      \
 285     /* __atomic_set_int */
 286
 287 #ifdef __powerpc64__
 288 #define __atomic_set_long(p, v, t)                              \
 289     __asm __volatile(                                           \
 290         "1:     ldarx   %0, 0, %2\n"                            \
 291         "       or      %0, %3, %0\n"                           \
 292         "       stdcx.  %0, 0, %2\n"                            \
 293         "       bne-    1b\n"                                   \
 294         : "=&r" (t), "=m" (*p)                                  \
 295         : "r" (p), "r" (v), "m" (*p)                            \
 296         : "cr0", "memory")                                      \
 297     /* __atomic_set_long */
 298 #else
 299 #define __atomic_set_long(p, v, t)                              \
 300     __asm __volatile(                                           \
 301         "1:     lwarx   %0, 0, %2\n"                            \
 302         "       or      %0, %3, %0\n"                           \
 303         "       stwcx.  %0, 0, %2\n"                            \
 304         "       bne-    1b\n"                                   \
 305         : "=&r" (t), "=m" (*p)                                  \
 306         : "r" (p), "r" (v), "m" (*p)                            \
 307         : "cr0", "memory")                                      \
 308     /* __atomic_set_long */
 309 #endif
 310
 311 #define _ATOMIC_SET(type)                                       \
 312     static __inline void                                        \
 313     atomic_set_##type(volatile u_##type *p, u_##type v) {       \
 314         u_##type t;                                             \
 315         __atomic_set_##type(p, v, t);                           \
 316     }                                                           \
 317                                                                 \
 318     static __inline void                                        \
 319     atomic_set_acq_##type(volatile u_##type *p, u_##type v) {   \
 320         u_##type t;                                             \
 321         __atomic_set_##type(p, v, t);                           \
 322         __ATOMIC_ACQ();                                         \
 323     }                                                           \
 324                                                                 \
 325     static __inline void                                        \
 326     atomic_set_rel_##type(volatile u_##type *p, u_##type v) {   \
 327         u_##type t;                                             \
 328         __ATOMIC_REL();                                         \
 329         __atomic_set_##type(p, v, t);                           \
 330     }                                                           \
 331     /* _ATOMIC_SET */
 332
 333 _ATOMIC_SET(int)
 334 _ATOMIC_SET(long)
 335
 336 #define atomic_set_32           atomic_set_int
 337 #define atomic_set_acq_32       atomic_set_acq_int
 338 #define atomic_set_rel_32       atomic_set_rel_int
 339
 340 #ifdef __powerpc64__
 341 #define atomic_set_64           atomic_set_long
 342 #define atomic_set_acq_64       atomic_set_acq_long
 343 #define atomic_set_rel_64       atomic_set_rel_long
 344
 345 #define atomic_set_ptr          atomic_set_long
 346 #define atomic_set_acq_ptr      atomic_set_acq_long
 347 #define atomic_set_rel_ptr      atomic_set_rel_long
 348 #else
 349 #define atomic_set_ptr          atomic_set_int
 350 #define atomic_set_acq_ptr      atomic_set_acq_int
 351 #define atomic_set_rel_ptr      atomic_set_rel_int
 352 #endif
 353 #undef _ATOMIC_SET
 354 #undef __atomic_set_long
 355 #undef __atomic_set_int
 356
 357 /*
 358  * atomic_subtract(p, v)
 359  * { *p -= v; }
 360  */
 361
 362 #define __atomic_subtract_int(p, v, t)                          \
 363     __asm __volatile(                                           \
 364         "1:     lwarx   %0, 0, %2\n"                            \
 365         "       subf    %0, %3, %0\n"                           \
 366         "       stwcx.  %0, 0, %2\n"                            \
 367         "       bne-    1b\n"                                   \
 368         : "=&r" (t), "=m" (*p)                                  \
 369         : "r" (p), "r" (v), "m" (*p)                            \
 370         : "cr0", "memory")                                      \
 371     /* __atomic_subtract_int */
 372
 373 #ifdef __powerpc64__
 374 #define __atomic_subtract_long(p, v, t)                         \
 375     __asm __volatile(                                           \
 376         "1:     ldarx   %0, 0, %2\n"                            \
 377         "       subf    %0, %3, %0\n"                           \
 378         "       stdcx.  %0, 0, %2\n"                            \
 379         "       bne-    1b\n"                                   \
 380         : "=&r" (t), "=m" (*p)                                  \
 381         : "r" (p), "r" (v), "m" (*p)                            \
 382         : "cr0", "memory")                                      \
 383     /* __atomic_subtract_long */
 384 #else
 385 #define __atomic_subtract_long(p, v, t)                         \
 386     __asm __volatile(                                           \
 387         "1:     lwarx   %0, 0, %2\n"                            \
 388         "       subf    %0, %3, %0\n"                           \
 389         "       stwcx.  %0, 0, %2\n"                            \
 390         "       bne-    1b\n"                                   \
 391         : "=&r" (t), "=m" (*p)                                  \
 392         : "r" (p), "r" (v), "m" (*p)                            \
 393         : "cr0", "memory")                                      \
 394     /* __atomic_subtract_long */
 395 #endif
 396
 397 #define _ATOMIC_SUBTRACT(type)                                          \
 398     static __inline void                                                \
 399     atomic_subtract_##type(volatile u_##type *p, u_##type v) {          \
 400         u_##type t;                                                     \
 401         __atomic_subtract_##type(p, v, t);                              \
 402     }                                                                   \
 403                                                                         \
 404     static __inline void                                                \
 405     atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) {      \
 406         u_##type t;                                                     \
 407         __atomic_subtract_##type(p, v, t);                              \
 408         __ATOMIC_ACQ();                                                 \
 409     }                                                                   \
 410                                                                         \
 411     static __inline void                                                \
 412     atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) {      \
 413         u_##type t;                                                     \
 414         __ATOMIC_REL();                                                 \
 415         __atomic_subtract_##type(p, v, t);                              \
 416     }                                                                   \
 417     /* _ATOMIC_SUBTRACT */
 418
 419 _ATOMIC_SUBTRACT(int)
 420 _ATOMIC_SUBTRACT(long)
 421
 422 #define atomic_subtract_32      atomic_subtract_int
 423 #define atomic_subtract_acq_32  atomic_subtract_acq_int
 424 #define atomic_subtract_rel_32  atomic_subtract_rel_int
 425
 426 #ifdef __powerpc64__
 427 #define atomic_subtract_64      atomic_subtract_long
 428 #define atomic_subtract_acq_64  atomic_subract_acq_long
 429 #define atomic_subtract_rel_64  atomic_subtract_rel_long
 430
 431 #define atomic_subtract_ptr     atomic_subtract_long
 432 #define atomic_subtract_acq_ptr atomic_subtract_acq_long
 433 #define atomic_subtract_rel_ptr atomic_subtract_rel_long
 434 #else
 435 #define atomic_subtract_ptr     atomic_subtract_int
 436 #define atomic_subtract_acq_ptr atomic_subtract_acq_int
 437 #define atomic_subtract_rel_ptr atomic_subtract_rel_int
 438 #endif
 439 #undef _ATOMIC_SUBTRACT
 440 #undef __atomic_subtract_long
 441 #undef __atomic_subtract_int
 442
 443 /*
 444  * atomic_store_rel(p, v)
 445  */
 446 /* TODO -- see below */
 447
 448 /*
 449  * Old/original implementations that still need revisiting.
 450  */
 451
 452 static __inline u_int
 453 atomic_readandclear_int(volatile u_int *addr)
 454 {
 455         u_int result,temp;
 456
 457         __asm __volatile (
 458                 "\tsync\n"                      /* drain writes */
 459                 "1:\tlwarx %0, 0, %3\n\t"       /* load old value */
 460                 "li %1, 0\n\t"                  /* load new value */
 461                 "stwcx. %1, 0, %3\n\t"          /* attempt to store */
 462                 "bne- 1b\n\t"                   /* spin if failed */
 463                 : "=&r"(result), "=&r"(temp), "=m" (*addr)
 464                 : "r" (addr), "m" (*addr)
 465                 : "cr0", "memory");
 466
 467         return (result);
 468 }
 469
 470 #ifdef __powerpc64__
 471 static __inline u_long
 472 atomic_readandclear_long(volatile u_long *addr)
 473 {
 474         u_long result,temp;
 475
 476         __asm __volatile (
 477                 "\tsync\n"                      /* drain writes */
 478                 "1:\tldarx %0, 0, %3\n\t"       /* load old value */
 479                 "li %1, 0\n\t"                  /* load new value */
 480                 "stdcx. %1, 0, %3\n\t"          /* attempt to store */
 481                 "bne- 1b\n\t"                   /* spin if failed */
 482                 : "=&r"(result), "=&r"(temp), "=m" (*addr)
 483                 : "r" (addr), "m" (*addr)
 484                 : "cr0", "memory");
 485
 486         return (result);
 487 }
 488 #endif
 489
 490 #define atomic_readandclear_32          atomic_readandclear_int
 491
 492 #ifdef __powerpc64__
 493 #define atomic_readandclear_64          atomic_readandclear_long
 494
 495 #define atomic_readandclear_ptr         atomic_readandclear_long
 496 #else
 497 static __inline u_long
 498 atomic_readandclear_long(volatile u_long *addr)
 499 {
 500
 501         return ((u_long)atomic_readandclear_int((volatile u_int *)addr));
 502 }
 503
 504 #define atomic_readandclear_ptr         atomic_readandclear_int
 505 #endif
 506
 507 /*
 508  * We assume that a = b will do atomic loads and stores.
 509  */
 510 #define ATOMIC_STORE_LOAD(TYPE)                                 \
 511 static __inline u_##TYPE                                        \
 512 atomic_load_acq_##TYPE(volatile u_##TYPE *p)                    \
 513 {                                                               \
 514         u_##TYPE v;                                             \
 515                                                                 \
 516         v = *p;                                                 \
 517         mb();                                                   \
 518         return (v);                                             \
 519 }                                                               \
 520                                                                 \
 521 static __inline void                                            \
 522 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)       \
 523 {                                                               \
 524                                                                 \
 525         powerpc_lwsync();                                       \
 526         *p = v;                                                 \
 527 }
 528
 529 ATOMIC_STORE_LOAD(int)
 530
 531 #define atomic_load_acq_32      atomic_load_acq_int
 532 #define atomic_store_rel_32     atomic_store_rel_int
 533
 534 #ifdef __powerpc64__
 535 ATOMIC_STORE_LOAD(long)
 536
 537 #define atomic_load_acq_64      atomic_load_acq_long
 538 #define atomic_store_rel_64     atomic_store_rel_long
 539
 540 #define atomic_load_acq_ptr     atomic_load_acq_long
 541 #define atomic_store_rel_ptr    atomic_store_rel_long
 542 #else
 543 static __inline u_long
 544 atomic_load_acq_long(volatile u_long *addr)
 545 {
 546
 547         return ((u_long)atomic_load_acq_int((volatile u_int *)addr));
 548 }
 549
 550 static __inline void
 551 atomic_store_rel_long(volatile u_long *addr, u_long val)
 552 {
 553
 554         atomic_store_rel_int((volatile u_int *)addr, (u_int)val);
 555 }
 556
 557 #define atomic_load_acq_ptr     atomic_load_acq_int
 558 #define atomic_store_rel_ptr    atomic_store_rel_int
 559 #endif
 560 #undef ATOMIC_STORE_LOAD
 561
 562 /*
 563  * Atomically compare the value stored at *p with cmpval and if the
 564  * two values are equal, update the value of *p with newval. Returns
 565  * zero if the compare failed, nonzero otherwise.
 566  */
 567 #ifdef ISA_206_ATOMICS
 568 static __inline int
 569 atomic_cmpset_char(volatile u_char *p, u_char cmpval, u_char newval)
 570 {
 571         int     ret;
 572
 573         __asm __volatile (
 574                 "1:\tlbarx %0, 0, %2\n\t"       /* load old value */
 575                 "cmplw %3, %0\n\t"              /* compare */
 576                 "bne- 2f\n\t"                   /* exit if not equal */
 577                 "stbcx. %4, 0, %2\n\t"          /* attempt to store */
 578                 "bne- 1b\n\t"                   /* spin if failed */
 579                 "li %0, 1\n\t"                  /* success - retval = 1 */
 580                 "b 3f\n\t"                      /* we've succeeded */
 581                 "2:\n\t"
 582                 "stbcx. %0, 0, %2\n\t"          /* clear reservation (74xx) */
 583                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 584                 "3:\n\t"
 585                 : "=&r" (ret), "=m" (*p)
 586                 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
 587                 : "cr0", "memory");
 588
 589         return (ret);
 590 }
 591
 592 static __inline int
 593 atomic_cmpset_short(volatile u_short *p, u_short cmpval, u_short newval)
 594 {
 595         int     ret;
 596
 597         __asm __volatile (
 598                 "1:\tlharx %0, 0, %2\n\t"       /* load old value */
 599                 "cmplw %3, %0\n\t"              /* compare */
 600                 "bne- 2f\n\t"                   /* exit if not equal */
 601                 "sthcx. %4, 0, %2\n\t"          /* attempt to store */
 602                 "bne- 1b\n\t"                   /* spin if failed */
 603                 "li %0, 1\n\t"                  /* success - retval = 1 */
 604                 "b 3f\n\t"                      /* we've succeeded */
 605                 "2:\n\t"
 606                 "sthcx. %0, 0, %2\n\t"          /* clear reservation (74xx) */
 607                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 608                 "3:\n\t"
 609                 : "=&r" (ret), "=m" (*p)
 610                 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
 611                 : "cr0", "memory");
 612
 613         return (ret);
 614 }
 615 #else
 616 static __inline int
 617 atomic_cmpset_masked(uint32_t *p, uint32_t cmpval, uint32_t newval,
 618     uint32_t mask)
 619 {
 620         int             ret;
 621         uint32_t        tmp;
 622
 623         __asm __volatile (
 624                 "1:\tlwarx %2, 0, %3\n\t"       /* load old value */
 625                 "and %0, %2, %7\n\t"
 626                 "cmplw %4, %0\n\t"              /* compare */
 627                 "bne- 2f\n\t"                   /* exit if not equal */
 628                 "andc %2, %2, %7\n\t"
 629                 "or %2, %2, %5\n\t"
 630                 "stwcx. %2, 0, %3\n\t"          /* attempt to store */
 631                 "bne- 1b\n\t"                   /* spin if failed */
 632                 "li %0, 1\n\t"                  /* success - retval = 1 */
 633                 "b 3f\n\t"                      /* we've succeeded */
 634                 "2:\n\t"
 635                 "stwcx. %2, 0, %3\n\t"          /* clear reservation (74xx) */
 636                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 637                 "3:\n\t"
 638                 : "=&r" (ret), "=m" (*p), "+&r" (tmp)
 639                 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p),
 640                   "r" (mask)
 641                 : "cr0", "memory");
 642
 643         return (ret);
 644 }
 645
 646 #define _atomic_cmpset_masked_word(a,o,v,m) atomic_cmpset_masked(a, o, v, m)
 647 #endif
 648
 649 static __inline int
 650 atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval)
 651 {
 652         int     ret;
 653
 654         __asm __volatile (
 655                 "1:\tlwarx %0, 0, %2\n\t"       /* load old value */
 656                 "cmplw %3, %0\n\t"              /* compare */
 657                 "bne 2f\n\t"                    /* exit if not equal */
 658                 "stwcx. %4, 0, %2\n\t"          /* attempt to store */
 659                 "bne- 1b\n\t"                   /* spin if failed */
 660                 "li %0, 1\n\t"                  /* success - retval = 1 */
 661                 "b 3f\n\t"                      /* we've succeeded */
 662                 "2:\n\t"
 663                 "stwcx. %0, 0, %2\n\t"          /* clear reservation (74xx) */
 664                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 665                 "3:\n\t"
 666                 : "=&r" (ret), "=m" (*p)
 667                 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
 668                 : "cr0", "memory");
 669
 670         return (ret);
 671 }
 672 static __inline int
 673 atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval)
 674 {
 675         int ret;
 676
 677         __asm __volatile (
 678             #ifdef __powerpc64__
 679                 "1:\tldarx %0, 0, %2\n\t"       /* load old value */
 680                 "cmpld %3, %0\n\t"              /* compare */
 681                 "bne 2f\n\t"                    /* exit if not equal */
 682                 "stdcx. %4, 0, %2\n\t"          /* attempt to store */
 683             #else
 684                 "1:\tlwarx %0, 0, %2\n\t"       /* load old value */
 685                 "cmplw %3, %0\n\t"              /* compare */
 686                 "bne 2f\n\t"                    /* exit if not equal */
 687                 "stwcx. %4, 0, %2\n\t"          /* attempt to store */
 688             #endif
 689                 "bne- 1b\n\t"                   /* spin if failed */
 690                 "li %0, 1\n\t"                  /* success - retval = 1 */
 691                 "b 3f\n\t"                      /* we've succeeded */
 692                 "2:\n\t"
 693             #ifdef __powerpc64__
 694                 "stdcx. %0, 0, %2\n\t"          /* clear reservation (74xx) */
 695             #else
 696                 "stwcx. %0, 0, %2\n\t"          /* clear reservation (74xx) */
 697             #endif
 698                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 699                 "3:\n\t"
 700                 : "=&r" (ret), "=m" (*p)
 701                 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
 702                 : "cr0", "memory");
 703
 704         return (ret);
 705 }
 706
 707 #define ATOMIC_CMPSET_ACQ_REL(type) \
 708     static __inline int \
 709     atomic_cmpset_acq_##type(volatile u_##type *p, \
 710             u_##type cmpval, u_##type newval)\
 711     {\
 712         u_##type retval; \
 713         retval = atomic_cmpset_##type(p, cmpval, newval);\
 714         __ATOMIC_ACQ();\
 715         return (retval);\
 716     }\
 717     static __inline int \
 718     atomic_cmpset_rel_##type(volatile u_##type *p, \
 719             u_##type cmpval, u_##type newval)\
 720     {\
 721         __ATOMIC_REL();\
 722         return (atomic_cmpset_##type(p, cmpval, newval));\
 723     }\
 724     struct hack
 725
 726 ATOMIC_CMPSET_ACQ_REL(int);
 727 ATOMIC_CMPSET_ACQ_REL(long);
 728
 729
 730 #define atomic_cmpset_8         atomic_cmpset_char
 731 #define atomic_cmpset_acq_8     atomic_cmpset_acq_char
 732 #define atomic_cmpset_rel_8     atomic_cmpset_rel_char
 733
 734 #define atomic_cmpset_16        atomic_cmpset_short
 735 #define atomic_cmpset_acq_16    atomic_cmpset_acq_short
 736 #define atomic_cmpset_rel_16    atomic_cmpset_rel_short
 737
 738 #define atomic_cmpset_32        atomic_cmpset_int
 739 #define atomic_cmpset_acq_32    atomic_cmpset_acq_int
 740 #define atomic_cmpset_rel_32    atomic_cmpset_rel_int
 741
 742 #ifdef __powerpc64__
 743 #define atomic_cmpset_64        atomic_cmpset_long
 744 #define atomic_cmpset_acq_64    atomic_cmpset_acq_long
 745 #define atomic_cmpset_rel_64    atomic_cmpset_rel_long
 746
 747 #define atomic_cmpset_ptr       atomic_cmpset_long
 748 #define atomic_cmpset_acq_ptr   atomic_cmpset_acq_long
 749 #define atomic_cmpset_rel_ptr   atomic_cmpset_rel_long
 750 #else
 751 #define atomic_cmpset_ptr       atomic_cmpset_int
 752 #define atomic_cmpset_acq_ptr   atomic_cmpset_acq_int
 753 #define atomic_cmpset_rel_ptr   atomic_cmpset_rel_int
 754 #endif
 755
 756 /*
 757  * Atomically compare the value stored at *p with *cmpval and if the
 758  * two values are equal, update the value of *p with newval. Returns
 759  * zero if the compare failed and sets *cmpval to the read value from *p,
 760  * nonzero otherwise.
 761  */
 762 #ifdef ISA_206_ATOMICS
 763 static __inline int
 764 atomic_fcmpset_char(volatile u_char *p, u_char *cmpval, u_char newval)
 765 {
 766         int     ret;
 767
 768         __asm __volatile (
 769                 "lbarx %0, 0, %3\n\t"           /* load old value */
 770                 "cmplw %4, %0\n\t"              /* compare */
 771                 "bne- 1f\n\t"                   /* exit if not equal */
 772                 "stbcx. %5, 0, %3\n\t"          /* attempt to store */
 773                 "bne- 1f\n\t"                   /* exit if failed */
 774                 "li %0, 1\n\t"                  /* success - retval = 1 */
 775                 "b 2f\n\t"                      /* we've succeeded */
 776                 "1:\n\t"
 777                 "stbcx. %0, 0, %3\n\t"          /* clear reservation (74xx) */
 778                 "stwx %0, 0, %7\n\t"
 779                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 780                 "2:\n\t"
 781                 : "=&r" (ret), "=m" (*p), "=m" (*cmpval)
 782                 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval)
 783                 : "cr0", "memory");
 784
 785         return (ret);
 786 }
 787
 788 static __inline int
 789 atomic_fcmpset_short(volatile u_short *p, u_short *cmpval, u_short newval)
 790 {
 791         int     ret;
 792
 793         __asm __volatile (
 794                 "lharx %0, 0, %3\n\t"           /* load old value */
 795                 "cmplw %4, %0\n\t"              /* compare */
 796                 "bne- 1f\n\t"                   /* exit if not equal */
 797                 "sthcx. %5, 0, %3\n\t"          /* attempt to store */
 798                 "bne- 1f\n\t"                   /* exit if failed */
 799                 "li %0, 1\n\t"                  /* success - retval = 1 */
 800                 "b 2f\n\t"                      /* we've succeeded */
 801                 "1:\n\t"
 802                 "sthcx. %0, 0, %3\n\t"          /* clear reservation (74xx) */
 803                 "stwx %0, 0, %7\n\t"
 804                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 805                 "2:\n\t"
 806                 : "=&r" (ret), "=m" (*p), "=m" (*cmpval)
 807                 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval)
 808                 : "cr0", "memory");
 809
 810         return (ret);
 811 }
 812 #endif  /* ISA_206_ATOMICS */
 813
 814 static __inline int
 815 atomic_fcmpset_int(volatile u_int *p, u_int *cmpval, u_int newval)
 816 {
 817         int     ret;
 818
 819         __asm __volatile (
 820                 "lwarx %0, 0, %3\n\t"           /* load old value */
 821                 "cmplw %4, %0\n\t"              /* compare */
 822                 "bne 1f\n\t"                    /* exit if not equal */
 823                 "stwcx. %5, 0, %3\n\t"          /* attempt to store */
 824                 "bne- 1f\n\t"                   /* exit if failed */
 825                 "li %0, 1\n\t"                  /* success - retval = 1 */
 826                 "b 2f\n\t"                      /* we've succeeded */
 827                 "1:\n\t"
 828                 "stwcx. %0, 0, %3\n\t"          /* clear reservation (74xx) */
 829                 "stwx %0, 0, %7\n\t"
 830                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 831                 "2:\n\t"
 832                 : "=&r" (ret), "=m" (*p), "=m" (*cmpval)
 833                 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval)
 834                 : "cr0", "memory");
 835
 836         return (ret);
 837 }
 838 static __inline int
 839 atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval)
 840 {
 841         int ret;
 842
 843         __asm __volatile (
 844             #ifdef __powerpc64__
 845                 "ldarx %0, 0, %3\n\t"           /* load old value */
 846                 "cmpld %4, %0\n\t"              /* compare */
 847                 "bne 1f\n\t"                    /* exit if not equal */
 848                 "stdcx. %5, 0, %3\n\t"          /* attempt to store */
 849             #else
 850                 "lwarx %0, 0, %3\n\t"           /* load old value */
 851                 "cmplw %4, %0\n\t"              /* compare */
 852                 "bne 1f\n\t"                    /* exit if not equal */
 853                 "stwcx. %5, 0, %3\n\t"          /* attempt to store */
 854             #endif
 855                 "bne- 1f\n\t"                   /* exit if failed */
 856                 "li %0, 1\n\t"                  /* success - retval = 1 */
 857                 "b 2f\n\t"                      /* we've succeeded */
 858                 "1:\n\t"
 859             #ifdef __powerpc64__
 860                 "stdcx. %0, 0, %3\n\t"          /* clear reservation (74xx) */
 861                 "stdx %0, 0, %7\n\t"
 862             #else
 863                 "stwcx. %0, 0, %3\n\t"          /* clear reservation (74xx) */
 864                 "stwx %0, 0, %7\n\t"
 865             #endif
 866                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 867                 "2:\n\t"
 868                 : "=&r" (ret), "=m" (*p), "=m" (*cmpval)
 869                 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval)
 870                 : "cr0", "memory");
 871
 872         return (ret);
 873 }
 874
 875 #define ATOMIC_FCMPSET_ACQ_REL(type) \
 876     static __inline int \
 877     atomic_fcmpset_acq_##type(volatile u_##type *p, \
 878             u_##type *cmpval, u_##type newval)\
 879     {\
 880         u_##type retval; \
 881         retval = atomic_fcmpset_##type(p, cmpval, newval);\
 882         __ATOMIC_ACQ();\
 883         return (retval);\
 884     }\
 885     static __inline int \
 886     atomic_fcmpset_rel_##type(volatile u_##type *p, \
 887             u_##type *cmpval, u_##type newval)\
 888     {\
 889         __ATOMIC_REL();\
 890         return (atomic_fcmpset_##type(p, cmpval, newval));\
 891     }\
 892     struct hack
 893
 894 ATOMIC_FCMPSET_ACQ_REL(int);
 895 ATOMIC_FCMPSET_ACQ_REL(long);
 896
 897 #define atomic_fcmpset_8        atomic_fcmpset_char
 898 #define atomic_fcmpset_acq_8    atomic_fcmpset_acq_char
 899 #define atomic_fcmpset_rel_8    atomic_fcmpset_rel_char
 900
 901 #define atomic_fcmpset_16       atomic_fcmpset_short
 902 #define atomic_fcmpset_acq_16   atomic_fcmpset_acq_short
 903 #define atomic_fcmpset_rel_16   atomic_fcmpset_rel_short
 904
 905 #define atomic_fcmpset_32       atomic_fcmpset_int
 906 #define atomic_fcmpset_acq_32   atomic_fcmpset_acq_int
 907 #define atomic_fcmpset_rel_32   atomic_fcmpset_rel_int
 908
 909 #ifdef __powerpc64__
 910 #define atomic_fcmpset_64       atomic_fcmpset_long
 911 #define atomic_fcmpset_acq_64   atomic_fcmpset_acq_long
 912 #define atomic_fcmpset_rel_64   atomic_fcmpset_rel_long
 913
 914 #define atomic_fcmpset_ptr      atomic_fcmpset_long
 915 #define atomic_fcmpset_acq_ptr  atomic_fcmpset_acq_long
 916 #define atomic_fcmpset_rel_ptr  atomic_fcmpset_rel_long
 917 #else
 918 #define atomic_fcmpset_ptr      atomic_fcmpset_int
 919 #define atomic_fcmpset_acq_ptr  atomic_fcmpset_acq_int
 920 #define atomic_fcmpset_rel_ptr  atomic_fcmpset_rel_int
 921 #endif
 922
 923 static __inline u_int
 924 atomic_fetchadd_int(volatile u_int *p, u_int v)
 925 {
 926         u_int value;
 927
 928         do {
 929                 value = *p;
 930         } while (!atomic_cmpset_int(p, value, value + v));
 931         return (value);
 932 }
 933
 934 static __inline u_long
 935 atomic_fetchadd_long(volatile u_long *p, u_long v)
 936 {
 937         u_long value;
 938
 939         do {
 940                 value = *p;
 941         } while (!atomic_cmpset_long(p, value, value + v));
 942         return (value);
 943 }
 944
 945 static __inline u_int
 946 atomic_swap_32(volatile u_int *p, u_int v)
 947 {
 948         u_int prev;
 949
 950         __asm __volatile(
 951         "1:     lwarx   %0,0,%2\n"
 952         "       stwcx.  %3,0,%2\n"
 953         "       bne-    1b\n"
 954         : "=&r" (prev), "+m" (*(volatile u_int *)p)
 955         : "r" (p), "r" (v)
 956         : "cr0", "memory");
 957
 958         return (prev);
 959 }
 960
 961 #ifdef __powerpc64__
 962 static __inline u_long
 963 atomic_swap_64(volatile u_long *p, u_long v)
 964 {
 965         u_long prev;
 966
 967         __asm __volatile(
 968         "1:     ldarx   %0,0,%2\n"
 969         "       stdcx.  %3,0,%2\n"
 970         "       bne-    1b\n"
 971         : "=&r" (prev), "+m" (*(volatile u_long *)p)
 972         : "r" (p), "r" (v)
 973         : "cr0", "memory");
 974
 975         return (prev);
 976 }
 977 #endif
 978
 979 #define atomic_fetchadd_32      atomic_fetchadd_int
 980 #define atomic_swap_int         atomic_swap_32
 981
 982 #ifdef __powerpc64__
 983 #define atomic_fetchadd_64      atomic_fetchadd_long
 984 #define atomic_swap_long        atomic_swap_64
 985 #define atomic_swap_ptr         atomic_swap_64
 986 #else
 987 #define atomic_swap_long(p,v)   atomic_swap_32((volatile u_int *)(p), v)
 988 #define atomic_swap_ptr(p,v)    atomic_swap_32((volatile u_int *)(p), v)
 989 #endif
 990
 991 static __inline void
 992 atomic_thread_fence_acq(void)
 993 {
 994
 995         powerpc_lwsync();
 996 }
 997
 998 static __inline void
 999 atomic_thread_fence_rel(void)
1000 {
1001
1002         powerpc_lwsync();
1003 }
1004
1005 static __inline void
1006 atomic_thread_fence_acq_rel(void)
1007 {
1008
1009         powerpc_lwsync();
1010 }
1011
1012 static __inline void
1013 atomic_thread_fence_seq_cst(void)
1014 {
1015
1016         __asm __volatile("sync" : : : "memory");
1017 }
1018
1019 #ifndef ISA_206_ATOMICS
1020 #include <sys/_atomic_subword.h>
1021 #endif
1022
1023 /* These need sys/_atomic_subword.h on non-ISA-2.06-atomic platforms. */
1024 ATOMIC_CMPSET_ACQ_REL(char);
1025 ATOMIC_CMPSET_ACQ_REL(short);
1026
1027 ATOMIC_FCMPSET_ACQ_REL(char);
1028 ATOMIC_FCMPSET_ACQ_REL(short);
1029
1030 #undef __ATOMIC_REL
1031 #undef __ATOMIC_ACQ
1032
1033 #endif /* ! _MACHINE_ATOMIC_H_ */