sys/powerpc/include/atomic.h

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2008 Marcel Moolenaar
   5  * Copyright (c) 2001 Benno Rice
   6  * Copyright (c) 2001 David E. O'Brien
   7  * Copyright (c) 1998 Doug Rabson
   8  * All rights reserved.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  29  * SUCH DAMAGE.
  30  *
  31  * $FreeBSD$
  32  */
  33
  34 #ifndef _MACHINE_ATOMIC_H_
  35 #define _MACHINE_ATOMIC_H_
  36
  37 #ifndef _SYS_CDEFS_H_
  38 #error this file needs sys/cdefs.h as a prerequisite
  39 #endif
  40
  41 #include <sys/atomic_common.h>
  42
  43 #ifndef __powerpc64__
  44 #include <sys/_atomic64e.h>
  45 #endif
  46
  47 /*
  48  * The __ATOMIC_REL/ACQ() macros provide memory barriers only in conjunction
  49  * with the atomic lXarx/stXcx. sequences below. They are not exposed outside
  50  * of this file. See also Appendix B.2 of Book II of the architecture manual.
  51  *
  52  * Note that not all Book-E processors accept the light-weight sync variant.
  53  * In particular, early models of E500 cores are known to wedge. Bank on all
  54  * 64-bit capable CPUs to accept lwsync properly and pressimize 32-bit CPUs
  55  * to use the heavier-weight sync.
  56  */
  57
  58 #ifdef __powerpc64__
  59 #define mb()            __asm __volatile("sync" : : : "memory")
  60 #define rmb()           __asm __volatile("lwsync" : : : "memory")
  61 #define wmb()           __asm __volatile("lwsync" : : : "memory")
  62 #define __ATOMIC_REL()  __asm __volatile("lwsync" : : : "memory")
  63 #define __ATOMIC_ACQ()  __asm __volatile("isync" : : : "memory")
  64 #else
  65 #define mb()            __asm __volatile("sync" : : : "memory")
  66 #define rmb()           __asm __volatile("sync" : : : "memory")
  67 #define wmb()           __asm __volatile("sync" : : : "memory")
  68 #define __ATOMIC_REL()  __asm __volatile("sync" : : : "memory")
  69 #define __ATOMIC_ACQ()  __asm __volatile("isync" : : : "memory")
  70 #endif
  71
  72 static __inline void
  73 powerpc_lwsync(void)
  74 {
  75
  76 #ifdef __powerpc64__
  77         __asm __volatile("lwsync" : : : "memory");
  78 #else
  79         __asm __volatile("sync" : : : "memory");
  80 #endif
  81 }
  82
  83 /*
  84  * atomic_add(p, v)
  85  * { *p += v; }
  86  */
  87
  88 #define __atomic_add_int(p, v, t)                               \
  89     __asm __volatile(                                           \
  90         "1:     lwarx   %0, 0, %2\n"                            \
  91         "       add     %0, %3, %0\n"                           \
  92         "       stwcx.  %0, 0, %2\n"                            \
  93         "       bne-    1b\n"                                   \
  94         : "=&r" (t), "=m" (*p)                                  \
  95         : "r" (p), "r" (v), "m" (*p)                            \
  96         : "cr0", "memory")                                      \
  97     /* __atomic_add_int */
  98
  99 #ifdef __powerpc64__
 100 #define __atomic_add_long(p, v, t)                              \
 101     __asm __volatile(                                           \
 102         "1:     ldarx   %0, 0, %2\n"                            \
 103         "       add     %0, %3, %0\n"                           \
 104         "       stdcx.  %0, 0, %2\n"                            \
 105         "       bne-    1b\n"                                   \
 106         : "=&r" (t), "=m" (*p)                                  \
 107         : "r" (p), "r" (v), "m" (*p)                            \
 108         : "cr0", "memory")                                      \
 109     /* __atomic_add_long */
 110 #else
 111 #define __atomic_add_long(p, v, t)                              \
 112     __asm __volatile(                                           \
 113         "1:     lwarx   %0, 0, %2\n"                            \
 114         "       add     %0, %3, %0\n"                           \
 115         "       stwcx.  %0, 0, %2\n"                            \
 116         "       bne-    1b\n"                                   \
 117         : "=&r" (t), "=m" (*p)                                  \
 118         : "r" (p), "r" (v), "m" (*p)                            \
 119         : "cr0", "memory")                                      \
 120     /* __atomic_add_long */
 121 #endif
 122
 123 #define _ATOMIC_ADD(type)                                       \
 124     static __inline void                                        \
 125     atomic_add_##type(volatile u_##type *p, u_##type v) {       \
 126         u_##type t;                                             \
 127         __atomic_add_##type(p, v, t);                           \
 128     }                                                           \
 129                                                                 \
 130     static __inline void                                        \
 131     atomic_add_acq_##type(volatile u_##type *p, u_##type v) {   \
 132         u_##type t;                                             \
 133         __atomic_add_##type(p, v, t);                           \
 134         __ATOMIC_ACQ();                                         \
 135     }                                                           \
 136                                                                 \
 137     static __inline void                                        \
 138     atomic_add_rel_##type(volatile u_##type *p, u_##type v) {   \
 139         u_##type t;                                             \
 140         __ATOMIC_REL();                                         \
 141         __atomic_add_##type(p, v, t);                           \
 142     }                                                           \
 143     /* _ATOMIC_ADD */
 144
 145 _ATOMIC_ADD(int)
 146 _ATOMIC_ADD(long)
 147
 148 #define atomic_add_32           atomic_add_int
 149 #define atomic_add_acq_32       atomic_add_acq_int
 150 #define atomic_add_rel_32       atomic_add_rel_int
 151
 152 #ifdef __powerpc64__
 153 #define atomic_add_64           atomic_add_long
 154 #define atomic_add_acq_64       atomic_add_acq_long
 155 #define atomic_add_rel_64       atomic_add_rel_long
 156
 157 #define atomic_add_ptr          atomic_add_long
 158 #define atomic_add_acq_ptr      atomic_add_acq_long
 159 #define atomic_add_rel_ptr      atomic_add_rel_long
 160 #else
 161 #define atomic_add_ptr          atomic_add_int
 162 #define atomic_add_acq_ptr      atomic_add_acq_int
 163 #define atomic_add_rel_ptr      atomic_add_rel_int
 164 #endif
 165 #undef _ATOMIC_ADD
 166 #undef __atomic_add_long
 167 #undef __atomic_add_int
 168
 169 /*
 170  * atomic_clear(p, v)
 171  * { *p &= ~v; }
 172  */
 173
 174 #define __atomic_clear_int(p, v, t)                             \
 175     __asm __volatile(                                           \
 176         "1:     lwarx   %0, 0, %2\n"                            \
 177         "       andc    %0, %0, %3\n"                           \
 178         "       stwcx.  %0, 0, %2\n"                            \
 179         "       bne-    1b\n"                                   \
 180         : "=&r" (t), "=m" (*p)                                  \
 181         : "r" (p), "r" (v), "m" (*p)                            \
 182         : "cr0", "memory")                                      \
 183     /* __atomic_clear_int */
 184
 185 #ifdef __powerpc64__
 186 #define __atomic_clear_long(p, v, t)                            \
 187     __asm __volatile(                                           \
 188         "1:     ldarx   %0, 0, %2\n"                            \
 189         "       andc    %0, %0, %3\n"                           \
 190         "       stdcx.  %0, 0, %2\n"                            \
 191         "       bne-    1b\n"                                   \
 192         : "=&r" (t), "=m" (*p)                                  \
 193         : "r" (p), "r" (v), "m" (*p)                            \
 194         : "cr0", "memory")                                      \
 195     /* __atomic_clear_long */
 196 #else
 197 #define __atomic_clear_long(p, v, t)                            \
 198     __asm __volatile(                                           \
 199         "1:     lwarx   %0, 0, %2\n"                            \
 200         "       andc    %0, %0, %3\n"                           \
 201         "       stwcx.  %0, 0, %2\n"                            \
 202         "       bne-    1b\n"                                   \
 203         : "=&r" (t), "=m" (*p)                                  \
 204         : "r" (p), "r" (v), "m" (*p)                            \
 205         : "cr0", "memory")                                      \
 206     /* __atomic_clear_long */
 207 #endif
 208
 209 #define _ATOMIC_CLEAR(type)                                     \
 210     static __inline void                                        \
 211     atomic_clear_##type(volatile u_##type *p, u_##type v) {     \
 212         u_##type t;                                             \
 213         __atomic_clear_##type(p, v, t);                         \
 214     }                                                           \
 215                                                                 \
 216     static __inline void                                        \
 217     atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \
 218         u_##type t;                                             \
 219         __atomic_clear_##type(p, v, t);                         \
 220         __ATOMIC_ACQ();                                         \
 221     }                                                           \
 222                                                                 \
 223     static __inline void                                        \
 224     atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \
 225         u_##type t;                                             \
 226         __ATOMIC_REL();                                         \
 227         __atomic_clear_##type(p, v, t);                         \
 228     }                                                           \
 229     /* _ATOMIC_CLEAR */
 230
 231 _ATOMIC_CLEAR(int)
 232 _ATOMIC_CLEAR(long)
 233
 234 #define atomic_clear_32         atomic_clear_int
 235 #define atomic_clear_acq_32     atomic_clear_acq_int
 236 #define atomic_clear_rel_32     atomic_clear_rel_int
 237
 238 #ifdef __powerpc64__
 239 #define atomic_clear_64         atomic_clear_long
 240 #define atomic_clear_acq_64     atomic_clear_acq_long
 241 #define atomic_clear_rel_64     atomic_clear_rel_long
 242
 243 #define atomic_clear_ptr        atomic_clear_long
 244 #define atomic_clear_acq_ptr    atomic_clear_acq_long
 245 #define atomic_clear_rel_ptr    atomic_clear_rel_long
 246 #else
 247 #define atomic_clear_ptr        atomic_clear_int
 248 #define atomic_clear_acq_ptr    atomic_clear_acq_int
 249 #define atomic_clear_rel_ptr    atomic_clear_rel_int
 250 #endif
 251 #undef _ATOMIC_CLEAR
 252 #undef __atomic_clear_long
 253 #undef __atomic_clear_int
 254
 255 /*
 256  * atomic_cmpset(p, o, n)
 257  */
 258 /* TODO -- see below */
 259
 260 /*
 261  * atomic_load_acq(p)
 262  */
 263 /* TODO -- see below */
 264
 265 /*
 266  * atomic_readandclear(p)
 267  */
 268 /* TODO -- see below */
 269
 270 /*
 271  * atomic_set(p, v)
 272  * { *p |= v; }
 273  */
 274
 275 #define __atomic_set_int(p, v, t)                               \
 276     __asm __volatile(                                           \
 277         "1:     lwarx   %0, 0, %2\n"                            \
 278         "       or      %0, %3, %0\n"                           \
 279         "       stwcx.  %0, 0, %2\n"                            \
 280         "       bne-    1b\n"                                   \
 281         : "=&r" (t), "=m" (*p)                                  \
 282         : "r" (p), "r" (v), "m" (*p)                            \
 283         : "cr0", "memory")                                      \
 284     /* __atomic_set_int */
 285
 286 #ifdef __powerpc64__
 287 #define __atomic_set_long(p, v, t)                              \
 288     __asm __volatile(                                           \
 289         "1:     ldarx   %0, 0, %2\n"                            \
 290         "       or      %0, %3, %0\n"                           \
 291         "       stdcx.  %0, 0, %2\n"                            \
 292         "       bne-    1b\n"                                   \
 293         : "=&r" (t), "=m" (*p)                                  \
 294         : "r" (p), "r" (v), "m" (*p)                            \
 295         : "cr0", "memory")                                      \
 296     /* __atomic_set_long */
 297 #else
 298 #define __atomic_set_long(p, v, t)                              \
 299     __asm __volatile(                                           \
 300         "1:     lwarx   %0, 0, %2\n"                            \
 301         "       or      %0, %3, %0\n"                           \
 302         "       stwcx.  %0, 0, %2\n"                            \
 303         "       bne-    1b\n"                                   \
 304         : "=&r" (t), "=m" (*p)                                  \
 305         : "r" (p), "r" (v), "m" (*p)                            \
 306         : "cr0", "memory")                                      \
 307     /* __atomic_set_long */
 308 #endif
 309
 310 #define _ATOMIC_SET(type)                                       \
 311     static __inline void                                        \
 312     atomic_set_##type(volatile u_##type *p, u_##type v) {       \
 313         u_##type t;                                             \
 314         __atomic_set_##type(p, v, t);                           \
 315     }                                                           \
 316                                                                 \
 317     static __inline void                                        \
 318     atomic_set_acq_##type(volatile u_##type *p, u_##type v) {   \
 319         u_##type t;                                             \
 320         __atomic_set_##type(p, v, t);                           \
 321         __ATOMIC_ACQ();                                         \
 322     }                                                           \
 323                                                                 \
 324     static __inline void                                        \
 325     atomic_set_rel_##type(volatile u_##type *p, u_##type v) {   \
 326         u_##type t;                                             \
 327         __ATOMIC_REL();                                         \
 328         __atomic_set_##type(p, v, t);                           \
 329     }                                                           \
 330     /* _ATOMIC_SET */
 331
 332 _ATOMIC_SET(int)
 333 _ATOMIC_SET(long)
 334
 335 #define atomic_set_32           atomic_set_int
 336 #define atomic_set_acq_32       atomic_set_acq_int
 337 #define atomic_set_rel_32       atomic_set_rel_int
 338
 339 #ifdef __powerpc64__
 340 #define atomic_set_64           atomic_set_long
 341 #define atomic_set_acq_64       atomic_set_acq_long
 342 #define atomic_set_rel_64       atomic_set_rel_long
 343
 344 #define atomic_set_ptr          atomic_set_long
 345 #define atomic_set_acq_ptr      atomic_set_acq_long
 346 #define atomic_set_rel_ptr      atomic_set_rel_long
 347 #else
 348 #define atomic_set_ptr          atomic_set_int
 349 #define atomic_set_acq_ptr      atomic_set_acq_int
 350 #define atomic_set_rel_ptr      atomic_set_rel_int
 351 #endif
 352 #undef _ATOMIC_SET
 353 #undef __atomic_set_long
 354 #undef __atomic_set_int
 355
 356 /*
 357  * atomic_subtract(p, v)
 358  * { *p -= v; }
 359  */
 360
 361 #define __atomic_subtract_int(p, v, t)                          \
 362     __asm __volatile(                                           \
 363         "1:     lwarx   %0, 0, %2\n"                            \
 364         "       subf    %0, %3, %0\n"                           \
 365         "       stwcx.  %0, 0, %2\n"                            \
 366         "       bne-    1b\n"                                   \
 367         : "=&r" (t), "=m" (*p)                                  \
 368         : "r" (p), "r" (v), "m" (*p)                            \
 369         : "cr0", "memory")                                      \
 370     /* __atomic_subtract_int */
 371
 372 #ifdef __powerpc64__
 373 #define __atomic_subtract_long(p, v, t)                         \
 374     __asm __volatile(                                           \
 375         "1:     ldarx   %0, 0, %2\n"                            \
 376         "       subf    %0, %3, %0\n"                           \
 377         "       stdcx.  %0, 0, %2\n"                            \
 378         "       bne-    1b\n"                                   \
 379         : "=&r" (t), "=m" (*p)                                  \
 380         : "r" (p), "r" (v), "m" (*p)                            \
 381         : "cr0", "memory")                                      \
 382     /* __atomic_subtract_long */
 383 #else
 384 #define __atomic_subtract_long(p, v, t)                         \
 385     __asm __volatile(                                           \
 386         "1:     lwarx   %0, 0, %2\n"                            \
 387         "       subf    %0, %3, %0\n"                           \
 388         "       stwcx.  %0, 0, %2\n"                            \
 389         "       bne-    1b\n"                                   \
 390         : "=&r" (t), "=m" (*p)                                  \
 391         : "r" (p), "r" (v), "m" (*p)                            \
 392         : "cr0", "memory")                                      \
 393     /* __atomic_subtract_long */
 394 #endif
 395
 396 #define _ATOMIC_SUBTRACT(type)                                          \
 397     static __inline void                                                \
 398     atomic_subtract_##type(volatile u_##type *p, u_##type v) {          \
 399         u_##type t;                                                     \
 400         __atomic_subtract_##type(p, v, t);                              \
 401     }                                                                   \
 402                                                                         \
 403     static __inline void                                                \
 404     atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) {      \
 405         u_##type t;                                                     \
 406         __atomic_subtract_##type(p, v, t);                              \
 407         __ATOMIC_ACQ();                                                 \
 408     }                                                                   \
 409                                                                         \
 410     static __inline void                                                \
 411     atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) {      \
 412         u_##type t;                                                     \
 413         __ATOMIC_REL();                                                 \
 414         __atomic_subtract_##type(p, v, t);                              \
 415     }                                                                   \
 416     /* _ATOMIC_SUBTRACT */
 417
 418 _ATOMIC_SUBTRACT(int)
 419 _ATOMIC_SUBTRACT(long)
 420
 421 #define atomic_subtract_32      atomic_subtract_int
 422 #define atomic_subtract_acq_32  atomic_subtract_acq_int
 423 #define atomic_subtract_rel_32  atomic_subtract_rel_int
 424
 425 #ifdef __powerpc64__
 426 #define atomic_subtract_64      atomic_subtract_long
 427 #define atomic_subtract_acq_64  atomic_subract_acq_long
 428 #define atomic_subtract_rel_64  atomic_subtract_rel_long
 429
 430 #define atomic_subtract_ptr     atomic_subtract_long
 431 #define atomic_subtract_acq_ptr atomic_subtract_acq_long
 432 #define atomic_subtract_rel_ptr atomic_subtract_rel_long
 433 #else
 434 #define atomic_subtract_ptr     atomic_subtract_int
 435 #define atomic_subtract_acq_ptr atomic_subtract_acq_int
 436 #define atomic_subtract_rel_ptr atomic_subtract_rel_int
 437 #endif
 438 #undef _ATOMIC_SUBTRACT
 439 #undef __atomic_subtract_long
 440 #undef __atomic_subtract_int
 441
 442 /*
 443  * atomic_store_rel(p, v)
 444  */
 445 /* TODO -- see below */
 446
 447 /*
 448  * Old/original implementations that still need revisiting.
 449  */
 450
 451 static __inline u_int
 452 atomic_readandclear_int(volatile u_int *addr)
 453 {
 454         u_int result,temp;
 455
 456         __asm __volatile (
 457                 "\tsync\n"                      /* drain writes */
 458                 "1:\tlwarx %0, 0, %3\n\t"       /* load old value */
 459                 "li %1, 0\n\t"                  /* load new value */
 460                 "stwcx. %1, 0, %3\n\t"          /* attempt to store */
 461                 "bne- 1b\n\t"                   /* spin if failed */
 462                 : "=&r"(result), "=&r"(temp), "=m" (*addr)
 463                 : "r" (addr), "m" (*addr)
 464                 : "cr0", "memory");
 465
 466         return (result);
 467 }
 468
 469 #ifdef __powerpc64__
 470 static __inline u_long
 471 atomic_readandclear_long(volatile u_long *addr)
 472 {
 473         u_long result,temp;
 474
 475         __asm __volatile (
 476                 "\tsync\n"                      /* drain writes */
 477                 "1:\tldarx %0, 0, %3\n\t"       /* load old value */
 478                 "li %1, 0\n\t"                  /* load new value */
 479                 "stdcx. %1, 0, %3\n\t"          /* attempt to store */
 480                 "bne- 1b\n\t"                   /* spin if failed */
 481                 : "=&r"(result), "=&r"(temp), "=m" (*addr)
 482                 : "r" (addr), "m" (*addr)
 483                 : "cr0", "memory");
 484
 485         return (result);
 486 }
 487 #endif
 488
 489 #define atomic_readandclear_32          atomic_readandclear_int
 490
 491 #ifdef __powerpc64__
 492 #define atomic_readandclear_64          atomic_readandclear_long
 493
 494 #define atomic_readandclear_ptr         atomic_readandclear_long
 495 #else
 496 static __inline u_long
 497 atomic_readandclear_long(volatile u_long *addr)
 498 {
 499
 500         return ((u_long)atomic_readandclear_int((volatile u_int *)addr));
 501 }
 502
 503 #define atomic_readandclear_ptr         atomic_readandclear_int
 504 #endif
 505
 506 /*
 507  * We assume that a = b will do atomic loads and stores.
 508  */
 509 #define ATOMIC_STORE_LOAD(TYPE)                                 \
 510 static __inline u_##TYPE                                        \
 511 atomic_load_acq_##TYPE(volatile u_##TYPE *p)                    \
 512 {                                                               \
 513         u_##TYPE v;                                             \
 514                                                                 \
 515         v = *p;                                                 \
 516         powerpc_lwsync();                                       \
 517         return (v);                                             \
 518 }                                                               \
 519                                                                 \
 520 static __inline void                                            \
 521 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)       \
 522 {                                                               \
 523                                                                 \
 524         powerpc_lwsync();                                       \
 525         *p = v;                                                 \
 526 }
 527
 528 ATOMIC_STORE_LOAD(int)
 529
 530 #define atomic_load_acq_32      atomic_load_acq_int
 531 #define atomic_store_rel_32     atomic_store_rel_int
 532
 533 #ifdef __powerpc64__
 534 ATOMIC_STORE_LOAD(long)
 535
 536 #define atomic_load_acq_64      atomic_load_acq_long
 537 #define atomic_store_rel_64     atomic_store_rel_long
 538
 539 #define atomic_load_acq_ptr     atomic_load_acq_long
 540 #define atomic_store_rel_ptr    atomic_store_rel_long
 541 #else
 542 static __inline u_long
 543 atomic_load_acq_long(volatile u_long *addr)
 544 {
 545
 546         return ((u_long)atomic_load_acq_int((volatile u_int *)addr));
 547 }
 548
 549 static __inline void
 550 atomic_store_rel_long(volatile u_long *addr, u_long val)
 551 {
 552
 553         atomic_store_rel_int((volatile u_int *)addr, (u_int)val);
 554 }
 555
 556 #define atomic_load_acq_ptr     atomic_load_acq_int
 557 #define atomic_store_rel_ptr    atomic_store_rel_int
 558 #endif
 559 #undef ATOMIC_STORE_LOAD
 560
 561 /*
 562  * Atomically compare the value stored at *p with cmpval and if the
 563  * two values are equal, update the value of *p with newval. Returns
 564  * zero if the compare failed, nonzero otherwise.
 565  */
 566 #ifdef ISA_206_ATOMICS
 567 static __inline int
 568 atomic_cmpset_char(volatile u_char *p, u_char cmpval, u_char newval)
 569 {
 570         int     ret;
 571
 572         __asm __volatile (
 573                 "1:\tlbarx %0, 0, %2\n\t"       /* load old value */
 574                 "cmplw %3, %0\n\t"              /* compare */
 575                 "bne- 2f\n\t"                   /* exit if not equal */
 576                 "stbcx. %4, 0, %2\n\t"          /* attempt to store */
 577                 "bne- 1b\n\t"                   /* spin if failed */
 578                 "li %0, 1\n\t"                  /* success - retval = 1 */
 579                 "b 3f\n\t"                      /* we've succeeded */
 580                 "2:\n\t"
 581                 "stbcx. %0, 0, %2\n\t"          /* clear reservation (74xx) */
 582                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 583                 "3:\n\t"
 584                 : "=&r" (ret), "=m" (*p)
 585                 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
 586                 : "cr0", "memory");
 587
 588         return (ret);
 589 }
 590
 591 static __inline int
 592 atomic_cmpset_short(volatile u_short *p, u_short cmpval, u_short newval)
 593 {
 594         int     ret;
 595
 596         __asm __volatile (
 597                 "1:\tlharx %0, 0, %2\n\t"       /* load old value */
 598                 "cmplw %3, %0\n\t"              /* compare */
 599                 "bne- 2f\n\t"                   /* exit if not equal */
 600                 "sthcx. %4, 0, %2\n\t"          /* attempt to store */
 601                 "bne- 1b\n\t"                   /* spin if failed */
 602                 "li %0, 1\n\t"                  /* success - retval = 1 */
 603                 "b 3f\n\t"                      /* we've succeeded */
 604                 "2:\n\t"
 605                 "sthcx. %0, 0, %2\n\t"          /* clear reservation (74xx) */
 606                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 607                 "3:\n\t"
 608                 : "=&r" (ret), "=m" (*p)
 609                 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
 610                 : "cr0", "memory");
 611
 612         return (ret);
 613 }
 614 #else
 615 static __inline int
 616 atomic_cmpset_masked(uint32_t *p, uint32_t cmpval, uint32_t newval,
 617     uint32_t mask)
 618 {
 619         int             ret;
 620         uint32_t        tmp;
 621
 622         __asm __volatile (
 623                 "1:\tlwarx %2, 0, %3\n\t"       /* load old value */
 624                 "and %0, %2, %7\n\t"
 625                 "cmplw %4, %0\n\t"              /* compare */
 626                 "bne- 2f\n\t"                   /* exit if not equal */
 627                 "andc %2, %2, %7\n\t"
 628                 "or %2, %2, %5\n\t"
 629                 "stwcx. %2, 0, %3\n\t"          /* attempt to store */
 630                 "bne- 1b\n\t"                   /* spin if failed */
 631                 "li %0, 1\n\t"                  /* success - retval = 1 */
 632                 "b 3f\n\t"                      /* we've succeeded */
 633                 "2:\n\t"
 634                 "stwcx. %2, 0, %3\n\t"          /* clear reservation (74xx) */
 635                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 636                 "3:\n\t"
 637                 : "=&r" (ret), "=m" (*p), "+&r" (tmp)
 638                 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p),
 639                   "r" (mask)
 640                 : "cr0", "memory");
 641
 642         return (ret);
 643 }
 644
 645 #define _atomic_cmpset_masked_word(a,o,v,m) atomic_cmpset_masked(a, o, v, m)
 646 #endif
 647
 648 static __inline int
 649 atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval)
 650 {
 651         int     ret;
 652
 653         __asm __volatile (
 654                 "1:\tlwarx %0, 0, %2\n\t"       /* load old value */
 655                 "cmplw %3, %0\n\t"              /* compare */
 656                 "bne- 2f\n\t"                   /* exit if not equal */
 657                 "stwcx. %4, 0, %2\n\t"          /* attempt to store */
 658                 "bne- 1b\n\t"                   /* spin if failed */
 659                 "li %0, 1\n\t"                  /* success - retval = 1 */
 660                 "b 3f\n\t"                      /* we've succeeded */
 661                 "2:\n\t"
 662                 "stwcx. %0, 0, %2\n\t"          /* clear reservation (74xx) */
 663                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 664                 "3:\n\t"
 665                 : "=&r" (ret), "=m" (*p)
 666                 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
 667                 : "cr0", "memory");
 668
 669         return (ret);
 670 }
 671 static __inline int
 672 atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval)
 673 {
 674         int ret;
 675
 676         __asm __volatile (
 677             #ifdef __powerpc64__
 678                 "1:\tldarx %0, 0, %2\n\t"       /* load old value */
 679                 "cmpld %3, %0\n\t"              /* compare */
 680                 "bne- 2f\n\t"                   /* exit if not equal */
 681                 "stdcx. %4, 0, %2\n\t"          /* attempt to store */
 682             #else
 683                 "1:\tlwarx %0, 0, %2\n\t"       /* load old value */
 684                 "cmplw %3, %0\n\t"              /* compare */
 685                 "bne- 2f\n\t"                   /* exit if not equal */
 686                 "stwcx. %4, 0, %2\n\t"          /* attempt to store */
 687             #endif
 688                 "bne- 1b\n\t"                   /* spin if failed */
 689                 "li %0, 1\n\t"                  /* success - retval = 1 */
 690                 "b 3f\n\t"                      /* we've succeeded */
 691                 "2:\n\t"
 692             #ifdef __powerpc64__
 693                 "stdcx. %0, 0, %2\n\t"          /* clear reservation (74xx) */
 694             #else
 695                 "stwcx. %0, 0, %2\n\t"          /* clear reservation (74xx) */
 696             #endif
 697                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 698                 "3:\n\t"
 699                 : "=&r" (ret), "=m" (*p)
 700                 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
 701                 : "cr0", "memory");
 702
 703         return (ret);
 704 }
 705
 706 #define ATOMIC_CMPSET_ACQ_REL(type) \
 707     static __inline int \
 708     atomic_cmpset_acq_##type(volatile u_##type *p, \
 709             u_##type cmpval, u_##type newval)\
 710     {\
 711         u_##type retval; \
 712         retval = atomic_cmpset_##type(p, cmpval, newval);\
 713         __ATOMIC_ACQ();\
 714         return (retval);\
 715     }\
 716     static __inline int \
 717     atomic_cmpset_rel_##type(volatile u_##type *p, \
 718             u_##type cmpval, u_##type newval)\
 719     {\
 720         __ATOMIC_REL();\
 721         return (atomic_cmpset_##type(p, cmpval, newval));\
 722     }\
 723     struct hack
 724
 725 ATOMIC_CMPSET_ACQ_REL(int);
 726 ATOMIC_CMPSET_ACQ_REL(long);
 727
 728 #ifdef ISA_206_ATOMICS
 729 #define atomic_cmpset_8         atomic_cmpset_char
 730 #endif
 731 #define atomic_cmpset_acq_8     atomic_cmpset_acq_char
 732 #define atomic_cmpset_rel_8     atomic_cmpset_rel_char
 733
 734 #ifdef ISA_206_ATOMICS
 735 #define atomic_cmpset_16        atomic_cmpset_short
 736 #endif
 737 #define atomic_cmpset_acq_16    atomic_cmpset_acq_short
 738 #define atomic_cmpset_rel_16    atomic_cmpset_rel_short
 739
 740 #define atomic_cmpset_32        atomic_cmpset_int
 741 #define atomic_cmpset_acq_32    atomic_cmpset_acq_int
 742 #define atomic_cmpset_rel_32    atomic_cmpset_rel_int
 743
 744 #ifdef __powerpc64__
 745 #define atomic_cmpset_64        atomic_cmpset_long
 746 #define atomic_cmpset_acq_64    atomic_cmpset_acq_long
 747 #define atomic_cmpset_rel_64    atomic_cmpset_rel_long
 748
 749 #define atomic_cmpset_ptr       atomic_cmpset_long
 750 #define atomic_cmpset_acq_ptr   atomic_cmpset_acq_long
 751 #define atomic_cmpset_rel_ptr   atomic_cmpset_rel_long
 752 #else
 753 #define atomic_cmpset_ptr       atomic_cmpset_int
 754 #define atomic_cmpset_acq_ptr   atomic_cmpset_acq_int
 755 #define atomic_cmpset_rel_ptr   atomic_cmpset_rel_int
 756 #endif
 757
 758 /*
 759  * Atomically compare the value stored at *p with *cmpval and if the
 760  * two values are equal, update the value of *p with newval. Returns
 761  * zero if the compare failed and sets *cmpval to the read value from *p,
 762  * nonzero otherwise.
 763  */
 764 #ifdef ISA_206_ATOMICS
 765 static __inline int
 766 atomic_fcmpset_char(volatile u_char *p, u_char *cmpval, u_char newval)
 767 {
 768         int     ret;
 769
 770         __asm __volatile (
 771                 "lbarx %0, 0, %3\n\t"           /* load old value */
 772                 "cmplw %4, %0\n\t"              /* compare */
 773                 "bne- 1f\n\t"                   /* exit if not equal */
 774                 "stbcx. %5, 0, %3\n\t"          /* attempt to store */
 775                 "bne- 1f\n\t"                   /* exit if failed */
 776                 "li %0, 1\n\t"                  /* success - retval = 1 */
 777                 "b 2f\n\t"                      /* we've succeeded */
 778                 "1:\n\t"
 779                 "stbcx. %0, 0, %3\n\t"          /* clear reservation (74xx) */
 780                 "stwx %0, 0, %7\n\t"
 781                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 782                 "2:\n\t"
 783                 : "=&r" (ret), "=m" (*p), "=m" (*cmpval)
 784                 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval)
 785                 : "cr0", "memory");
 786
 787         return (ret);
 788 }
 789
 790 static __inline int
 791 atomic_fcmpset_short(volatile u_short *p, u_short *cmpval, u_short newval)
 792 {
 793         int     ret;
 794
 795         __asm __volatile (
 796                 "lharx %0, 0, %3\n\t"           /* load old value */
 797                 "cmplw %4, %0\n\t"              /* compare */
 798                 "bne- 1f\n\t"                   /* exit if not equal */
 799                 "sthcx. %5, 0, %3\n\t"          /* attempt to store */
 800                 "bne- 1f\n\t"                   /* exit if failed */
 801                 "li %0, 1\n\t"                  /* success - retval = 1 */
 802                 "b 2f\n\t"                      /* we've succeeded */
 803                 "1:\n\t"
 804                 "sthcx. %0, 0, %3\n\t"          /* clear reservation (74xx) */
 805                 "stwx %0, 0, %7\n\t"
 806                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 807                 "2:\n\t"
 808                 : "=&r" (ret), "=m" (*p), "=m" (*cmpval)
 809                 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval)
 810                 : "cr0", "memory");
 811
 812         return (ret);
 813 }
 814 #endif  /* ISA_206_ATOMICS */
 815
 816 static __inline int
 817 atomic_fcmpset_int(volatile u_int *p, u_int *cmpval, u_int newval)
 818 {
 819         int     ret;
 820
 821         __asm __volatile (
 822                 "lwarx %0, 0, %3\n\t"           /* load old value */
 823                 "cmplw %4, %0\n\t"              /* compare */
 824                 "bne- 1f\n\t"                   /* exit if not equal */
 825                 "stwcx. %5, 0, %3\n\t"          /* attempt to store */
 826                 "bne- 1f\n\t"                   /* exit if failed */
 827                 "li %0, 1\n\t"                  /* success - retval = 1 */
 828                 "b 2f\n\t"                      /* we've succeeded */
 829                 "1:\n\t"
 830                 "stwcx. %0, 0, %3\n\t"          /* clear reservation (74xx) */
 831                 "stwx %0, 0, %7\n\t"
 832                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 833                 "2:\n\t"
 834                 : "=&r" (ret), "=m" (*p), "=m" (*cmpval)
 835                 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval)
 836                 : "cr0", "memory");
 837
 838         return (ret);
 839 }
 840 static __inline int
 841 atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval)
 842 {
 843         int ret;
 844
 845         __asm __volatile (
 846             #ifdef __powerpc64__
 847                 "ldarx %0, 0, %3\n\t"           /* load old value */
 848                 "cmpld %4, %0\n\t"              /* compare */
 849                 "bne- 1f\n\t"                   /* exit if not equal */
 850                 "stdcx. %5, 0, %3\n\t"          /* attempt to store */
 851             #else
 852                 "lwarx %0, 0, %3\n\t"           /* load old value */
 853                 "cmplw %4, %0\n\t"              /* compare */
 854                 "bne- 1f\n\t"                   /* exit if not equal */
 855                 "stwcx. %5, 0, %3\n\t"          /* attempt to store */
 856             #endif
 857                 "bne- 1f\n\t"                   /* exit if failed */
 858                 "li %0, 1\n\t"                  /* success - retval = 1 */
 859                 "b 2f\n\t"                      /* we've succeeded */
 860                 "1:\n\t"
 861             #ifdef __powerpc64__
 862                 "stdcx. %0, 0, %3\n\t"          /* clear reservation (74xx) */
 863                 "stdx %0, 0, %7\n\t"
 864             #else
 865                 "stwcx. %0, 0, %3\n\t"          /* clear reservation (74xx) */
 866                 "stwx %0, 0, %7\n\t"
 867             #endif
 868                 "li %0, 0\n\t"                  /* failure - retval = 0 */
 869                 "2:\n\t"
 870                 : "=&r" (ret), "=m" (*p), "=m" (*cmpval)
 871                 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval)
 872                 : "cr0", "memory");
 873
 874         return (ret);
 875 }
 876
 877 #define ATOMIC_FCMPSET_ACQ_REL(type) \
 878     static __inline int \
 879     atomic_fcmpset_acq_##type(volatile u_##type *p, \
 880             u_##type *cmpval, u_##type newval)\
 881     {\
 882         u_##type retval; \
 883         retval = atomic_fcmpset_##type(p, cmpval, newval);\
 884         __ATOMIC_ACQ();\
 885         return (retval);\
 886     }\
 887     static __inline int \
 888     atomic_fcmpset_rel_##type(volatile u_##type *p, \
 889             u_##type *cmpval, u_##type newval)\
 890     {\
 891         __ATOMIC_REL();\
 892         return (atomic_fcmpset_##type(p, cmpval, newval));\
 893     }\
 894     struct hack
 895
 896 ATOMIC_FCMPSET_ACQ_REL(int);
 897 ATOMIC_FCMPSET_ACQ_REL(long);
 898
 899 #ifdef ISA_206_ATOMICS
 900 #define atomic_fcmpset_8        atomic_fcmpset_char
 901 #endif
 902 #define atomic_fcmpset_acq_8    atomic_fcmpset_acq_char
 903 #define atomic_fcmpset_rel_8    atomic_fcmpset_rel_char
 904
 905 #ifdef ISA_206_ATOMICS
 906 #define atomic_fcmpset_16       atomic_fcmpset_short
 907 #endif
 908 #define atomic_fcmpset_acq_16   atomic_fcmpset_acq_short
 909 #define atomic_fcmpset_rel_16   atomic_fcmpset_rel_short
 910
 911 #define atomic_fcmpset_32       atomic_fcmpset_int
 912 #define atomic_fcmpset_acq_32   atomic_fcmpset_acq_int
 913 #define atomic_fcmpset_rel_32   atomic_fcmpset_rel_int
 914
 915 #ifdef __powerpc64__
 916 #define atomic_fcmpset_64       atomic_fcmpset_long
 917 #define atomic_fcmpset_acq_64   atomic_fcmpset_acq_long
 918 #define atomic_fcmpset_rel_64   atomic_fcmpset_rel_long
 919
 920 #define atomic_fcmpset_ptr      atomic_fcmpset_long
 921 #define atomic_fcmpset_acq_ptr  atomic_fcmpset_acq_long
 922 #define atomic_fcmpset_rel_ptr  atomic_fcmpset_rel_long
 923 #else
 924 #define atomic_fcmpset_ptr      atomic_fcmpset_int
 925 #define atomic_fcmpset_acq_ptr  atomic_fcmpset_acq_int
 926 #define atomic_fcmpset_rel_ptr  atomic_fcmpset_rel_int
 927 #endif
 928
 929 static __inline u_int
 930 atomic_fetchadd_int(volatile u_int *p, u_int v)
 931 {
 932         u_int value;
 933
 934         do {
 935                 value = *p;
 936         } while (!atomic_cmpset_int(p, value, value + v));
 937         return (value);
 938 }
 939
 940 static __inline u_long
 941 atomic_fetchadd_long(volatile u_long *p, u_long v)
 942 {
 943         u_long value;
 944
 945         do {
 946                 value = *p;
 947         } while (!atomic_cmpset_long(p, value, value + v));
 948         return (value);
 949 }
 950
 951 static __inline u_int
 952 atomic_swap_32(volatile u_int *p, u_int v)
 953 {
 954         u_int prev;
 955
 956         __asm __volatile(
 957         "1:     lwarx   %0,0,%2\n"
 958         "       stwcx.  %3,0,%2\n"
 959         "       bne-    1b\n"
 960         : "=&r" (prev), "+m" (*(volatile u_int *)p)
 961         : "r" (p), "r" (v)
 962         : "cr0", "memory");
 963
 964         return (prev);
 965 }
 966
 967 #ifdef __powerpc64__
 968 static __inline u_long
 969 atomic_swap_64(volatile u_long *p, u_long v)
 970 {
 971         u_long prev;
 972
 973         __asm __volatile(
 974         "1:     ldarx   %0,0,%2\n"
 975         "       stdcx.  %3,0,%2\n"
 976         "       bne-    1b\n"
 977         : "=&r" (prev), "+m" (*(volatile u_long *)p)
 978         : "r" (p), "r" (v)
 979         : "cr0", "memory");
 980
 981         return (prev);
 982 }
 983 #endif
 984
 985 #define atomic_fetchadd_32      atomic_fetchadd_int
 986 #define atomic_swap_int         atomic_swap_32
 987
 988 #ifdef __powerpc64__
 989 #define atomic_fetchadd_64      atomic_fetchadd_long
 990 #define atomic_swap_long        atomic_swap_64
 991 #define atomic_swap_ptr         atomic_swap_64
 992 #else
 993 #define atomic_swap_long(p,v)   atomic_swap_32((volatile u_int *)(p), v)
 994 #define atomic_swap_ptr(p,v)    atomic_swap_32((volatile u_int *)(p), v)
 995 #endif
 996
 997 static __inline void
 998 atomic_thread_fence_acq(void)
 999 {
1000
1001         powerpc_lwsync();
1002 }
1003
1004 static __inline void
1005 atomic_thread_fence_rel(void)
1006 {
1007
1008         powerpc_lwsync();
1009 }
1010
1011 static __inline void
1012 atomic_thread_fence_acq_rel(void)
1013 {
1014
1015         powerpc_lwsync();
1016 }
1017
1018 static __inline void
1019 atomic_thread_fence_seq_cst(void)
1020 {
1021
1022         __asm __volatile("sync" : : : "memory");
1023 }
1024
1025 #ifndef ISA_206_ATOMICS
1026 #include <sys/_atomic_subword.h>
1027 #define atomic_cmpset_char      atomic_cmpset_8
1028 #define atomic_cmpset_short     atomic_cmpset_16
1029 #define atomic_fcmpset_char     atomic_fcmpset_8
1030 #define atomic_fcmpset_short    atomic_fcmpset_16
1031 #endif
1032
1033 /* These need sys/_atomic_subword.h on non-ISA-2.06-atomic platforms. */
1034 ATOMIC_CMPSET_ACQ_REL(char);
1035 ATOMIC_CMPSET_ACQ_REL(short);
1036
1037 ATOMIC_FCMPSET_ACQ_REL(char);
1038 ATOMIC_FCMPSET_ACQ_REL(short);
1039
1040 #undef __ATOMIC_REL
1041 #undef __ATOMIC_ACQ
1042
1043 #endif /* ! _MACHINE_ATOMIC_H_ */