sys/arm64/include/atomic.h

   1 /*-
   2  * Copyright (c) 2013 Andrew Turner <andrew@freebsd.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  *
  26  * $FreeBSD$
  27  */
  28
  29 #ifndef _MACHINE_ATOMIC_H_
  30 #define _MACHINE_ATOMIC_H_
  31
  32 #include <sys/atomic_common.h>
  33
  34 #define isb()           __asm __volatile("isb" : : : "memory")
  35
  36 /*
  37  * Options for DMB and DSB:
  38  *      oshld   Outer Shareable, load
  39  *      oshst   Outer Shareable, store
  40  *      osh     Outer Shareable, all
  41  *      nshld   Non-shareable, load
  42  *      nshst   Non-shareable, store
  43  *      nsh     Non-shareable, all
  44  *      ishld   Inner Shareable, load
  45  *      ishst   Inner Shareable, store
  46  *      ish     Inner Shareable, all
  47  *      ld      Full system, load
  48  *      st      Full system, store
  49  *      sy      Full system, all
  50  */
  51 #define dsb(opt)        __asm __volatile("dsb " __STRING(opt) : : : "memory")
  52 #define dmb(opt)        __asm __volatile("dmb " __STRING(opt) : : : "memory")
  53
  54 #define mb()    dmb(sy) /* Full system memory barrier all */
  55 #define wmb()   dmb(st) /* Full system memory barrier store */
  56 #define rmb()   dmb(ld) /* Full system memory barrier load */
  57
  58 #define ATOMIC_OP(op, asm_op, bar, a, l)                                \
  59 static __inline void                                                    \
  60 atomic_##op##_##bar##32(volatile uint32_t *p, uint32_t val)             \
  61 {                                                                       \
  62         uint32_t tmp;                                                   \
  63         int res;                                                        \
  64                                                                         \
  65         __asm __volatile(                                               \
  66             "1: ld"#a"xr   %w0, [%2]      \n"                           \
  67             "   "#asm_op"  %w0, %w0, %w3  \n"                           \
  68             "   st"#l"xr   %w1, %w0, [%2] \n"                           \
  69             "   cbnz       %w1, 1b        \n"                           \
  70             : "=&r"(tmp), "=&r"(res)                                    \
  71             : "r" (p), "r" (val)                                        \
  72             : "memory"                                                  \
  73         );                                                              \
  74 }                                                                       \
  75                                                                         \
  76 static __inline void                                                    \
  77 atomic_##op##_##bar##64(volatile uint64_t *p, uint64_t val)             \
  78 {                                                                       \
  79         uint64_t tmp;                                                   \
  80         int res;                                                        \
  81                                                                         \
  82         __asm __volatile(                                               \
  83             "1: ld"#a"xr   %0, [%2]      \n"                            \
  84             "   "#asm_op"  %0, %0, %3    \n"                            \
  85             "   st"#l"xr   %w1, %0, [%2] \n"                            \
  86             "   cbnz       %w1, 1b       \n"                            \
  87             : "=&r"(tmp), "=&r"(res)                                    \
  88             : "r" (p), "r" (val)                                        \
  89             : "memory"                                                  \
  90         );                                                              \
  91 }
  92
  93 #define ATOMIC(op, asm_op)                                              \
  94     ATOMIC_OP(op, asm_op,     ,  ,  )                                   \
  95     ATOMIC_OP(op, asm_op, acq_, a,  )                                   \
  96     ATOMIC_OP(op, asm_op, rel_,  , l)                                   \
  97
  98 ATOMIC(add,      add)
  99 ATOMIC(clear,    bic)
 100 ATOMIC(set,      orr)
 101 ATOMIC(subtract, sub)
 102
 103 #define ATOMIC_FCMPSET(bar, a, l)                                       \
 104 static __inline int                                                     \
 105 atomic_fcmpset_##bar##8(volatile uint8_t *p, uint8_t *cmpval,           \
 106     uint8_t newval)                                                     \
 107 {                                                                       \
 108         uint8_t tmp;                                                    \
 109         uint8_t _cmpval = *cmpval;                                      \
 110         int res;                                                        \
 111                                                                         \
 112         __asm __volatile(                                               \
 113             "1: mov      %w1, #1        \n"                             \
 114             "   ld"#a"xrb %w0, [%2]     \n"                             \
 115             "   cmp      %w0, %w3       \n"                             \
 116             "   b.ne     2f             \n"                             \
 117             "   st"#l"xrb %w1, %w4, [%2]\n"                             \
 118             "2:"                                                        \
 119             : "=&r"(tmp), "=&r"(res)                                    \
 120             : "r" (p), "r" (_cmpval), "r" (newval)                      \
 121             : "cc", "memory"                                            \
 122         );                                                              \
 123         *cmpval = tmp;                                                  \
 124                                                                         \
 125         return (!res);                                                  \
 126 }                                                                       \
 127                                                                         \
 128 static __inline int                                                     \
 129 atomic_fcmpset_##bar##16(volatile uint16_t *p, uint16_t *cmpval,        \
 130     uint16_t newval)                                                    \
 131 {                                                                       \
 132         uint16_t tmp;                                                   \
 133         uint16_t _cmpval = *cmpval;                                     \
 134         int res;                                                        \
 135                                                                         \
 136         __asm __volatile(                                               \
 137             "1: mov      %w1, #1        \n"                             \
 138             "   ld"#a"xh %w0, [%2]      \n"                             \
 139             "   cmp      %w0, %w3       \n"                             \
 140             "   b.ne     2f             \n"                             \
 141             "   st"#l"xh %w1, %w4, [%2] \n"                             \
 142             "2:"                                                        \
 143             : "=&r"(tmp), "=&r"(res)                                    \
 144             : "r" (p), "r" (_cmpval), "r" (newval)                      \
 145             : "cc", "memory"                                            \
 146         );                                                              \
 147         *cmpval = tmp;                                                  \
 148                                                                         \
 149         return (!res);                                                  \
 150 }                                                                       \
 151                                                                         \
 152 static __inline int                                                     \
 153 atomic_fcmpset_##bar##32(volatile uint32_t *p, uint32_t *cmpval,        \
 154     uint32_t newval)                                                    \
 155 {                                                                       \
 156         uint32_t tmp;                                                   \
 157         uint32_t _cmpval = *cmpval;                                     \
 158         int res;                                                        \
 159                                                                         \
 160         __asm __volatile(                                               \
 161             "1: mov      %w1, #1        \n"                             \
 162             "   ld"#a"xr %w0, [%2]      \n"                             \
 163             "   cmp      %w0, %w3       \n"                             \
 164             "   b.ne     2f             \n"                             \
 165             "   st"#l"xr %w1, %w4, [%2] \n"                             \
 166             "2:"                                                        \
 167             : "=&r"(tmp), "=&r"(res)                                    \
 168             : "r" (p), "r" (_cmpval), "r" (newval)                      \
 169             : "cc", "memory"                                            \
 170         );                                                              \
 171         *cmpval = tmp;                                                  \
 172                                                                         \
 173         return (!res);                                                  \
 174 }                                                                       \
 175                                                                         \
 176 static __inline int                                                     \
 177 atomic_fcmpset_##bar##64(volatile uint64_t *p, uint64_t *cmpval,        \
 178     uint64_t newval)                                                    \
 179 {                                                                       \
 180         uint64_t tmp;                                                   \
 181         uint64_t _cmpval = *cmpval;                                     \
 182         int res;                                                        \
 183                                                                         \
 184         __asm __volatile(                                               \
 185             "1: mov      %w1, #1       \n"                              \
 186             "   ld"#a"xr %0, [%2]      \n"                              \
 187             "   cmp      %0, %3        \n"                              \
 188             "   b.ne     2f            \n"                              \
 189             "   st"#l"xr %w1, %4, [%2] \n"                              \
 190             "2:"                                                        \
 191             : "=&r"(tmp), "=&r"(res)                                    \
 192             : "r" (p), "r" (_cmpval), "r" (newval)                      \
 193             : "cc", "memory"                                            \
 194         );                                                              \
 195         *cmpval = tmp;                                                  \
 196                                                                         \
 197         return (!res);                                                  \
 198 }
 199
 200 ATOMIC_FCMPSET(    ,  , )
 201 ATOMIC_FCMPSET(acq_, a, )
 202 ATOMIC_FCMPSET(rel_,  ,l)
 203
 204 #undef ATOMIC_FCMPSET
 205
 206 #define ATOMIC_CMPSET(bar, a, l)                                        \
 207 static __inline int                                                     \
 208 atomic_cmpset_##bar##32(volatile uint32_t *p, uint32_t cmpval,          \
 209     uint32_t newval)                                                    \
 210 {                                                                       \
 211         uint32_t tmp;                                                   \
 212         int res;                                                        \
 213                                                                         \
 214         __asm __volatile(                                               \
 215             "1: mov      %w1, #1        \n"                             \
 216             "   ld"#a"xr %w0, [%2]      \n"                             \
 217             "   cmp      %w0, %w3       \n"                             \
 218             "   b.ne     2f             \n"                             \
 219             "   st"#l"xr %w1, %w4, [%2] \n"                             \
 220             "   cbnz     %w1, 1b        \n"                             \
 221             "2:"                                                        \
 222             : "=&r"(tmp), "=&r"(res)                                    \
 223             : "r" (p), "r" (cmpval), "r" (newval)                       \
 224             : "cc", "memory"                                                    \
 225         );                                                              \
 226                                                                         \
 227         return (!res);                                                  \
 228 }                                                                       \
 229                                                                         \
 230 static __inline int                                                     \
 231 atomic_cmpset_##bar##64(volatile uint64_t *p, uint64_t cmpval,          \
 232     uint64_t newval)                                                    \
 233 {                                                                       \
 234         uint64_t tmp;                                                   \
 235         int res;                                                        \
 236                                                                         \
 237         __asm __volatile(                                               \
 238             "1: mov      %w1, #1       \n"                              \
 239             "   ld"#a"xr %0, [%2]      \n"                              \
 240             "   cmp      %0, %3        \n"                              \
 241             "   b.ne     2f            \n"                              \
 242             "   st"#l"xr %w1, %4, [%2] \n"                              \
 243             "   cbnz     %w1, 1b       \n"                              \
 244             "2:"                                                        \
 245             : "=&r"(tmp), "=&r"(res)                                    \
 246             : "r" (p), "r" (cmpval), "r" (newval)                       \
 247             : "cc", "memory"                                                    \
 248         );                                                              \
 249                                                                         \
 250         return (!res);                                                  \
 251 }
 252
 253 ATOMIC_CMPSET(    ,  , )
 254 ATOMIC_CMPSET(acq_, a, )
 255 ATOMIC_CMPSET(rel_,  ,l)
 256
 257 static __inline uint32_t
 258 atomic_fetchadd_32(volatile uint32_t *p, uint32_t val)
 259 {
 260         uint32_t tmp, ret;
 261         int res;
 262
 263         __asm __volatile(
 264             "1: ldxr    %w2, [%3]      \n"
 265             "   add     %w0, %w2, %w4  \n"
 266             "   stxr    %w1, %w0, [%3] \n"
 267             "   cbnz    %w1, 1b        \n"
 268             : "=&r"(tmp), "=&r"(res), "=&r"(ret)
 269             : "r" (p), "r" (val)
 270             : "memory"
 271         );
 272
 273         return (ret);
 274 }
 275
 276 static __inline uint64_t
 277 atomic_fetchadd_64(volatile uint64_t *p, uint64_t val)
 278 {
 279         uint64_t tmp, ret;
 280         int res;
 281
 282         __asm __volatile(
 283             "1: ldxr    %2, [%3]      \n"
 284             "   add     %0, %2, %4    \n"
 285             "   stxr    %w1, %0, [%3] \n"
 286             "   cbnz    %w1, 1b       \n"
 287             : "=&r"(tmp), "=&r"(res), "=&r"(ret)
 288             : "r" (p), "r" (val)
 289             : "memory"
 290         );
 291
 292         return (ret);
 293 }
 294
 295 static __inline uint32_t
 296 atomic_readandclear_32(volatile uint32_t *p)
 297 {
 298         uint32_t ret;
 299         int res;
 300
 301         __asm __volatile(
 302             "1: ldxr    %w1, [%2]      \n"
 303             "   stxr    %w0, wzr, [%2] \n"
 304             "   cbnz    %w0, 1b        \n"
 305             : "=&r"(res), "=&r"(ret)
 306             : "r" (p)
 307             : "memory"
 308         );
 309
 310         return (ret);
 311 }
 312
 313 static __inline uint64_t
 314 atomic_readandclear_64(volatile uint64_t *p)
 315 {
 316         uint64_t ret;
 317         int res;
 318
 319         __asm __volatile(
 320             "1: ldxr    %1, [%2]      \n"
 321             "   stxr    %w0, xzr, [%2] \n"
 322             "   cbnz    %w0, 1b        \n"
 323             : "=&r"(res), "=&r"(ret)
 324             : "r" (p)
 325             : "memory"
 326         );
 327
 328         return (ret);
 329 }
 330
 331 static __inline uint32_t
 332 atomic_swap_32(volatile uint32_t *p, uint32_t val)
 333 {
 334         uint32_t ret;
 335         int res;
 336
 337         __asm __volatile(
 338             "1: ldxr    %w0, [%2]      \n"
 339             "   stxr    %w1, %w3, [%2] \n"
 340             "   cbnz    %w1, 1b        \n"
 341             : "=&r"(ret), "=&r"(res)
 342             : "r" (p), "r" (val)
 343             : "memory"
 344         );
 345
 346         return (ret);
 347 }
 348
 349 static __inline uint64_t
 350 atomic_swap_64(volatile uint64_t *p, uint64_t val)
 351 {
 352         uint64_t ret;
 353         int res;
 354
 355         __asm __volatile(
 356             "1: ldxr    %0, [%2]      \n"
 357             "   stxr    %w1, %3, [%2] \n"
 358             "   cbnz    %w1, 1b       \n"
 359             : "=&r"(ret), "=&r"(res)
 360             : "r" (p), "r" (val)
 361             : "memory"
 362         );
 363
 364         return (ret);
 365 }
 366
 367 static __inline uint32_t
 368 atomic_load_acq_32(volatile uint32_t *p)
 369 {
 370         uint32_t ret;
 371
 372         __asm __volatile(
 373             "ldar       %w0, [%1] \n"
 374             : "=&r" (ret)
 375             : "r" (p)
 376             : "memory");
 377
 378         return (ret);
 379 }
 380
 381 static __inline uint64_t
 382 atomic_load_acq_64(volatile uint64_t *p)
 383 {
 384         uint64_t ret;
 385
 386         __asm __volatile(
 387             "ldar       %0, [%1] \n"
 388             : "=&r" (ret)
 389             : "r" (p)
 390             : "memory");
 391
 392         return (ret);
 393 }
 394
 395 static __inline void
 396 atomic_store_rel_32(volatile uint32_t *p, uint32_t val)
 397 {
 398
 399         __asm __volatile(
 400             "stlr       %w0, [%1] \n"
 401             :
 402             : "r" (val), "r" (p)
 403             : "memory");
 404 }
 405
 406 static __inline void
 407 atomic_store_rel_64(volatile uint64_t *p, uint64_t val)
 408 {
 409
 410         __asm __volatile(
 411             "stlr       %0, [%1] \n"
 412             :
 413             : "r" (val), "r" (p)
 414             : "memory");
 415 }
 416
 417
 418 #define atomic_add_int                  atomic_add_32
 419 #define atomic_fcmpset_int              atomic_fcmpset_32
 420 #define atomic_clear_int                atomic_clear_32
 421 #define atomic_cmpset_int               atomic_cmpset_32
 422 #define atomic_fetchadd_int             atomic_fetchadd_32
 423 #define atomic_readandclear_int         atomic_readandclear_32
 424 #define atomic_set_int                  atomic_set_32
 425 #define atomic_swap_int                 atomic_swap_32
 426 #define atomic_subtract_int             atomic_subtract_32
 427
 428 #define atomic_add_acq_int              atomic_add_acq_32
 429 #define atomic_fcmpset_acq_int          atomic_fcmpset_acq_32
 430 #define atomic_clear_acq_int            atomic_clear_acq_32
 431 #define atomic_cmpset_acq_int           atomic_cmpset_acq_32
 432 #define atomic_load_acq_int             atomic_load_acq_32
 433 #define atomic_set_acq_int              atomic_set_acq_32
 434 #define atomic_subtract_acq_int         atomic_subtract_acq_32
 435
 436 #define atomic_add_rel_int              atomic_add_rel_32
 437 #define atomic_fcmpset_rel_int          atomic_fcmpset_rel_32
 438 #define atomic_clear_rel_int            atomic_clear_rel_32
 439 #define atomic_cmpset_rel_int           atomic_cmpset_rel_32
 440 #define atomic_set_rel_int              atomic_set_rel_32
 441 #define atomic_subtract_rel_int         atomic_subtract_rel_32
 442 #define atomic_store_rel_int            atomic_store_rel_32
 443
 444 #define atomic_add_long                 atomic_add_64
 445 #define atomic_fcmpset_long             atomic_fcmpset_64
 446 #define atomic_clear_long               atomic_clear_64
 447 #define atomic_cmpset_long              atomic_cmpset_64
 448 #define atomic_fetchadd_long            atomic_fetchadd_64
 449 #define atomic_readandclear_long        atomic_readandclear_64
 450 #define atomic_set_long                 atomic_set_64
 451 #define atomic_swap_long                atomic_swap_64
 452 #define atomic_subtract_long            atomic_subtract_64
 453
 454 #define atomic_add_ptr                  atomic_add_64
 455 #define atomic_fcmpset_ptr              atomic_fcmpset_64
 456 #define atomic_clear_ptr                atomic_clear_64
 457 #define atomic_cmpset_ptr               atomic_cmpset_64
 458 #define atomic_fetchadd_ptr             atomic_fetchadd_64
 459 #define atomic_readandclear_ptr         atomic_readandclear_64
 460 #define atomic_set_ptr                  atomic_set_64
 461 #define atomic_swap_ptr                 atomic_swap_64
 462 #define atomic_subtract_ptr             atomic_subtract_64
 463
 464 #define atomic_add_acq_long             atomic_add_acq_64
 465 #define atomic_fcmpset_acq_long         atomic_fcmpset_acq_64
 466 #define atomic_clear_acq_long           atomic_clear_acq_64
 467 #define atomic_cmpset_acq_long          atomic_cmpset_acq_64
 468 #define atomic_load_acq_long            atomic_load_acq_64
 469 #define atomic_set_acq_long             atomic_set_acq_64
 470 #define atomic_subtract_acq_long        atomic_subtract_acq_64
 471
 472 #define atomic_add_acq_ptr              atomic_add_acq_64
 473 #define atomic_fcmpset_acq_ptr          atomic_fcmpset_acq_64
 474 #define atomic_clear_acq_ptr            atomic_clear_acq_64
 475 #define atomic_cmpset_acq_ptr           atomic_cmpset_acq_64
 476 #define atomic_load_acq_ptr             atomic_load_acq_64
 477 #define atomic_set_acq_ptr              atomic_set_acq_64
 478 #define atomic_subtract_acq_ptr         atomic_subtract_acq_64
 479
 480 #define atomic_add_rel_long             atomic_add_rel_64
 481 #define atomic_fcmpset_rel_long         atomic_fcmpset_rel_64
 482 #define atomic_clear_rel_long           atomic_clear_rel_64
 483 #define atomic_cmpset_rel_long          atomic_cmpset_rel_64
 484 #define atomic_set_rel_long             atomic_set_rel_64
 485 #define atomic_subtract_rel_long        atomic_subtract_rel_64
 486 #define atomic_store_rel_long           atomic_store_rel_64
 487
 488 #define atomic_add_rel_ptr              atomic_add_rel_64
 489 #define atomic_fcmpset_rel_ptr          atomic_fcmpset_rel_64
 490 #define atomic_clear_rel_ptr            atomic_clear_rel_64
 491 #define atomic_cmpset_rel_ptr           atomic_cmpset_rel_64
 492 #define atomic_set_rel_ptr              atomic_set_rel_64
 493 #define atomic_subtract_rel_ptr         atomic_subtract_rel_64
 494 #define atomic_store_rel_ptr            atomic_store_rel_64
 495
 496 static __inline void
 497 atomic_thread_fence_acq(void)
 498 {
 499
 500         dmb(ld);
 501 }
 502
 503 static __inline void
 504 atomic_thread_fence_rel(void)
 505 {
 506
 507         dmb(sy);
 508 }
 509
 510 static __inline void
 511 atomic_thread_fence_acq_rel(void)
 512 {
 513
 514         dmb(sy);
 515 }
 516
 517 static __inline void
 518 atomic_thread_fence_seq_cst(void)
 519 {
 520
 521         dmb(sy);
 522 }
 523
 524 #endif /* _MACHINE_ATOMIC_H_ */
 525