sys/arm64/include/atomic.h

   1 /*-
   2  * Copyright (c) 2013 Andrew Turner <andrew@freebsd.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  *
  26  * $FreeBSD$
  27  */
  28
  29 #ifndef _MACHINE_ATOMIC_H_
  30 #define _MACHINE_ATOMIC_H_
  31
  32 #include <sys/atomic_common.h>
  33
  34 #define isb()           __asm __volatile("isb" : : : "memory")
  35
  36 /*
  37  * Options for DMB and DSB:
  38  *      oshld   Outer Shareable, load
  39  *      oshst   Outer Shareable, store
  40  *      osh     Outer Shareable, all
  41  *      nshld   Non-shareable, load
  42  *      nshst   Non-shareable, store
  43  *      nsh     Non-shareable, all
  44  *      ishld   Inner Shareable, load
  45  *      ishst   Inner Shareable, store
  46  *      ish     Inner Shareable, all
  47  *      ld      Full system, load
  48  *      st      Full system, store
  49  *      sy      Full system, all
  50  */
  51 #define dsb(opt)        __asm __volatile("dsb " __STRING(opt) : : : "memory")
  52 #define dmb(opt)        __asm __volatile("dmb " __STRING(opt) : : : "memory")
  53
  54 #define mb()    dmb(sy) /* Full system memory barrier all */
  55 #define wmb()   dmb(st) /* Full system memory barrier store */
  56 #define rmb()   dmb(ld) /* Full system memory barrier load */
  57
  58 #define ATOMIC_OP(op, asm_op, bar, a, l)                                \
  59 static __inline void                                                    \
  60 atomic_##op##_##bar##32(volatile uint32_t *p, uint32_t val)             \
  61 {                                                                       \
  62         uint32_t tmp;                                                   \
  63         int res;                                                        \
  64                                                                         \
  65         __asm __volatile(                                               \
  66             "1: ld"#a"xr   %w0, [%2]      \n"                           \
  67             "   "#asm_op"  %w0, %w0, %w3  \n"                           \
  68             "   st"#l"xr   %w1, %w0, [%2] \n"                           \
  69             "   cbnz       %w1, 1b        \n"                           \
  70             : "=&r"(tmp), "=&r"(res)                                    \
  71             : "r" (p), "r" (val)                                        \
  72             : "memory"                                                  \
  73         );                                                              \
  74 }                                                                       \
  75                                                                         \
  76 static __inline void                                                    \
  77 atomic_##op##_##bar##64(volatile uint64_t *p, uint64_t val)             \
  78 {                                                                       \
  79         uint64_t tmp;                                                   \
  80         int res;                                                        \
  81                                                                         \
  82         __asm __volatile(                                               \
  83             "1: ld"#a"xr   %0, [%2]      \n"                            \
  84             "   "#asm_op"  %0, %0, %3    \n"                            \
  85             "   st"#l"xr   %w1, %0, [%2] \n"                            \
  86             "   cbnz       %w1, 1b       \n"                            \
  87             : "=&r"(tmp), "=&r"(res)                                    \
  88             : "r" (p), "r" (val)                                        \
  89             : "memory"                                                  \
  90         );                                                              \
  91 }
  92
  93 #define ATOMIC(op, asm_op)                                              \
  94     ATOMIC_OP(op, asm_op,     ,  ,  )                                   \
  95     ATOMIC_OP(op, asm_op, acq_, a,  )                                   \
  96     ATOMIC_OP(op, asm_op, rel_,  , l)                                   \
  97
  98 ATOMIC(add,      add)
  99 ATOMIC(clear,    bic)
 100 ATOMIC(set,      orr)
 101 ATOMIC(subtract, sub)
 102
 103 #define ATOMIC_FCMPSET(bar, a, l)                                       \
 104 static __inline int                                                     \
 105 atomic_fcmpset_##bar##32(volatile uint32_t *p, uint32_t *cmpval,        \
 106     uint32_t newval)                                                    \
 107 {                                                                       \
 108         uint32_t tmp;                                                   \
 109         uint32_t _cmpval = *cmpval;                                     \
 110         int res;                                                        \
 111                                                                         \
 112         __asm __volatile(                                               \
 113             "1: mov      %w1, #1        \n"                             \
 114             "   ld"#a"xr %w0, [%2]      \n"                             \
 115             "   cmp      %w0, %w3       \n"                             \
 116             "   b.ne     2f             \n"                             \
 117             "   st"#l"xr %w1, %w4, [%2] \n"                             \
 118             "2:"                                                        \
 119             : "=&r"(tmp), "=&r"(res)                                    \
 120             : "r" (p), "r" (_cmpval), "r" (newval)                      \
 121             : "cc", "memory"                                            \
 122         );                                                              \
 123         *cmpval = tmp;                                                  \
 124                                                                         \
 125         return (!res);                                                  \
 126 }                                                                       \
 127                                                                         \
 128 static __inline int                                                     \
 129 atomic_fcmpset_##bar##64(volatile uint64_t *p, uint64_t *cmpval,        \
 130     uint64_t newval)                                                    \
 131 {                                                                       \
 132         uint64_t tmp;                                                   \
 133         uint64_t _cmpval = *cmpval;                                     \
 134         int res;                                                        \
 135                                                                         \
 136         __asm __volatile(                                               \
 137             "1: mov      %w1, #1       \n"                              \
 138             "   ld"#a"xr %0, [%2]      \n"                              \
 139             "   cmp      %0, %3        \n"                              \
 140             "   b.ne     2f            \n"                              \
 141             "   st"#l"xr %w1, %4, [%2] \n"                              \
 142             "2:"                                                        \
 143             : "=&r"(tmp), "=&r"(res)                                    \
 144             : "r" (p), "r" (_cmpval), "r" (newval)                      \
 145             : "cc", "memory"                                            \
 146         );                                                              \
 147         *cmpval = tmp;                                                  \
 148                                                                         \
 149         return (!res);                                                  \
 150 }
 151
 152 ATOMIC_FCMPSET(    ,  , )
 153 ATOMIC_FCMPSET(acq_, a, )
 154 ATOMIC_FCMPSET(rel_,  ,l)
 155
 156 #undef ATOMIC_FCMPSET
 157
 158 #define ATOMIC_CMPSET(bar, a, l)                                        \
 159 static __inline int                                                     \
 160 atomic_cmpset_##bar##32(volatile uint32_t *p, uint32_t cmpval,          \
 161     uint32_t newval)                                                    \
 162 {                                                                       \
 163         uint32_t tmp;                                                   \
 164         int res;                                                        \
 165                                                                         \
 166         __asm __volatile(                                               \
 167             "1: mov      %w1, #1        \n"                             \
 168             "   ld"#a"xr %w0, [%2]      \n"                             \
 169             "   cmp      %w0, %w3       \n"                             \
 170             "   b.ne     2f             \n"                             \
 171             "   st"#l"xr %w1, %w4, [%2] \n"                             \
 172             "   cbnz     %w1, 1b        \n"                             \
 173             "2:"                                                        \
 174             : "=&r"(tmp), "=&r"(res)                                    \
 175             : "r" (p), "r" (cmpval), "r" (newval)                       \
 176             : "cc", "memory"                                                    \
 177         );                                                              \
 178                                                                         \
 179         return (!res);                                                  \
 180 }                                                                       \
 181                                                                         \
 182 static __inline int                                                     \
 183 atomic_cmpset_##bar##64(volatile uint64_t *p, uint64_t cmpval,          \
 184     uint64_t newval)                                                    \
 185 {                                                                       \
 186         uint64_t tmp;                                                   \
 187         int res;                                                        \
 188                                                                         \
 189         __asm __volatile(                                               \
 190             "1: mov      %w1, #1       \n"                              \
 191             "   ld"#a"xr %0, [%2]      \n"                              \
 192             "   cmp      %0, %3        \n"                              \
 193             "   b.ne     2f            \n"                              \
 194             "   st"#l"xr %w1, %4, [%2] \n"                              \
 195             "   cbnz     %w1, 1b       \n"                              \
 196             "2:"                                                        \
 197             : "=&r"(tmp), "=&r"(res)                                    \
 198             : "r" (p), "r" (cmpval), "r" (newval)                       \
 199             : "cc", "memory"                                                    \
 200         );                                                              \
 201                                                                         \
 202         return (!res);                                                  \
 203 }
 204
 205 ATOMIC_CMPSET(    ,  , )
 206 ATOMIC_CMPSET(acq_, a, )
 207 ATOMIC_CMPSET(rel_,  ,l)
 208
 209 static __inline uint32_t
 210 atomic_fetchadd_32(volatile uint32_t *p, uint32_t val)
 211 {
 212         uint32_t tmp, ret;
 213         int res;
 214
 215         __asm __volatile(
 216             "1: ldxr    %w2, [%3]      \n"
 217             "   add     %w0, %w2, %w4  \n"
 218             "   stxr    %w1, %w0, [%3] \n"
 219             "   cbnz    %w1, 1b        \n"
 220             : "=&r"(tmp), "=&r"(res), "=&r"(ret)
 221             : "r" (p), "r" (val)
 222             : "memory"
 223         );
 224
 225         return (ret);
 226 }
 227
 228 static __inline uint64_t
 229 atomic_fetchadd_64(volatile uint64_t *p, uint64_t val)
 230 {
 231         uint64_t tmp, ret;
 232         int res;
 233
 234         __asm __volatile(
 235             "1: ldxr    %2, [%3]      \n"
 236             "   add     %0, %2, %4    \n"
 237             "   stxr    %w1, %0, [%3] \n"
 238             "   cbnz    %w1, 1b       \n"
 239             : "=&r"(tmp), "=&r"(res), "=&r"(ret)
 240             : "r" (p), "r" (val)
 241             : "memory"
 242         );
 243
 244         return (ret);
 245 }
 246
 247 static __inline uint32_t
 248 atomic_readandclear_32(volatile uint32_t *p)
 249 {
 250         uint32_t ret;
 251         int res;
 252
 253         __asm __volatile(
 254             "1: ldxr    %w1, [%2]      \n"
 255             "   stxr    %w0, wzr, [%2] \n"
 256             "   cbnz    %w0, 1b        \n"
 257             : "=&r"(res), "=&r"(ret)
 258             : "r" (p)
 259             : "memory"
 260         );
 261
 262         return (ret);
 263 }
 264
 265 static __inline uint64_t
 266 atomic_readandclear_64(volatile uint64_t *p)
 267 {
 268         uint64_t ret;
 269         int res;
 270
 271         __asm __volatile(
 272             "1: ldxr    %1, [%2]      \n"
 273             "   stxr    %w0, xzr, [%2] \n"
 274             "   cbnz    %w0, 1b        \n"
 275             : "=&r"(res), "=&r"(ret)
 276             : "r" (p)
 277             : "memory"
 278         );
 279
 280         return (ret);
 281 }
 282
 283 static __inline uint32_t
 284 atomic_swap_32(volatile uint32_t *p, uint32_t val)
 285 {
 286         uint32_t ret;
 287         int res;
 288
 289         __asm __volatile(
 290             "1: ldxr    %w0, [%2]      \n"
 291             "   stxr    %w1, %w3, [%2] \n"
 292             "   cbnz    %w1, 1b        \n"
 293             : "=&r"(ret), "=&r"(res)
 294             : "r" (p), "r" (val)
 295             : "memory"
 296         );
 297
 298         return (ret);
 299 }
 300
 301 static __inline uint64_t
 302 atomic_swap_64(volatile uint64_t *p, uint64_t val)
 303 {
 304         uint64_t ret;
 305         int res;
 306
 307         __asm __volatile(
 308             "1: ldxr    %0, [%2]      \n"
 309             "   stxr    %w1, %3, [%2] \n"
 310             "   cbnz    %w1, 1b       \n"
 311             : "=&r"(ret), "=&r"(res)
 312             : "r" (p), "r" (val)
 313             : "memory"
 314         );
 315
 316         return (ret);
 317 }
 318
 319 static __inline uint32_t
 320 atomic_load_acq_32(volatile uint32_t *p)
 321 {
 322         uint32_t ret;
 323
 324         __asm __volatile(
 325             "ldar       %w0, [%1] \n"
 326             : "=&r" (ret)
 327             : "r" (p)
 328             : "memory");
 329
 330         return (ret);
 331 }
 332
 333 static __inline uint64_t
 334 atomic_load_acq_64(volatile uint64_t *p)
 335 {
 336         uint64_t ret;
 337
 338         __asm __volatile(
 339             "ldar       %0, [%1] \n"
 340             : "=&r" (ret)
 341             : "r" (p)
 342             : "memory");
 343
 344         return (ret);
 345 }
 346
 347 static __inline void
 348 atomic_store_rel_32(volatile uint32_t *p, uint32_t val)
 349 {
 350
 351         __asm __volatile(
 352             "stlr       %w0, [%1] \n"
 353             :
 354             : "r" (val), "r" (p)
 355             : "memory");
 356 }
 357
 358 static __inline void
 359 atomic_store_rel_64(volatile uint64_t *p, uint64_t val)
 360 {
 361
 362         __asm __volatile(
 363             "stlr       %0, [%1] \n"
 364             :
 365             : "r" (val), "r" (p)
 366             : "memory");
 367 }
 368
 369
 370 #define atomic_add_int                  atomic_add_32
 371 #define atomic_fcmpset_int              atomic_fcmpset_32
 372 #define atomic_clear_int                atomic_clear_32
 373 #define atomic_cmpset_int               atomic_cmpset_32
 374 #define atomic_fetchadd_int             atomic_fetchadd_32
 375 #define atomic_readandclear_int         atomic_readandclear_32
 376 #define atomic_set_int                  atomic_set_32
 377 #define atomic_swap_int                 atomic_swap_32
 378 #define atomic_subtract_int             atomic_subtract_32
 379
 380 #define atomic_add_acq_int              atomic_add_acq_32
 381 #define atomic_fcmpset_acq_int          atomic_fcmpset_acq_32
 382 #define atomic_clear_acq_int            atomic_clear_acq_32
 383 #define atomic_cmpset_acq_int           atomic_cmpset_acq_32
 384 #define atomic_load_acq_int             atomic_load_acq_32
 385 #define atomic_set_acq_int              atomic_set_acq_32
 386 #define atomic_subtract_acq_int         atomic_subtract_acq_32
 387
 388 #define atomic_add_rel_int              atomic_add_rel_32
 389 #define atomic_fcmpset_rel_int          atomic_fcmpset_rel_32
 390 #define atomic_clear_rel_int            atomic_clear_rel_32
 391 #define atomic_cmpset_rel_int           atomic_cmpset_rel_32
 392 #define atomic_set_rel_int              atomic_set_rel_32
 393 #define atomic_subtract_rel_int         atomic_subtract_rel_32
 394 #define atomic_store_rel_int            atomic_store_rel_32
 395
 396 #define atomic_add_long                 atomic_add_64
 397 #define atomic_fcmpset_long             atomic_fcmpset_64
 398 #define atomic_clear_long               atomic_clear_64
 399 #define atomic_cmpset_long              atomic_cmpset_64
 400 #define atomic_fetchadd_long            atomic_fetchadd_64
 401 #define atomic_readandclear_long        atomic_readandclear_64
 402 #define atomic_set_long                 atomic_set_64
 403 #define atomic_swap_long                atomic_swap_64
 404 #define atomic_subtract_long            atomic_subtract_64
 405
 406 #define atomic_add_ptr                  atomic_add_64
 407 #define atomic_fcmpset_ptr              atomic_fcmpset_64
 408 #define atomic_clear_ptr                atomic_clear_64
 409 #define atomic_cmpset_ptr               atomic_cmpset_64
 410 #define atomic_fetchadd_ptr             atomic_fetchadd_64
 411 #define atomic_readandclear_ptr         atomic_readandclear_64
 412 #define atomic_set_ptr                  atomic_set_64
 413 #define atomic_swap_ptr                 atomic_swap_64
 414 #define atomic_subtract_ptr             atomic_subtract_64
 415
 416 #define atomic_add_acq_long             atomic_add_acq_64
 417 #define atomic_fcmpset_acq_long         atomic_fcmpset_acq_64
 418 #define atomic_clear_acq_long           atomic_clear_acq_64
 419 #define atomic_cmpset_acq_long          atomic_cmpset_acq_64
 420 #define atomic_load_acq_long            atomic_load_acq_64
 421 #define atomic_set_acq_long             atomic_set_acq_64
 422 #define atomic_subtract_acq_long        atomic_subtract_acq_64
 423
 424 #define atomic_add_acq_ptr              atomic_add_acq_64
 425 #define atomic_fcmpset_acq_ptr          atomic_fcmpset_acq_64
 426 #define atomic_clear_acq_ptr            atomic_clear_acq_64
 427 #define atomic_cmpset_acq_ptr           atomic_cmpset_acq_64
 428 #define atomic_load_acq_ptr             atomic_load_acq_64
 429 #define atomic_set_acq_ptr              atomic_set_acq_64
 430 #define atomic_subtract_acq_ptr         atomic_subtract_acq_64
 431
 432 #define atomic_add_rel_long             atomic_add_rel_64
 433 #define atomic_fcmpset_rel_long         atomic_fcmpset_rel_64
 434 #define atomic_clear_rel_long           atomic_clear_rel_64
 435 #define atomic_cmpset_rel_long          atomic_cmpset_rel_64
 436 #define atomic_set_rel_long             atomic_set_rel_64
 437 #define atomic_subtract_rel_long        atomic_subtract_rel_64
 438 #define atomic_store_rel_long           atomic_store_rel_64
 439
 440 #define atomic_add_rel_ptr              atomic_add_rel_64
 441 #define atomic_fcmpset_rel_ptr          atomic_fcmpset_rel_64
 442 #define atomic_clear_rel_ptr            atomic_clear_rel_64
 443 #define atomic_cmpset_rel_ptr           atomic_cmpset_rel_64
 444 #define atomic_set_rel_ptr              atomic_set_rel_64
 445 #define atomic_subtract_rel_ptr         atomic_subtract_rel_64
 446 #define atomic_store_rel_ptr            atomic_store_rel_64
 447
 448 static __inline void
 449 atomic_thread_fence_acq(void)
 450 {
 451
 452         dmb(ld);
 453 }
 454
 455 static __inline void
 456 atomic_thread_fence_rel(void)
 457 {
 458
 459         dmb(sy);
 460 }
 461
 462 static __inline void
 463 atomic_thread_fence_acq_rel(void)
 464 {
 465
 466         dmb(sy);
 467 }
 468
 469 static __inline void
 470 atomic_thread_fence_seq_cst(void)
 471 {
 472
 473         dmb(sy);
 474 }
 475
 476 #endif /* _MACHINE_ATOMIC_H_ */
 477