sys/arm64/include/atomic.h

   1 /*-
   2  * Copyright (c) 2013 Andrew Turner <andrew@freebsd.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  *
  26  * $FreeBSD$
  27  */
  28
  29 #ifndef _MACHINE_ATOMIC_H_
  30 #define _MACHINE_ATOMIC_H_
  31
  32 #define isb()           __asm __volatile("isb" : : : "memory")
  33
  34 /*
  35  * Options for DMB and DSB:
  36  *      oshld   Outer Shareable, load
  37  *      oshst   Outer Shareable, store
  38  *      osh     Outer Shareable, all
  39  *      nshld   Non-shareable, load
  40  *      nshst   Non-shareable, store
  41  *      nsh     Non-shareable, all
  42  *      ishld   Inner Shareable, load
  43  *      ishst   Inner Shareable, store
  44  *      ish     Inner Shareable, all
  45  *      ld      Full system, load
  46  *      st      Full system, store
  47  *      sy      Full system, all
  48  */
  49 #define dsb(opt)        __asm __volatile("dsb " __STRING(opt) : : : "memory")
  50 #define dmb(opt)        __asm __volatile("dmb " __STRING(opt) : : : "memory")
  51
  52 #define mb()    dmb(sy) /* Full system memory barrier all */
  53 #define wmb()   dmb(st) /* Full system memory barrier store */
  54 #define rmb()   dmb(ld) /* Full system memory barrier load */
  55
  56 #if defined(KCSAN) && !defined(KCSAN_RUNTIME)
  57 #include <sys/_cscan_atomic.h>
  58 #else
  59
  60 #include <sys/atomic_common.h>
  61
  62 #ifdef _KERNEL
  63 extern bool lse_supported;
  64
  65 #ifdef LSE_ATOMICS
  66 #define _ATOMIC_LSE_SUPPORTED   1
  67 #else
  68 #define _ATOMIC_LSE_SUPPORTED   lse_supported
  69 #endif
  70 #else
  71 #define _ATOMIC_LSE_SUPPORTED   0
  72 #endif
  73
  74 #define _ATOMIC_OP_PROTO(t, op, bar, flav)                              \
  75 static __inline void                                                    \
  76 atomic_##op##_##bar##t##flav(volatile uint##t##_t *p, uint##t##_t val)
  77
  78 #define _ATOMIC_OP_IMPL(t, w, s, op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \
  79 _ATOMIC_OP_PROTO(t, op, bar, _llsc)                                     \
  80 {                                                                       \
  81         uint##t##_t tmp;                                                \
  82         int res;                                                        \
  83                                                                         \
  84         pre;                                                            \
  85         __asm __volatile(                                               \
  86             "1: ld"#a"xr"#s"    %"#w"0, [%2]\n"                         \
  87             "   "#llsc_asm_op"  %"#w"0, %"#w"0, %"#w"3\n"               \
  88             "   st"#l"xr"#s"    %w1, %"#w"0, [%2]\n"                    \
  89             "   cbnz            %w1, 1b\n"                              \
  90             : "=&r"(tmp), "=&r"(res)                                    \
  91             : "r" (p), "r" (val)                                        \
  92             : "memory"                                                  \
  93         );                                                              \
  94 }                                                                       \
  95                                                                         \
  96 _ATOMIC_OP_PROTO(t, op, bar, _lse)                                      \
  97 {                                                                       \
  98         uint##t##_t tmp;                                                \
  99                                                                         \
 100         pre;                                                            \
 101         __asm __volatile(                                               \
 102             ".arch_extension lse\n"                                     \
 103             "ld"#lse_asm_op#a#l#s"      %"#w"2, %"#w"0, [%1]\n"         \
 104             ".arch_extension nolse\n"                                   \
 105             : "=r" (tmp)                                                \
 106             : "r" (p), "r" (val)                                        \
 107             : "memory"                                                  \
 108         );                                                              \
 109 }                                                                       \
 110                                                                         \
 111 _ATOMIC_OP_PROTO(t, op, bar, )                                          \
 112 {                                                                       \
 113         if (_ATOMIC_LSE_SUPPORTED)                                      \
 114                 atomic_##op##_##bar##t##_lse(p, val);                   \
 115         else                                                            \
 116                 atomic_##op##_##bar##t##_llsc(p, val);                  \
 117 }
 118
 119 #define __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, bar, a, l)        \
 120         _ATOMIC_OP_IMPL(8,  w, b, op, llsc_asm_op, lse_asm_op, pre,     \
 121             bar, a, l)                                                  \
 122         _ATOMIC_OP_IMPL(16, w, h, op, llsc_asm_op, lse_asm_op, pre,     \
 123             bar, a, l)                                                  \
 124         _ATOMIC_OP_IMPL(32, w,  , op, llsc_asm_op, lse_asm_op, pre,     \
 125             bar, a, l)                                                  \
 126         _ATOMIC_OP_IMPL(64,  ,  , op, llsc_asm_op, lse_asm_op, pre,     \
 127             bar, a, l)
 128
 129 #define _ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre)                    \
 130         __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre,     ,  ,  )       \
 131         __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, acq_, a,  )       \
 132         __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, rel_,  , l)
 133
 134 _ATOMIC_OP(add,      add, add, )
 135 _ATOMIC_OP(clear,    bic, clr, )
 136 _ATOMIC_OP(set,      orr, set, )
 137 _ATOMIC_OP(subtract, add, add, val = -val)
 138
 139 #define _ATOMIC_CMPSET_PROTO(t, bar, flav)                              \
 140 static __inline int                                                     \
 141 atomic_cmpset_##bar##t##flav(volatile uint##t##_t *p,                   \
 142     uint##t##_t cmpval, uint##t##_t newval)
 143
 144 #define _ATOMIC_FCMPSET_PROTO(t, bar, flav)                             \
 145 static __inline int                                                     \
 146 atomic_fcmpset_##bar##t##flav(volatile uint##t##_t *p,                  \
 147     uint##t##_t *cmpval, uint##t##_t newval)
 148
 149 #define _ATOMIC_CMPSET_IMPL(t, w, s, bar, a, l)                         \
 150 _ATOMIC_CMPSET_PROTO(t, bar, _llsc)                                     \
 151 {                                                                       \
 152         uint##t##_t tmp;                                                \
 153         int res;                                                        \
 154                                                                         \
 155         __asm __volatile(                                               \
 156             "1: mov             %w1, #1\n"                              \
 157             "   ld"#a"xr"#s"    %"#w"0, [%2]\n"                         \
 158             "   cmp             %"#w"0, %"#w"3\n"                       \
 159             "   b.ne            2f\n"                                   \
 160             "   st"#l"xr"#s"    %w1, %"#w"4, [%2]\n"                    \
 161             "   cbnz            %w1, 1b\n"                              \
 162             "2:"                                                        \
 163             : "=&r"(tmp), "=&r"(res)                                    \
 164             : "r" (p), "r" (cmpval), "r" (newval)                       \
 165             : "cc", "memory"                                            \
 166         );                                                              \
 167                                                                         \
 168         return (!res);                                                  \
 169 }                                                                       \
 170                                                                         \
 171 _ATOMIC_CMPSET_PROTO(t, bar, _lse)                                      \
 172 {                                                                       \
 173         uint##t##_t oldval;                                             \
 174         int res;                                                        \
 175                                                                         \
 176         oldval = cmpval;                                                \
 177         __asm __volatile(                                               \
 178             ".arch_extension lse\n"                                     \
 179             "cas"#a#l#s"        %"#w"1, %"#w"4, [%3]\n"                 \
 180             "cmp                %"#w"1, %"#w"2\n"                       \
 181             "cset               %w0, eq\n"                              \
 182             ".arch_extension nolse\n"                                   \
 183             : "=r" (res), "+&r" (cmpval)                                \
 184             : "r" (oldval), "r" (p), "r" (newval)                       \
 185             : "cc", "memory"                                            \
 186         );                                                              \
 187                                                                         \
 188         return (res);                                                   \
 189 }                                                                       \
 190                                                                         \
 191 _ATOMIC_CMPSET_PROTO(t, bar, )                                          \
 192 {                                                                       \
 193         if (_ATOMIC_LSE_SUPPORTED)                                      \
 194                 return (atomic_cmpset_##bar##t##_lse(p, cmpval,         \
 195                     newval));                                           \
 196         else                                                            \
 197                 return (atomic_cmpset_##bar##t##_llsc(p, cmpval,        \
 198                     newval));                                           \
 199 }                                                                       \
 200                                                                         \
 201 _ATOMIC_FCMPSET_PROTO(t, bar, _llsc)                                    \
 202 {                                                                       \
 203         uint##t##_t _cmpval, tmp;                                       \
 204         int res;                                                        \
 205                                                                         \
 206         _cmpval = *cmpval;                                              \
 207         __asm __volatile(                                               \
 208             "   mov             %w1, #1\n"                              \
 209             "   ld"#a"xr"#s"    %"#w"0, [%2]\n"                         \
 210             "   cmp             %"#w"0, %"#w"3\n"                       \
 211             "   b.ne            1f\n"                                   \
 212             "   st"#l"xr"#s"    %w1, %"#w"4, [%2]\n"                    \
 213             "1:"                                                        \
 214             : "=&r"(tmp), "=&r"(res)                                    \
 215             : "r" (p), "r" (_cmpval), "r" (newval)                      \
 216             : "cc", "memory"                                            \
 217         );                                                              \
 218         *cmpval = tmp;                                                  \
 219                                                                         \
 220         return (!res);                                                  \
 221 }                                                                       \
 222                                                                         \
 223 _ATOMIC_FCMPSET_PROTO(t, bar, _lse)                                     \
 224 {                                                                       \
 225         uint##t##_t _cmpval, tmp;                                       \
 226         int res;                                                        \
 227                                                                         \
 228         _cmpval = tmp = *cmpval;                                        \
 229         __asm __volatile(                                               \
 230             ".arch_extension lse\n"                                     \
 231             "cas"#a#l#s"        %"#w"1, %"#w"4, [%3]\n"                 \
 232             "cmp                %"#w"1, %"#w"2\n"                       \
 233             "cset               %w0, eq\n"                              \
 234             ".arch_extension nolse\n"                                   \
 235             : "=r" (res), "+&r" (tmp)                                   \
 236             : "r" (_cmpval), "r" (p), "r" (newval)                      \
 237             : "cc", "memory"                                            \
 238         );                                                              \
 239         *cmpval = tmp;                                                  \
 240                                                                         \
 241         return (res);                                                   \
 242 }                                                                       \
 243                                                                         \
 244 _ATOMIC_FCMPSET_PROTO(t, bar, )                                         \
 245 {                                                                       \
 246         if (_ATOMIC_LSE_SUPPORTED)                                      \
 247                 return (atomic_fcmpset_##bar##t##_lse(p, cmpval,        \
 248                     newval));                                           \
 249         else                                                            \
 250                 return (atomic_fcmpset_##bar##t##_llsc(p, cmpval,       \
 251                     newval));                                           \
 252 }
 253
 254 #define _ATOMIC_CMPSET(bar, a, l)                                       \
 255         _ATOMIC_CMPSET_IMPL(8,  w, b, bar, a, l)                        \
 256         _ATOMIC_CMPSET_IMPL(16, w, h, bar, a, l)                        \
 257         _ATOMIC_CMPSET_IMPL(32, w,  , bar, a, l)                        \
 258         _ATOMIC_CMPSET_IMPL(64,  ,  , bar, a, l)
 259
 260 #define atomic_cmpset_8         atomic_cmpset_8
 261 #define atomic_fcmpset_8        atomic_fcmpset_8
 262 #define atomic_cmpset_16        atomic_cmpset_16
 263 #define atomic_fcmpset_16       atomic_fcmpset_16
 264
 265 _ATOMIC_CMPSET(    ,  , )
 266 _ATOMIC_CMPSET(acq_, a, )
 267 _ATOMIC_CMPSET(rel_,  ,l)
 268
 269 #define _ATOMIC_FETCHADD_PROTO(t, flav)                                 \
 270 static __inline uint##t##_t                                             \
 271 atomic_fetchadd_##t##flav(volatile uint##t##_t *p, uint##t##_t val)
 272
 273 #define _ATOMIC_FETCHADD_IMPL(t, w)                                     \
 274 _ATOMIC_FETCHADD_PROTO(t, _llsc)                                        \
 275 {                                                                       \
 276         uint##t##_t ret, tmp;                                           \
 277         int res;                                                        \
 278                                                                         \
 279         __asm __volatile(                                               \
 280             "1: ldxr    %"#w"2, [%3]\n"                                 \
 281             "   add     %"#w"0, %"#w"2, %"#w"4\n"                       \
 282             "   stxr    %w1, %"#w"0, [%3]\n"                            \
 283             "   cbnz    %w1, 1b\n"                                      \
 284             : "=&r" (tmp), "=&r" (res), "=&r" (ret)                     \
 285             : "r" (p), "r" (val)                                        \
 286             : "memory"                                                  \
 287         );                                                              \
 288                                                                         \
 289         return (ret);                                                   \
 290 }                                                                       \
 291                                                                         \
 292 _ATOMIC_FETCHADD_PROTO(t, _lse)                                         \
 293 {                                                                       \
 294         uint##t##_t ret;                                                \
 295                                                                         \
 296         __asm __volatile(                                               \
 297             ".arch_extension lse\n"                                     \
 298             "ldadd      %"#w"2, %"#w"0, [%1]\n"                         \
 299             ".arch_extension nolse\n"                                   \
 300             : "=r" (ret)                                                \
 301             : "r" (p), "r" (val)                                        \
 302             : "memory"                                                  \
 303         );                                                              \
 304                                                                         \
 305         return (ret);                                                   \
 306 }                                                                       \
 307                                                                         \
 308 _ATOMIC_FETCHADD_PROTO(t, )                                             \
 309 {                                                                       \
 310         if (_ATOMIC_LSE_SUPPORTED)                                      \
 311                 return (atomic_fetchadd_##t##_lse(p, val));             \
 312         else                                                            \
 313                 return (atomic_fetchadd_##t##_llsc(p, val));            \
 314 }
 315
 316 _ATOMIC_FETCHADD_IMPL(32, w)
 317 _ATOMIC_FETCHADD_IMPL(64,  )
 318
 319 #define _ATOMIC_SWAP_PROTO(t, flav)                                     \
 320 static __inline uint##t##_t                                             \
 321 atomic_swap_##t##flav(volatile uint##t##_t *p, uint##t##_t val)
 322
 323 #define _ATOMIC_READANDCLEAR_PROTO(t, flav)                             \
 324 static __inline uint##t##_t                                             \
 325 atomic_readandclear_##t##flav(volatile uint##t##_t *p)
 326
 327 #define _ATOMIC_SWAP_IMPL(t, w, zreg)                                   \
 328 _ATOMIC_SWAP_PROTO(t, _llsc)                                            \
 329 {                                                                       \
 330         uint##t##_t ret;                                                \
 331         int res;                                                        \
 332                                                                         \
 333         __asm __volatile(                                               \
 334             "1: ldxr    %"#w"1, [%2]\n"                                 \
 335             "   stxr    %w0, %"#w"3, [%2]\n"                            \
 336             "   cbnz    %w0, 1b\n"                                      \
 337             : "=&r" (res), "=&r" (ret)                                  \
 338             : "r" (p), "r" (val)                                        \
 339             : "memory"                                                  \
 340         );                                                              \
 341                                                                         \
 342         return (ret);                                                   \
 343 }                                                                       \
 344                                                                         \
 345 _ATOMIC_SWAP_PROTO(t, _lse)                                             \
 346 {                                                                       \
 347         uint##t##_t ret;                                                \
 348                                                                         \
 349         __asm __volatile(                                               \
 350             ".arch_extension lse\n"                                     \
 351             "swp        %"#w"2, %"#w"0, [%1]\n"                         \
 352             ".arch_extension nolse\n"                                   \
 353             : "=r" (ret)                                                \
 354             : "r" (p), "r" (val)                                        \
 355             : "memory"                                                  \
 356         );                                                              \
 357                                                                         \
 358         return (ret);                                                   \
 359 }                                                                       \
 360                                                                         \
 361 _ATOMIC_SWAP_PROTO(t, )                                                 \
 362 {                                                                       \
 363         if (_ATOMIC_LSE_SUPPORTED)                                      \
 364                 return (atomic_swap_##t##_lse(p, val));                 \
 365         else                                                            \
 366                 return (atomic_swap_##t##_llsc(p, val));                \
 367 }                                                                       \
 368                                                                         \
 369 _ATOMIC_READANDCLEAR_PROTO(t, _llsc)                                    \
 370 {                                                                       \
 371         uint##t##_t ret;                                                \
 372         int res;                                                        \
 373                                                                         \
 374         __asm __volatile(                                               \
 375             "1: ldxr    %"#w"1, [%2]\n"                                 \
 376             "   stxr    %w0, "#zreg", [%2]\n"                           \
 377             "   cbnz    %w0, 1b\n"                                      \
 378             : "=&r" (res), "=&r" (ret)                                  \
 379             : "r" (p)                                                   \
 380             : "memory"                                                  \
 381         );                                                              \
 382                                                                         \
 383         return (ret);                                                   \
 384 }                                                                       \
 385                                                                         \
 386 _ATOMIC_READANDCLEAR_PROTO(t, _lse)                                     \
 387 {                                                                       \
 388         return (atomic_swap_##t##_lse(p, 0));                           \
 389 }                                                                       \
 390                                                                         \
 391 _ATOMIC_READANDCLEAR_PROTO(t, )                                         \
 392 {                                                                       \
 393         if (_ATOMIC_LSE_SUPPORTED)                                      \
 394                 return (atomic_readandclear_##t##_lse(p));              \
 395         else                                                            \
 396                 return (atomic_readandclear_##t##_llsc(p));             \
 397 }
 398
 399 _ATOMIC_SWAP_IMPL(32, w, wzr)
 400 _ATOMIC_SWAP_IMPL(64,  , xzr)
 401
 402 #define _ATOMIC_TEST_OP_PROTO(t, op, flav)                              \
 403 static __inline int                                                     \
 404 atomic_testand##op##_##t##flav(volatile uint##t##_t *p, u_int val)
 405
 406 #define _ATOMIC_TEST_OP_IMPL(t, w, op, llsc_asm_op, lse_asm_op)         \
 407 _ATOMIC_TEST_OP_PROTO(t, op, _llsc)                                     \
 408 {                                                                       \
 409         uint##t##_t mask, old, tmp;                                     \
 410         int res;                                                        \
 411                                                                         \
 412         mask = 1u << (val & 0x1f);                                      \
 413         __asm __volatile(                                               \
 414             "1: ldxr            %"#w"2, [%3]\n"                         \
 415             "  "#llsc_asm_op"   %"#w"0, %"#w"2, %"#w"4\n"               \
 416             "   stxr            %w1, %"#w"0, [%3]\n"                    \
 417             "   cbnz            %w1, 1b\n"                              \
 418             : "=&r" (tmp), "=&r" (res), "=&r" (old)                     \
 419             : "r" (p), "r" (mask)                                       \
 420             : "memory"                                                  \
 421         );                                                              \
 422                                                                         \
 423         return ((old & mask) != 0);                                     \
 424 }                                                                       \
 425                                                                         \
 426 _ATOMIC_TEST_OP_PROTO(t, op, _lse)                                      \
 427 {                                                                       \
 428         uint##t##_t mask, old;                                          \
 429                                                                         \
 430         mask = 1u << (val & 0x1f);                                      \
 431         __asm __volatile(                                               \
 432             ".arch_extension lse\n"                                     \
 433             "ld"#lse_asm_op"    %"#w"2, %"#w"0, [%1]\n"                 \
 434             ".arch_extension nolse\n"                                   \
 435             : "=r" (old)                                                \
 436             : "r" (p), "r" (mask)                                       \
 437             : "memory"                                                  \
 438         );                                                              \
 439                                                                         \
 440         return ((old & mask) != 0);                                     \
 441 }                                                                       \
 442                                                                         \
 443 _ATOMIC_TEST_OP_PROTO(t, op, )                                          \
 444 {                                                                       \
 445         if (_ATOMIC_LSE_SUPPORTED)                                      \
 446                 return (atomic_testand##op##_##t##_lse(p, val));        \
 447         else                                                            \
 448                 return (atomic_testand##op##_##t##_llsc(p, val));       \
 449 }
 450
 451 #define _ATOMIC_TEST_OP(op, llsc_asm_op, lse_asm_op)                    \
 452         _ATOMIC_TEST_OP_IMPL(32, w, op, llsc_asm_op, lse_asm_op)        \
 453         _ATOMIC_TEST_OP_IMPL(64,  , op, llsc_asm_op, lse_asm_op)
 454
 455 _ATOMIC_TEST_OP(clear, bic, clr)
 456 _ATOMIC_TEST_OP(set,   orr, set)
 457
 458 #define _ATOMIC_LOAD_ACQ_IMPL(t, w, s)                                  \
 459 static __inline uint##t##_t                                             \
 460 atomic_load_acq_##t(volatile uint##t##_t *p)                            \
 461 {                                                                       \
 462         uint##t##_t ret;                                                \
 463                                                                         \
 464         __asm __volatile(                                               \
 465             "ldar"#s"   %"#w"0, [%1]\n"                                 \
 466             : "=&r" (ret)                                               \
 467             : "r" (p)                                                   \
 468             : "memory");                                                \
 469                                                                         \
 470         return (ret);                                                   \
 471 }
 472
 473 #define atomic_load_acq_8       atomic_load_acq_8
 474 #define atomic_load_acq_16      atomic_load_acq_16
 475 _ATOMIC_LOAD_ACQ_IMPL(8,  w, b)
 476 _ATOMIC_LOAD_ACQ_IMPL(16, w, h)
 477 _ATOMIC_LOAD_ACQ_IMPL(32, w,  )
 478 _ATOMIC_LOAD_ACQ_IMPL(64,  ,  )
 479
 480 #define _ATOMIC_STORE_REL_IMPL(t, w, s)                                 \
 481 static __inline void                                                    \
 482 atomic_store_rel_##t(volatile uint##t##_t *p, uint##t##_t val)          \
 483 {                                                                       \
 484         __asm __volatile(                                               \
 485             "stlr"#s"   %"#w"0, [%1]\n"                                 \
 486             :                                                           \
 487             : "r" (val), "r" (p)                                        \
 488             : "memory");                                                \
 489 }
 490
 491 _ATOMIC_STORE_REL_IMPL(8,  w, b)
 492 _ATOMIC_STORE_REL_IMPL(16, w, h)
 493 _ATOMIC_STORE_REL_IMPL(32, w,  )
 494 _ATOMIC_STORE_REL_IMPL(64,  ,  )
 495
 496 #define atomic_add_int                  atomic_add_32
 497 #define atomic_fcmpset_int              atomic_fcmpset_32
 498 #define atomic_clear_int                atomic_clear_32
 499 #define atomic_cmpset_int               atomic_cmpset_32
 500 #define atomic_fetchadd_int             atomic_fetchadd_32
 501 #define atomic_readandclear_int         atomic_readandclear_32
 502 #define atomic_set_int                  atomic_set_32
 503 #define atomic_swap_int                 atomic_swap_32
 504 #define atomic_subtract_int             atomic_subtract_32
 505 #define atomic_testandclear_int         atomic_testandclear_32
 506 #define atomic_testandset_int           atomic_testandset_32
 507
 508 #define atomic_add_acq_int              atomic_add_acq_32
 509 #define atomic_fcmpset_acq_int          atomic_fcmpset_acq_32
 510 #define atomic_clear_acq_int            atomic_clear_acq_32
 511 #define atomic_cmpset_acq_int           atomic_cmpset_acq_32
 512 #define atomic_load_acq_int             atomic_load_acq_32
 513 #define atomic_set_acq_int              atomic_set_acq_32
 514 #define atomic_subtract_acq_int         atomic_subtract_acq_32
 515
 516 #define atomic_add_rel_int              atomic_add_rel_32
 517 #define atomic_fcmpset_rel_int          atomic_fcmpset_rel_32
 518 #define atomic_clear_rel_int            atomic_clear_rel_32
 519 #define atomic_cmpset_rel_int           atomic_cmpset_rel_32
 520 #define atomic_set_rel_int              atomic_set_rel_32
 521 #define atomic_subtract_rel_int         atomic_subtract_rel_32
 522 #define atomic_store_rel_int            atomic_store_rel_32
 523
 524 #define atomic_add_long                 atomic_add_64
 525 #define atomic_fcmpset_long             atomic_fcmpset_64
 526 #define atomic_clear_long               atomic_clear_64
 527 #define atomic_cmpset_long              atomic_cmpset_64
 528 #define atomic_fetchadd_long            atomic_fetchadd_64
 529 #define atomic_readandclear_long        atomic_readandclear_64
 530 #define atomic_set_long                 atomic_set_64
 531 #define atomic_swap_long                atomic_swap_64
 532 #define atomic_subtract_long            atomic_subtract_64
 533 #define atomic_testandclear_long        atomic_testandclear_64
 534 #define atomic_testandset_long          atomic_testandset_64
 535
 536 #define atomic_add_ptr                  atomic_add_64
 537 #define atomic_fcmpset_ptr              atomic_fcmpset_64
 538 #define atomic_clear_ptr                atomic_clear_64
 539 #define atomic_cmpset_ptr               atomic_cmpset_64
 540 #define atomic_fetchadd_ptr             atomic_fetchadd_64
 541 #define atomic_readandclear_ptr         atomic_readandclear_64
 542 #define atomic_set_ptr                  atomic_set_64
 543 #define atomic_swap_ptr                 atomic_swap_64
 544 #define atomic_subtract_ptr             atomic_subtract_64
 545
 546 #define atomic_add_acq_long             atomic_add_acq_64
 547 #define atomic_fcmpset_acq_long         atomic_fcmpset_acq_64
 548 #define atomic_clear_acq_long           atomic_clear_acq_64
 549 #define atomic_cmpset_acq_long          atomic_cmpset_acq_64
 550 #define atomic_load_acq_long            atomic_load_acq_64
 551 #define atomic_set_acq_long             atomic_set_acq_64
 552 #define atomic_subtract_acq_long        atomic_subtract_acq_64
 553
 554 #define atomic_add_acq_ptr              atomic_add_acq_64
 555 #define atomic_fcmpset_acq_ptr          atomic_fcmpset_acq_64
 556 #define atomic_clear_acq_ptr            atomic_clear_acq_64
 557 #define atomic_cmpset_acq_ptr           atomic_cmpset_acq_64
 558 #define atomic_load_acq_ptr             atomic_load_acq_64
 559 #define atomic_set_acq_ptr              atomic_set_acq_64
 560 #define atomic_subtract_acq_ptr         atomic_subtract_acq_64
 561
 562 #define atomic_add_rel_long             atomic_add_rel_64
 563 #define atomic_fcmpset_rel_long         atomic_fcmpset_rel_64
 564 #define atomic_clear_rel_long           atomic_clear_rel_64
 565 #define atomic_cmpset_rel_long          atomic_cmpset_rel_64
 566 #define atomic_set_rel_long             atomic_set_rel_64
 567 #define atomic_subtract_rel_long        atomic_subtract_rel_64
 568 #define atomic_store_rel_long           atomic_store_rel_64
 569
 570 #define atomic_add_rel_ptr              atomic_add_rel_64
 571 #define atomic_fcmpset_rel_ptr          atomic_fcmpset_rel_64
 572 #define atomic_clear_rel_ptr            atomic_clear_rel_64
 573 #define atomic_cmpset_rel_ptr           atomic_cmpset_rel_64
 574 #define atomic_set_rel_ptr              atomic_set_rel_64
 575 #define atomic_subtract_rel_ptr         atomic_subtract_rel_64
 576 #define atomic_store_rel_ptr            atomic_store_rel_64
 577
 578 static __inline void
 579 atomic_thread_fence_acq(void)
 580 {
 581
 582         dmb(ld);
 583 }
 584
 585 static __inline void
 586 atomic_thread_fence_rel(void)
 587 {
 588
 589         dmb(sy);
 590 }
 591
 592 static __inline void
 593 atomic_thread_fence_acq_rel(void)
 594 {
 595
 596         dmb(sy);
 597 }
 598
 599 static __inline void
 600 atomic_thread_fence_seq_cst(void)
 601 {
 602
 603         dmb(sy);
 604 }
 605
 606 #include <sys/_atomic_subword.h>
 607
 608 #endif /* KCSAN && !KCSAN_RUNTIME */
 609 #endif /* _MACHINE_ATOMIC_H_ */