sys/arm/include/atomic-v6.h

   1 /* $NetBSD: atomic.h,v 1.1 2002/10/19 12:22:34 bsh Exp $ */
   2
   3 /*-
   4  * Copyright (C) 2003-2004 Olivier Houchard
   5  * Copyright (C) 1994-1997 Mark Brinicombe
   6  * Copyright (C) 1994 Brini
   7  * All rights reserved.
   8  *
   9  * This code is derived from software written for Brini by Mark Brinicombe
  10  *
  11  * Redistribution and use in source and binary forms, with or without
  12  * modification, are permitted provided that the following conditions
  13  * are met:
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in the
  18  *    documentation and/or other materials provided with the distribution.
  19  * 3. All advertising materials mentioning features or use of this software
  20  *    must display the following acknowledgement:
  21  *      This product includes software developed by Brini.
  22  * 4. The name of Brini may not be used to endorse or promote products
  23  *    derived from this software without specific prior written permission.
  24  *
  25  * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR
  26  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  27  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  28  * IN NO EVENT SHALL BRINI BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  29  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  30  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  31  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  32  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  33  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  34  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  35  *
  36  * $FreeBSD$
  37  */
  38
  39 #ifndef _MACHINE_ATOMIC_V6_H_
  40 #define _MACHINE_ATOMIC_V6_H_
  41
  42 #ifndef _MACHINE_ATOMIC_H_
  43 #error Do not include this file directly, use <machine/atomic.h>
  44 #endif
  45
  46 #if __ARM_ARCH >= 7
  47 #define isb()  __asm __volatile("isb" : : : "memory")
  48 #define dsb()  __asm __volatile("dsb" : : : "memory")
  49 #define dmb()  __asm __volatile("dmb" : : : "memory")
  50 #elif __ARM_ARCH >= 6
  51 #define isb()  __asm __volatile("mcr p15, 0, %0, c7, c5, 4" : : "r" (0) : "memory")
  52 #define dsb()  __asm __volatile("mcr p15, 0, %0, c7, c10, 4" : : "r" (0) : "memory")
  53 #define dmb()  __asm __volatile("mcr p15, 0, %0, c7, c10, 5" : : "r" (0) : "memory")
  54 #else
  55 #error Only use this file with ARMv6 and later
  56 #endif
  57
  58 #define mb()   dmb()
  59 #define wmb()  dmb()
  60 #define rmb()  dmb()
  61
  62 #define ARM_HAVE_ATOMIC64
  63
  64 #define ATOMIC_ACQ_REL_LONG(NAME)                                       \
  65 static __inline void                                                    \
  66 atomic_##NAME##_acq_long(__volatile u_long *p, u_long v)                \
  67 {                                                                       \
  68         atomic_##NAME##_long(p, v);                                     \
  69         dmb();                                                          \
  70 }                                                                       \
  71                                                                         \
  72 static __inline  void                                                   \
  73 atomic_##NAME##_rel_long(__volatile u_long *p, u_long v)                \
  74 {                                                                       \
  75         dmb();                                                          \
  76         atomic_##NAME##_long(p, v);                                     \
  77 }
  78
  79 #define ATOMIC_ACQ_REL(NAME, WIDTH)                                     \
  80 static __inline  void                                                   \
  81 atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\
  82 {                                                                       \
  83         atomic_##NAME##_##WIDTH(p, v);                                  \
  84         dmb();                                                          \
  85 }                                                                       \
  86                                                                         \
  87 static __inline  void                                                   \
  88 atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\
  89 {                                                                       \
  90         dmb();                                                          \
  91         atomic_##NAME##_##WIDTH(p, v);                                  \
  92 }
  93
  94
  95 static __inline void
  96 atomic_add_32(volatile uint32_t *p, uint32_t val)
  97 {
  98         uint32_t tmp = 0, tmp2 = 0;
  99
 100         __asm __volatile(
 101             "1: ldrex   %0, [%2]        \n"
 102             "   add     %0, %0, %3      \n"
 103             "   strex   %1, %0, [%2]    \n"
 104             "   cmp     %1, #0          \n"
 105             "   it      ne              \n"
 106             "   bne     1b              \n"
 107             : "=&r" (tmp), "+r" (tmp2)
 108             ,"+r" (p), "+r" (val) : : "cc", "memory");
 109 }
 110
 111 static __inline void
 112 atomic_add_64(volatile uint64_t *p, uint64_t val)
 113 {
 114         uint64_t tmp;
 115         uint32_t exflag;
 116
 117         __asm __volatile(
 118             "1:                                                 \n"
 119             "   ldrexd  %Q[tmp], %R[tmp], [%[ptr]]              \n"
 120             "   adds    %Q[tmp], %Q[val]                        \n"
 121             "   adc     %R[tmp], %R[tmp], %R[val]               \n"
 122             "   strexd  %[exf], %Q[tmp], %R[tmp], [%[ptr]]      \n"
 123             "   teq     %[exf], #0                              \n"
 124             "   it      ne                                      \n"
 125             "   bne     1b                                      \n"
 126             : [exf] "=&r" (exflag),
 127               [tmp] "=&r" (tmp)
 128             : [ptr] "r"   (p),
 129               [val] "r"   (val)
 130             : "cc", "memory");
 131 }
 132
 133 static __inline void
 134 atomic_add_long(volatile u_long *p, u_long val)
 135 {
 136
 137         atomic_add_32((volatile uint32_t *)p, val);
 138 }
 139
 140 ATOMIC_ACQ_REL(add, 32)
 141 ATOMIC_ACQ_REL(add, 64)
 142 ATOMIC_ACQ_REL_LONG(add)
 143
 144 static __inline void
 145 atomic_clear_32(volatile uint32_t *address, uint32_t setmask)
 146 {
 147         uint32_t tmp = 0, tmp2 = 0;
 148
 149         __asm __volatile(
 150             "1: ldrex   %0, [%2]        \n"
 151             "   bic     %0, %0, %3      \n"
 152             "   strex   %1, %0, [%2]    \n"
 153             "   cmp     %1, #0          \n"
 154             "   it      ne              \n"
 155             "   bne     1b              \n"
 156             : "=&r" (tmp), "+r" (tmp2), "+r" (address), "+r" (setmask)
 157             : : "cc", "memory");
 158 }
 159
 160 static __inline void
 161 atomic_clear_64(volatile uint64_t *p, uint64_t val)
 162 {
 163         uint64_t tmp;
 164         uint32_t exflag;
 165
 166         __asm __volatile(
 167             "1:                                                 \n"
 168             "   ldrexd  %Q[tmp], %R[tmp], [%[ptr]]              \n"
 169             "   bic     %Q[tmp], %Q[val]                        \n"
 170             "   bic     %R[tmp], %R[val]                        \n"
 171             "   strexd  %[exf], %Q[tmp], %R[tmp], [%[ptr]]      \n"
 172             "   teq     %[exf], #0                              \n"
 173             "   it      ne                                      \n"
 174             "   bne     1b                                      \n"
 175             : [exf] "=&r" (exflag),
 176               [tmp] "=&r" (tmp)
 177             : [ptr] "r"   (p),
 178               [val] "r"   (val)
 179             : "cc", "memory");
 180 }
 181
 182 static __inline void
 183 atomic_clear_long(volatile u_long *address, u_long setmask)
 184 {
 185
 186         atomic_clear_32((volatile uint32_t *)address, setmask);
 187 }
 188
 189 ATOMIC_ACQ_REL(clear, 32)
 190 ATOMIC_ACQ_REL(clear, 64)
 191 ATOMIC_ACQ_REL_LONG(clear)
 192
 193 static __inline uint32_t
 194 atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)
 195 {
 196         uint32_t ret;
 197
 198         __asm __volatile(
 199             "1: ldrex   %0, [%1]        \n"
 200             "   cmp     %0, %2          \n"
 201             "   itt     ne              \n"
 202             "   movne   %0, #0          \n"
 203             "   bne     2f              \n"
 204             "   strex   %0, %3, [%1]    \n"
 205             "   cmp     %0, #0          \n"
 206             "   ite     eq              \n"
 207             "   moveq   %0, #1          \n"
 208             "   bne     1b              \n"
 209             "2:"
 210             : "=&r" (ret), "+r" (p), "+r" (cmpval), "+r" (newval)
 211             : : "cc", "memory");
 212         return (ret);
 213 }
 214
 215 static __inline int
 216 atomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval)
 217 {
 218         uint64_t tmp;
 219         uint32_t ret;
 220
 221         __asm __volatile(
 222             "1:                                                 \n"
 223             "   ldrexd  %Q[tmp], %R[tmp], [%[ptr]]              \n"
 224             "   teq     %Q[tmp], %Q[cmpval]                     \n"
 225             "   itee    eq                                      \n"
 226             "   teqeq   %R[tmp], %R[cmpval]                     \n"
 227             "   movne   %[ret], #0                              \n"
 228             "   bne     2f                                      \n"
 229             "   strexd  %[ret], %Q[newval], %R[newval], [%[ptr]]\n"
 230             "   teq     %[ret], #0                              \n"
 231             "   it      ne                                      \n"
 232             "   bne     1b                                      \n"
 233             "   mov     %[ret], #1                              \n"
 234             "2:                                                 \n"
 235             : [ret]    "=&r" (ret),
 236               [tmp]    "=&r" (tmp)
 237             : [ptr]    "r"   (p),
 238               [cmpval] "r"   (cmpval),
 239               [newval] "r"   (newval)
 240             : "cc", "memory");
 241         return (ret);
 242 }
 243
 244 static __inline u_long
 245 atomic_cmpset_long(volatile u_long *p, u_long cmpval, u_long newval)
 246 {
 247
 248         return (atomic_cmpset_32((volatile uint32_t *)p, cmpval, newval));
 249 }
 250
 251 static __inline uint32_t
 252 atomic_cmpset_acq_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)
 253 {
 254         uint32_t ret;
 255
 256         ret = atomic_cmpset_32(p, cmpval, newval);
 257         dmb();
 258         return (ret);
 259 }
 260
 261 static __inline uint64_t
 262 atomic_cmpset_acq_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval)
 263 {
 264         uint64_t ret;
 265
 266         ret = atomic_cmpset_64(p, cmpval, newval);
 267         dmb();
 268         return (ret);
 269 }
 270
 271 static __inline u_long
 272 atomic_cmpset_acq_long(volatile u_long *p, u_long cmpval, u_long newval)
 273 {
 274         u_long ret;
 275
 276         ret = atomic_cmpset_long(p, cmpval, newval);
 277         dmb();
 278         return (ret);
 279 }
 280
 281 static __inline uint32_t
 282 atomic_cmpset_rel_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)
 283 {
 284
 285         dmb();
 286         return (atomic_cmpset_32(p, cmpval, newval));
 287 }
 288
 289 static __inline uint64_t
 290 atomic_cmpset_rel_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval)
 291 {
 292
 293         dmb();
 294         return (atomic_cmpset_64(p, cmpval, newval));
 295 }
 296
 297 static __inline u_long
 298 atomic_cmpset_rel_long(volatile u_long *p, u_long cmpval, u_long newval)
 299 {
 300
 301         dmb();
 302         return (atomic_cmpset_long(p, cmpval, newval));
 303 }
 304
 305 static __inline uint32_t
 306 atomic_fetchadd_32(volatile uint32_t *p, uint32_t val)
 307 {
 308         uint32_t tmp = 0, tmp2 = 0, ret = 0;
 309
 310         __asm __volatile(
 311             "1: ldrex   %0, [%3]        \n"
 312             "   add     %1, %0, %4      \n"
 313             "   strex   %2, %1, [%3]    \n"
 314             "   cmp     %2, #0          \n"
 315             "   it      ne              \n"
 316             "   bne     1b              \n"
 317             : "+r" (ret), "=&r" (tmp), "+r" (tmp2), "+r" (p), "+r" (val)
 318             : : "cc", "memory");
 319         return (ret);
 320 }
 321
 322 static __inline uint64_t
 323 atomic_fetchadd_64(volatile uint64_t *p, uint64_t val)
 324 {
 325         uint64_t ret, tmp;
 326         uint32_t exflag;
 327
 328         __asm __volatile(
 329             "1:                                                 \n"
 330             "   ldrexd  %Q[tmp], %R[tmp], [%[ptr]]              \n"
 331             "   adds    %Q[tmp], %Q[ret], %Q[val]               \n"
 332             "   adc     %R[tmp], %R[ret], %R[val]               \n"
 333             "   strexd  %[exf], %Q[tmp], %R[tmp], [%[ptr]]      \n"
 334             "   teq     %[exf], #0                              \n"
 335             "   it      ne                                      \n"
 336             "   bne     1b                                      \n"
 337             : [ret] "=&r" (ret),
 338               [exf] "=&r" (exflag),
 339               [tmp] "=&r" (tmp)
 340             : [ptr] "r"   (p),
 341               [val] "r"   (val)
 342             : "cc", "memory");
 343         return (ret);
 344 }
 345
 346 static __inline u_long
 347 atomic_fetchadd_long(volatile u_long *p, u_long val)
 348 {
 349
 350         return (atomic_fetchadd_32((volatile uint32_t *)p, val));
 351 }
 352
 353 static __inline uint32_t
 354 atomic_load_acq_32(volatile uint32_t *p)
 355 {
 356         uint32_t v;
 357
 358         v = *p;
 359         dmb();
 360         return (v);
 361 }
 362
 363 static __inline uint64_t
 364 atomic_load_64(volatile uint64_t *p)
 365 {
 366         uint64_t ret;
 367
 368         /*
 369          * The only way to atomically load 64 bits is with LDREXD which puts the
 370          * exclusive monitor into the exclusive state, so reset it to open state
 371          * with CLREX because we don't actually need to store anything.
 372          */
 373         __asm __volatile(
 374             "ldrexd     %Q[ret], %R[ret], [%[ptr]]      \n"
 375             "clrex                                      \n"
 376             : [ret] "=&r" (ret)
 377             : [ptr] "r"   (p)
 378             : "cc", "memory");
 379         return (ret);
 380 }
 381
 382 static __inline uint64_t
 383 atomic_load_acq_64(volatile uint64_t *p)
 384 {
 385         uint64_t ret;
 386
 387         ret = atomic_load_64(p);
 388         dmb();
 389         return (ret);
 390 }
 391
 392 static __inline u_long
 393 atomic_load_acq_long(volatile u_long *p)
 394 {
 395         u_long v;
 396
 397         v = *p;
 398         dmb();
 399         return (v);
 400 }
 401
 402 static __inline uint32_t
 403 atomic_readandclear_32(volatile uint32_t *p)
 404 {
 405         uint32_t ret, tmp = 0, tmp2 = 0;
 406
 407         __asm __volatile(
 408             "1: ldrex   %0, [%3]        \n"
 409             "   mov     %1, #0          \n"
 410             "   strex   %2, %1, [%3]    \n"
 411             "   cmp     %2, #0          \n"
 412             "   it      ne              \n"
 413             "   bne     1b              \n"
 414             : "=r" (ret), "=&r" (tmp), "+r" (tmp2), "+r" (p)
 415             : : "cc", "memory");
 416         return (ret);
 417 }
 418
 419 static __inline uint64_t
 420 atomic_readandclear_64(volatile uint64_t *p)
 421 {
 422         uint64_t ret, tmp;
 423         uint32_t exflag;
 424
 425         __asm __volatile(
 426             "1:                                                 \n"
 427             "   ldrexd  %Q[ret], %R[ret], [%[ptr]]              \n"
 428             "   mov     %Q[tmp], #0                             \n"
 429             "   mov     %R[tmp], #0                             \n"
 430             "   strexd  %[exf], %Q[tmp], %R[tmp], [%[ptr]]      \n"
 431             "   teq     %[exf], #0                              \n"
 432             "   it      ne                                      \n"
 433             "   bne     1b                                      \n"
 434             : [ret] "=&r" (ret),
 435               [exf] "=&r" (exflag),
 436               [tmp] "=&r" (tmp)
 437             : [ptr] "r"   (p)
 438             : "cc", "memory");
 439         return (ret);
 440 }
 441
 442 static __inline u_long
 443 atomic_readandclear_long(volatile u_long *p)
 444 {
 445
 446         return (atomic_readandclear_32((volatile uint32_t *)p));
 447 }
 448
 449 static __inline void
 450 atomic_set_32(volatile uint32_t *address, uint32_t setmask)
 451 {
 452         uint32_t tmp = 0, tmp2 = 0;
 453
 454         __asm __volatile(
 455             "1: ldrex   %0, [%2]        \n"
 456             "   orr     %0, %0, %3      \n"
 457             "   strex   %1, %0, [%2]    \n"
 458             "   cmp     %1, #0          \n"
 459             "   it      ne              \n"
 460             "   bne     1b              \n"
 461             : "=&r" (tmp), "+r" (tmp2), "+r" (address), "+r" (setmask)
 462             : : "cc", "memory");
 463 }
 464
 465 static __inline void
 466 atomic_set_64(volatile uint64_t *p, uint64_t val)
 467 {
 468         uint64_t tmp;
 469         uint32_t exflag;
 470
 471         __asm __volatile(
 472             "1:                                                 \n"
 473             "   ldrexd  %Q[tmp], %R[tmp], [%[ptr]]              \n"
 474             "   orr     %Q[tmp], %Q[val]                        \n"
 475             "   orr     %R[tmp], %R[val]                        \n"
 476             "   strexd  %[exf], %Q[tmp], %R[tmp], [%[ptr]]      \n"
 477             "   teq     %[exf], #0                              \n"
 478             "   it      ne                                      \n"
 479             "   bne     1b                                      \n"
 480             : [exf] "=&r" (exflag),
 481               [tmp] "=&r" (tmp)
 482             : [ptr] "r"   (p),
 483               [val] "r"   (val)
 484             : "cc", "memory");
 485 }
 486
 487 static __inline void
 488 atomic_set_long(volatile u_long *address, u_long setmask)
 489 {
 490
 491         atomic_set_32((volatile uint32_t *)address, setmask);
 492 }
 493
 494 ATOMIC_ACQ_REL(set, 32)
 495 ATOMIC_ACQ_REL(set, 64)
 496 ATOMIC_ACQ_REL_LONG(set)
 497
 498 static __inline void
 499 atomic_subtract_32(volatile uint32_t *p, uint32_t val)
 500 {
 501         uint32_t tmp = 0, tmp2 = 0;
 502
 503         __asm __volatile(
 504             "1: ldrex   %0, [%2]        \n"
 505             "   sub     %0, %0, %3      \n"
 506             "   strex   %1, %0, [%2]    \n"
 507             "   cmp     %1, #0          \n"
 508             "   it      ne              \n"
 509             "   bne     1b              \n"
 510             : "=&r" (tmp), "+r" (tmp2), "+r" (p), "+r" (val)
 511             : : "cc", "memory");
 512 }
 513
 514 static __inline void
 515 atomic_subtract_64(volatile uint64_t *p, uint64_t val)
 516 {
 517         uint64_t tmp;
 518         uint32_t exflag;
 519
 520         __asm __volatile(
 521             "1:                                                 \n"
 522             "   ldrexd  %Q[tmp], %R[tmp], [%[ptr]]              \n"
 523             "   subs    %Q[tmp], %Q[val]                        \n"
 524             "   sbc     %R[tmp], %R[tmp], %R[val]               \n"
 525             "   strexd  %[exf], %Q[tmp], %R[tmp], [%[ptr]]      \n"
 526             "   teq     %[exf], #0                              \n"
 527             "   it      ne                                      \n"
 528             "   bne     1b                                      \n"
 529             : [exf] "=&r" (exflag),
 530               [tmp] "=&r" (tmp)
 531             : [ptr] "r"   (p),
 532               [val] "r"   (val)
 533             : "cc", "memory");
 534 }
 535
 536 static __inline void
 537 atomic_subtract_long(volatile u_long *p, u_long val)
 538 {
 539
 540         atomic_subtract_32((volatile uint32_t *)p, val);
 541 }
 542
 543 ATOMIC_ACQ_REL(subtract, 32)
 544 ATOMIC_ACQ_REL(subtract, 64)
 545 ATOMIC_ACQ_REL_LONG(subtract)
 546
 547 static __inline void
 548 atomic_store_64(volatile uint64_t *p, uint64_t val)
 549 {
 550         uint64_t tmp;
 551         uint32_t exflag;
 552
 553         /*
 554          * The only way to atomically store 64 bits is with STREXD, which will
 555          * succeed only if paired up with a preceeding LDREXD using the same
 556          * address, so we read and discard the existing value before storing.
 557          */
 558         __asm __volatile(
 559             "1:                                                 \n"
 560             "   ldrexd  %Q[tmp], %R[tmp], [%[ptr]]              \n"
 561             "   strexd  %[exf], %Q[val], %R[val], [%[ptr]]      \n"
 562             "   teq     %[exf], #0                              \n"
 563             "   it      ne                                      \n"
 564             "   bne     1b                                      \n"
 565             : [tmp] "=&r" (tmp),
 566               [exf] "=&r" (exflag)
 567             : [ptr] "r"   (p),
 568               [val] "r"   (val)
 569             : "cc", "memory");
 570 }
 571
 572 static __inline void
 573 atomic_store_rel_32(volatile uint32_t *p, uint32_t v)
 574 {
 575
 576         dmb();
 577         *p = v;
 578 }
 579
 580 static __inline void
 581 atomic_store_rel_64(volatile uint64_t *p, uint64_t val)
 582 {
 583
 584         dmb();
 585         atomic_store_64(p, val);
 586 }
 587
 588 static __inline void
 589 atomic_store_rel_long(volatile u_long *p, u_long v)
 590 {
 591
 592         dmb();
 593         *p = v;
 594 }
 595
 596 static __inline int
 597 atomic_testandset_32(volatile uint32_t *p, u_int v)
 598 {
 599         uint32_t tmp, tmp2, res, mask;
 600
 601         mask = 1u << (v & 0x1f);
 602         tmp = tmp2 = 0;
 603         __asm __volatile(
 604         "1:     ldrex   %0, [%4]        \n"
 605         "       orr     %1, %0, %3      \n"
 606         "       strex   %2, %1, [%4]    \n"
 607         "       cmp     %2, #0          \n"
 608         "       it      ne              \n"
 609         "       bne     1b              \n"
 610         : "=&r" (res), "=&r" (tmp), "=&r" (tmp2)
 611         : "r" (mask), "r" (p)
 612         : "cc", "memory");
 613         return ((res & mask) != 0);
 614 }
 615
 616 static __inline int
 617 atomic_testandset_int(volatile u_int *p, u_int v)
 618 {
 619
 620         return (atomic_testandset_32((volatile uint32_t *)p, v));
 621 }
 622
 623 static __inline int
 624 atomic_testandset_long(volatile u_long *p, u_int v)
 625 {
 626
 627         return (atomic_testandset_32((volatile uint32_t *)p, v));
 628 }
 629
 630 static __inline int
 631 atomic_testandset_64(volatile uint64_t *p, u_int v)
 632 {
 633         volatile uint32_t *p32;
 634
 635         p32 = (volatile uint32_t *)p;
 636         /* Assume little-endian */
 637         if (v >= 32) {
 638                 v &= 0x1f;
 639                 p32++;
 640         }
 641         return (atomic_testandset_32(p32, v));
 642 }
 643
 644 static __inline uint32_t
 645 atomic_swap_32(volatile uint32_t *p, uint32_t v)
 646 {
 647         uint32_t ret, exflag;
 648
 649         __asm __volatile(
 650             "1: ldrex   %[ret], [%[ptr]]                \n"
 651             "   strex   %[exf], %[val], [%[ptr]]        \n"
 652             "   teq     %[exf], #0                      \n"
 653             "   it      ne                              \n"
 654             "   bne     1b                              \n"
 655             : [ret] "=r"  (ret),
 656               [exf] "=&r" (exflag)
 657             : [val] "r"  (v),
 658               [ptr] "r"  (p)
 659             : "cc", "memory");
 660         return (ret);
 661 }
 662
 663 #undef ATOMIC_ACQ_REL
 664 #undef ATOMIC_ACQ_REL_LONG
 665
 666 static __inline void
 667 atomic_thread_fence_acq(void)
 668 {
 669
 670         dmb();
 671 }
 672
 673 static __inline void
 674 atomic_thread_fence_rel(void)
 675 {
 676
 677         dmb();
 678 }
 679
 680 static __inline void
 681 atomic_thread_fence_acq_rel(void)
 682 {
 683
 684         dmb();
 685 }
 686
 687 static __inline void
 688 atomic_thread_fence_seq_cst(void)
 689 {
 690
 691         dmb();
 692 }
 693
 694 #endif /* _MACHINE_ATOMIC_V6_H_ */