sys/arm/include/atomic.h

   1 /* $NetBSD: atomic.h,v 1.1 2002/10/19 12:22:34 bsh Exp $ */
   2
   3 /*-
   4  * Copyright (C) 2003-2004 Olivier Houchard
   5  * Copyright (C) 1994-1997 Mark Brinicombe
   6  * Copyright (C) 1994 Brini
   7  * All rights reserved.
   8  *
   9  * This code is derived from software written for Brini by Mark Brinicombe
  10  *
  11  * Redistribution and use in source and binary forms, with or without
  12  * modification, are permitted provided that the following conditions
  13  * are met:
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in the
  18  *    documentation and/or other materials provided with the distribution.
  19  * 3. All advertising materials mentioning features or use of this software
  20  *    must display the following acknowledgement:
  21  *      This product includes software developed by Brini.
  22  * 4. The name of Brini may not be used to endorse or promote products
  23  *    derived from this software without specific prior written permission.
  24  *
  25  * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR
  26  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  27  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  28  * IN NO EVENT SHALL BRINI BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  29  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  30  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  31  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  32  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  33  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  34  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  35  *
  36  * $FreeBSD$
  37  */
  38
  39 #ifndef _MACHINE_ATOMIC_H_
  40 #define _MACHINE_ATOMIC_H_
  41
  42 #include <sys/types.h>
  43 #include <machine/armreg.h>
  44
  45 #ifndef _KERNEL
  46 #include <machine/sysarch.h>
  47 #else
  48 #include <machine/cpuconf.h>
  49 #endif
  50
  51 #if defined (__ARM_ARCH_7__) || defined (__ARM_ARCH_7A__)
  52 #define isb()  __asm __volatile("isb" : : : "memory")
  53 #define dsb()  __asm __volatile("dsb" : : : "memory")
  54 #define dmb()  __asm __volatile("dmb" : : : "memory")
  55 #elif defined (__ARM_ARCH_6__) || defined (__ARM_ARCH_6J__) || \
  56   defined (__ARM_ARCH_6K__) || defined (__ARM_ARCH_6T2__) || \
  57   defined (__ARM_ARCH_6Z__) || defined (__ARM_ARCH_6ZK__)
  58 #define isb()  __asm __volatile("mcr p15, 0, %0, c7, c5, 4" : : "r" (0) : "memory")
  59 #define dsb()  __asm __volatile("mcr p15, 0, %0, c7, c10, 4" : : "r" (0) : "memory")
  60 #define dmb()  __asm __volatile("mcr p15, 0, %0, c7, c10, 5" : : "r" (0) : "memory")
  61 #else
  62 #define isb()  __asm __volatile("mcr p15, 0, %0, c7, c5, 4" : : "r" (0) : "memory")
  63 #define dsb()  __asm __volatile("mcr p15, 0, %0, c7, c10, 4" : : "r" (0) : "memory")
  64 #define dmb()  dsb()
  65 #endif
  66
  67 #define mb()   dmb()
  68 #define wmb()  dmb()
  69 #define rmb()  dmb()
  70
  71
  72
  73 /*
  74  * It would be nice to use _HAVE_ARMv6_INSTRUCTIONS from machine/asm.h
  75  * here, but that header can't be included here because this is C
  76  * code.  I would like to move the _HAVE_ARMv6_INSTRUCTIONS definition
  77  * out of asm.h so it can be used in both asm and C code. - kientzle@
  78  */
  79 #if defined (__ARM_ARCH_7__) || \
  80         defined (__ARM_ARCH_7A__)  || \
  81         defined (__ARM_ARCH_6__)   || \
  82         defined (__ARM_ARCH_6J__)  || \
  83         defined (__ARM_ARCH_6K__)  || \
  84         defined (__ARM_ARCH_6T2__) || \
  85         defined (__ARM_ARCH_6Z__)  || \
  86         defined (__ARM_ARCH_6ZK__)
  87 #define ARM_HAVE_ATOMIC64
  88
  89 static __inline void
  90 __do_dmb(void)
  91 {
  92
  93 #if defined (__ARM_ARCH_7__) || defined (__ARM_ARCH_7A__)
  94         __asm __volatile("dmb" : : : "memory");
  95 #else
  96         __asm __volatile("mcr p15, 0, r0, c7, c10, 5" : : : "memory");
  97 #endif
  98 }
  99
 100 #define ATOMIC_ACQ_REL_LONG(NAME)                                       \
 101 static __inline void                                                    \
 102 atomic_##NAME##_acq_long(__volatile u_long *p, u_long v)                \
 103 {                                                                       \
 104         atomic_##NAME##_long(p, v);                                     \
 105         __do_dmb();                                                     \
 106 }                                                                       \
 107                                                                         \
 108 static __inline  void                                                   \
 109 atomic_##NAME##_rel_long(__volatile u_long *p, u_long v)                \
 110 {                                                                       \
 111         __do_dmb();                                                     \
 112         atomic_##NAME##_long(p, v);                                     \
 113 }
 114
 115 #define ATOMIC_ACQ_REL(NAME, WIDTH)                                     \
 116 static __inline  void                                                   \
 117 atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\
 118 {                                                                       \
 119         atomic_##NAME##_##WIDTH(p, v);                                  \
 120         __do_dmb();                                                     \
 121 }                                                                       \
 122                                                                         \
 123 static __inline  void                                                   \
 124 atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\
 125 {                                                                       \
 126         __do_dmb();                                                     \
 127         atomic_##NAME##_##WIDTH(p, v);                                  \
 128 }
 129
 130 static __inline void
 131 atomic_set_32(volatile uint32_t *address, uint32_t setmask)
 132 {
 133         uint32_t tmp = 0, tmp2 = 0;
 134
 135         __asm __volatile("1: ldrex %0, [%2]\n"
 136                             "orr %0, %0, %3\n"
 137                             "strex %1, %0, [%2]\n"
 138                             "cmp %1, #0\n"
 139                             "it ne\n"
 140                             "bne        1b\n"
 141                            : "=&r" (tmp), "+r" (tmp2)
 142                            , "+r" (address), "+r" (setmask) : : "cc", "memory");
 143
 144 }
 145
 146 static __inline void
 147 atomic_set_64(volatile uint64_t *p, uint64_t val)
 148 {
 149         uint64_t tmp;
 150         uint32_t exflag;
 151
 152         __asm __volatile(
 153                 "1:          \n"
 154                 "   ldrexd   %[tmp], [%[ptr]]\n"
 155                 "   orr      %Q[tmp], %Q[val]\n"
 156                 "   orr      %R[tmp], %R[val]\n"
 157                 "   strexd   %[exf], %[tmp], [%[ptr]]\n"
 158                 "   teq      %[exf], #0\n"
 159                 "   it ne    \n"
 160                 "   bne      1b\n"
 161                 :   [exf]    "=&r"  (exflag),
 162                     [tmp]    "=&r"  (tmp)
 163                 :   [ptr]    "r"    (p),
 164                     [val]    "r"    (val)
 165                 :   "cc", "memory");
 166 }
 167
 168 static __inline void
 169 atomic_set_long(volatile u_long *address, u_long setmask)
 170 {
 171         u_long tmp = 0, tmp2 = 0;
 172
 173         __asm __volatile("1: ldrex %0, [%2]\n"
 174                             "orr %0, %0, %3\n"
 175                             "strex %1, %0, [%2]\n"
 176                             "cmp %1, #0\n"
 177                             "it ne\n"
 178                             "bne        1b\n"
 179                            : "=&r" (tmp), "+r" (tmp2)
 180                            , "+r" (address), "+r" (setmask) : : "cc", "memory");
 181
 182 }
 183
 184 static __inline void
 185 atomic_clear_32(volatile uint32_t *address, uint32_t setmask)
 186 {
 187         uint32_t tmp = 0, tmp2 = 0;
 188
 189         __asm __volatile("1: ldrex %0, [%2]\n"
 190                             "bic %0, %0, %3\n"
 191                             "strex %1, %0, [%2]\n"
 192                             "cmp %1, #0\n"
 193                             "it ne\n"
 194                             "bne        1b\n"
 195                            : "=&r" (tmp), "+r" (tmp2)
 196                            ,"+r" (address), "+r" (setmask) : : "cc", "memory");
 197 }
 198
 199 static __inline void
 200 atomic_clear_64(volatile uint64_t *p, uint64_t val)
 201 {
 202         uint64_t tmp;
 203         uint32_t exflag;
 204
 205         __asm __volatile(
 206                 "1:          \n"
 207                 "   ldrexd   %[tmp], [%[ptr]]\n"
 208                 "   bic      %Q[tmp], %Q[val]\n"
 209                 "   bic      %R[tmp], %R[val]\n"
 210                 "   strexd   %[exf], %[tmp], [%[ptr]]\n"
 211                 "   teq      %[exf], #0\n"
 212                 "   it ne    \n"
 213                 "   bne      1b\n"
 214                 :   [exf]    "=&r"  (exflag),
 215                     [tmp]    "=&r"  (tmp)
 216                 :   [ptr]    "r"    (p),
 217                     [val]    "r"    (val)
 218                 :   "cc", "memory");
 219 }
 220
 221 static __inline void
 222 atomic_clear_long(volatile u_long *address, u_long setmask)
 223 {
 224         u_long tmp = 0, tmp2 = 0;
 225
 226         __asm __volatile("1: ldrex %0, [%2]\n"
 227                             "bic %0, %0, %3\n"
 228                             "strex %1, %0, [%2]\n"
 229                             "cmp %1, #0\n"
 230                             "it ne\n"
 231                             "bne        1b\n"
 232                            : "=&r" (tmp), "+r" (tmp2)
 233                            ,"+r" (address), "+r" (setmask) : : "cc", "memory");
 234 }
 235
 236 static __inline u_int32_t
 237 atomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval)
 238 {
 239         uint32_t ret;
 240
 241         __asm __volatile("1: ldrex %0, [%1]\n"
 242                          "cmp %0, %2\n"
 243                          "itt ne\n"
 244                          "movne %0, #0\n"
 245                          "bne 2f\n"
 246                          "strex %0, %3, [%1]\n"
 247                          "cmp %0, #0\n"
 248                          "ite eq\n"
 249                          "moveq %0, #1\n"
 250                          "bne   1b\n"
 251                          "2:"
 252                          : "=&r" (ret)
 253                          ,"+r" (p), "+r" (cmpval), "+r" (newval) : : "cc",
 254                          "memory");
 255         return (ret);
 256 }
 257
 258 static __inline int
 259 atomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval)
 260 {
 261         uint64_t tmp;
 262         uint32_t ret;
 263
 264         __asm __volatile(
 265                 "1:          \n"
 266                 "   ldrexd   %[tmp], [%[ptr]]\n"
 267                 "   teq      %Q[tmp], %Q[cmpval]\n"
 268                 "   itee eq  \n"
 269                 "   teqeq    %R[tmp], %R[cmpval]\n"
 270                 "   movne    %[ret], #0\n"
 271                 "   bne      2f\n"
 272                 "   strexd   %[ret], %[newval], [%[ptr]]\n"
 273                 "   teq      %[ret], #0\n"
 274                 "   it ne    \n"
 275                 "   bne      1b\n"
 276                 "   mov      %[ret], #1\n"
 277                 "2:          \n"
 278                 :   [ret]    "=&r"  (ret),
 279                     [tmp]    "=&r"  (tmp)
 280                 :   [ptr]    "r"    (p),
 281                     [cmpval] "r"    (cmpval),
 282                     [newval] "r"    (newval)
 283                 :   "cc", "memory");
 284         return (ret);
 285 }
 286
 287 static __inline u_long
 288 atomic_cmpset_long(volatile u_long *p, volatile u_long cmpval, volatile u_long newval)
 289 {
 290         u_long ret;
 291
 292         __asm __volatile("1: ldrex %0, [%1]\n"
 293                          "cmp %0, %2\n"
 294                          "itt ne\n"
 295                          "movne %0, #0\n"
 296                          "bne 2f\n"
 297                          "strex %0, %3, [%1]\n"
 298                          "cmp %0, #0\n"
 299                          "ite eq\n"
 300                          "moveq %0, #1\n"
 301                          "bne   1b\n"
 302                          "2:"
 303                          : "=&r" (ret)
 304                          ,"+r" (p), "+r" (cmpval), "+r" (newval) : : "cc",
 305                          "memory");
 306         return (ret);
 307 }
 308
 309 static __inline u_int32_t
 310 atomic_cmpset_acq_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval)
 311 {
 312         u_int32_t ret = atomic_cmpset_32(p, cmpval, newval);
 313
 314         __do_dmb();
 315         return (ret);
 316 }
 317
 318 static __inline uint64_t
 319 atomic_cmpset_acq_64(volatile uint64_t *p, volatile uint64_t cmpval, volatile uint64_t newval)
 320 {
 321         uint64_t ret = atomic_cmpset_64(p, cmpval, newval);
 322
 323         __do_dmb();
 324         return (ret);
 325 }
 326
 327 static __inline u_long
 328 atomic_cmpset_acq_long(volatile u_long *p, volatile u_long cmpval, volatile u_long newval)
 329 {
 330         u_long ret = atomic_cmpset_long(p, cmpval, newval);
 331
 332         __do_dmb();
 333         return (ret);
 334 }
 335
 336 static __inline u_int32_t
 337 atomic_cmpset_rel_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval)
 338 {
 339
 340         __do_dmb();
 341         return (atomic_cmpset_32(p, cmpval, newval));
 342 }
 343
 344 static __inline uint64_t
 345 atomic_cmpset_rel_64(volatile uint64_t *p, volatile uint64_t cmpval, volatile uint64_t newval)
 346 {
 347
 348         __do_dmb();
 349         return (atomic_cmpset_64(p, cmpval, newval));
 350 }
 351
 352 static __inline u_long
 353 atomic_cmpset_rel_long(volatile u_long *p, volatile u_long cmpval, volatile u_long newval)
 354 {
 355
 356         __do_dmb();
 357         return (atomic_cmpset_long(p, cmpval, newval));
 358 }
 359
 360
 361 static __inline void
 362 atomic_add_32(volatile u_int32_t *p, u_int32_t val)
 363 {
 364         uint32_t tmp = 0, tmp2 = 0;
 365
 366         __asm __volatile("1: ldrex %0, [%2]\n"
 367                             "add %0, %0, %3\n"
 368                             "strex %1, %0, [%2]\n"
 369                             "cmp %1, #0\n"
 370                             "it ne\n"
 371                             "bne        1b\n"
 372                             : "=&r" (tmp), "+r" (tmp2)
 373                             ,"+r" (p), "+r" (val) : : "cc", "memory");
 374 }
 375
 376 static __inline void
 377 atomic_add_64(volatile uint64_t *p, uint64_t val)
 378 {
 379         uint64_t tmp;
 380         uint32_t exflag;
 381
 382         __asm __volatile(
 383                 "1:          \n"
 384                 "   ldrexd   %[tmp], [%[ptr]]\n"
 385                 "   adds     %Q[tmp], %Q[val]\n"
 386                 "   adc      %R[tmp], %R[val]\n"
 387                 "   strexd   %[exf], %[tmp], [%[ptr]]\n"
 388                 "   teq      %[exf], #0\n"
 389                 "   it ne    \n"
 390                 "   bne      1b\n"
 391                 :   [exf]    "=&r"  (exflag),
 392                     [tmp]    "=&r"  (tmp)
 393                 :   [ptr]    "r"    (p),
 394                     [val]    "r"    (val)
 395                 :   "cc", "memory");
 396 }
 397
 398 static __inline void
 399 atomic_add_long(volatile u_long *p, u_long val)
 400 {
 401         u_long tmp = 0, tmp2 = 0;
 402
 403         __asm __volatile("1: ldrex %0, [%2]\n"
 404                             "add %0, %0, %3\n"
 405                             "strex %1, %0, [%2]\n"
 406                             "cmp %1, #0\n"
 407                             "it ne\n"
 408                             "bne        1b\n"
 409                             : "=&r" (tmp), "+r" (tmp2)
 410                             ,"+r" (p), "+r" (val) : : "cc", "memory");
 411 }
 412
 413 static __inline void
 414 atomic_subtract_32(volatile u_int32_t *p, u_int32_t val)
 415 {
 416         uint32_t tmp = 0, tmp2 = 0;
 417
 418         __asm __volatile("1: ldrex %0, [%2]\n"
 419                             "sub %0, %0, %3\n"
 420                             "strex %1, %0, [%2]\n"
 421                             "cmp %1, #0\n"
 422                             "it ne\n"
 423                             "bne        1b\n"
 424                             : "=&r" (tmp), "+r" (tmp2)
 425                             ,"+r" (p), "+r" (val) : : "cc", "memory");
 426 }
 427
 428 static __inline void
 429 atomic_subtract_64(volatile uint64_t *p, uint64_t val)
 430 {
 431         uint64_t tmp;
 432         uint32_t exflag;
 433
 434         __asm __volatile(
 435                 "1:          \n"
 436                 "   ldrexd   %[tmp], [%[ptr]]\n"
 437                 "   subs     %Q[tmp], %Q[val]\n"
 438                 "   sbc      %R[tmp], %R[val]\n"
 439                 "   strexd   %[exf], %[tmp], [%[ptr]]\n"
 440                 "   teq      %[exf], #0\n"
 441                 "   it ne    \n"
 442                 "   bne      1b\n"
 443                 :   [exf]    "=&r"  (exflag),
 444                     [tmp]    "=&r"  (tmp)
 445                 :   [ptr]    "r"    (p),
 446                     [val]    "r"    (val)
 447                 :   "cc", "memory");
 448 }
 449
 450 static __inline void
 451 atomic_subtract_long(volatile u_long *p, u_long val)
 452 {
 453         u_long tmp = 0, tmp2 = 0;
 454
 455         __asm __volatile("1: ldrex %0, [%2]\n"
 456                             "sub %0, %0, %3\n"
 457                             "strex %1, %0, [%2]\n"
 458                             "cmp %1, #0\n"
 459                             "it ne\n"
 460                             "bne        1b\n"
 461                             : "=&r" (tmp), "+r" (tmp2)
 462                             ,"+r" (p), "+r" (val) : : "cc", "memory");
 463 }
 464
 465 ATOMIC_ACQ_REL(clear, 32)
 466 ATOMIC_ACQ_REL(add, 32)
 467 ATOMIC_ACQ_REL(subtract, 32)
 468 ATOMIC_ACQ_REL(set, 32)
 469 ATOMIC_ACQ_REL(clear, 64)
 470 ATOMIC_ACQ_REL(add, 64)
 471 ATOMIC_ACQ_REL(subtract, 64)
 472 ATOMIC_ACQ_REL(set, 64)
 473 ATOMIC_ACQ_REL_LONG(clear)
 474 ATOMIC_ACQ_REL_LONG(add)
 475 ATOMIC_ACQ_REL_LONG(subtract)
 476 ATOMIC_ACQ_REL_LONG(set)
 477
 478 #undef ATOMIC_ACQ_REL
 479 #undef ATOMIC_ACQ_REL_LONG
 480
 481 static __inline uint32_t
 482 atomic_fetchadd_32(volatile uint32_t *p, uint32_t val)
 483 {
 484         uint32_t tmp = 0, tmp2 = 0, ret = 0;
 485
 486         __asm __volatile("1: ldrex %0, [%3]\n"
 487                             "add %1, %0, %4\n"
 488                             "strex %2, %1, [%3]\n"
 489                             "cmp %2, #0\n"
 490                             "it ne\n"
 491                             "bne        1b\n"
 492                            : "+r" (ret), "=&r" (tmp), "+r" (tmp2)
 493                            ,"+r" (p), "+r" (val) : : "cc", "memory");
 494         return (ret);
 495 }
 496
 497 static __inline uint32_t
 498 atomic_readandclear_32(volatile u_int32_t *p)
 499 {
 500         uint32_t ret, tmp = 0, tmp2 = 0;
 501
 502         __asm __volatile("1: ldrex %0, [%3]\n"
 503                          "mov %1, #0\n"
 504                          "strex %2, %1, [%3]\n"
 505                          "cmp %2, #0\n"
 506                          "it ne\n"
 507                          "bne 1b\n"
 508                          : "=r" (ret), "=&r" (tmp), "+r" (tmp2)
 509                          ,"+r" (p) : : "cc", "memory");
 510         return (ret);
 511 }
 512
 513 static __inline uint32_t
 514 atomic_load_acq_32(volatile uint32_t *p)
 515 {
 516         uint32_t v;
 517
 518         v = *p;
 519         __do_dmb();
 520         return (v);
 521 }
 522
 523 static __inline void
 524 atomic_store_rel_32(volatile uint32_t *p, uint32_t v)
 525 {
 526
 527         __do_dmb();
 528         *p = v;
 529 }
 530
 531 static __inline uint64_t
 532 atomic_fetchadd_64(volatile uint64_t *p, uint64_t val)
 533 {
 534         uint64_t ret, tmp;
 535         uint32_t exflag;
 536
 537         __asm __volatile(
 538                 "1:          \n"
 539                 "   ldrexd   %[ret], [%[ptr]]\n"
 540                 "   adds     %Q[tmp], %Q[ret], %Q[val]\n"
 541                 "   adc      %R[tmp], %R[ret], %R[val]\n"
 542                 "   strexd   %[exf], %[tmp], [%[ptr]]\n"
 543                 "   teq      %[exf], #0\n"
 544                 "   it ne    \n"
 545                 "   bne      1b\n"
 546                 :   [ret]    "=&r"  (ret),
 547                     [exf]    "=&r"  (exflag),
 548                     [tmp]    "=&r"  (tmp)
 549                 :   [ptr]    "r"    (p),
 550                     [val]    "r"    (val)
 551                 :   "cc", "memory");
 552         return (ret);
 553 }
 554
 555 static __inline uint64_t
 556 atomic_readandclear_64(volatile uint64_t *p)
 557 {
 558         uint64_t ret, tmp;
 559         uint32_t exflag;
 560
 561         __asm __volatile(
 562                 "1:          \n"
 563                 "   ldrexd   %[ret], [%[ptr]]\n"
 564                 "   mov      %Q[tmp], #0\n"
 565                 "   mov      %R[tmp], #0\n"
 566                 "   strexd   %[exf], %[tmp], [%[ptr]]\n"
 567                 "   teq      %[exf], #0\n"
 568                 "   it ne    \n"
 569                 "   bne      1b\n"
 570                 :   [ret]    "=&r"  (ret),
 571                     [exf]    "=&r"  (exflag),
 572                     [tmp]    "=&r"  (tmp)
 573                 :   [ptr]    "r"    (p)
 574                 :   "cc", "memory");
 575         return (ret);
 576 }
 577
 578 static __inline uint64_t
 579 atomic_load_64(volatile uint64_t *p)
 580 {
 581         uint64_t ret;
 582
 583         /*
 584          * The only way to atomically load 64 bits is with LDREXD which puts the
 585          * exclusive monitor into the exclusive state, so reset it to open state
 586          * with CLREX because we don't actually need to store anything.
 587          */
 588         __asm __volatile(
 589                 "1:          \n"
 590                 "   ldrexd   %[ret], [%[ptr]]\n"
 591                 "   clrex    \n"
 592                 :   [ret]    "=&r"  (ret)
 593                 :   [ptr]    "r"    (p)
 594                 :   "cc", "memory");
 595         return (ret);
 596 }
 597
 598 static __inline uint64_t
 599 atomic_load_acq_64(volatile uint64_t *p)
 600 {
 601         uint64_t ret;
 602
 603         ret = atomic_load_64(p);
 604         __do_dmb();
 605         return (ret);
 606 }
 607
 608 static __inline void
 609 atomic_store_64(volatile uint64_t *p, uint64_t val)
 610 {
 611         uint64_t tmp;
 612         uint32_t exflag;
 613
 614         /*
 615          * The only way to atomically store 64 bits is with STREXD, which will
 616          * succeed only if paired up with a preceeding LDREXD using the same
 617          * address, so we read and discard the existing value before storing.
 618          */
 619         __asm __volatile(
 620                 "1:          \n"
 621                 "   ldrexd   %[tmp], [%[ptr]]\n"
 622                 "   strexd   %[exf], %[val], [%[ptr]]\n"
 623                 "   teq      %[exf], #0\n"
 624                 "   it ne    \n"
 625                 "   bne      1b\n"
 626                 :   [tmp]    "=&r"  (tmp),
 627                     [exf]    "=&r"  (exflag)
 628                 :   [ptr]    "r"    (p),
 629                     [val]    "r"    (val)
 630                 :   "cc", "memory");
 631 }
 632
 633 static __inline void
 634 atomic_store_rel_64(volatile uint64_t *p, uint64_t val)
 635 {
 636
 637         __do_dmb();
 638         atomic_store_64(p, val);
 639 }
 640
 641 static __inline u_long
 642 atomic_fetchadd_long(volatile u_long *p, u_long val)
 643 {
 644         u_long tmp = 0, tmp2 = 0, ret = 0;
 645
 646         __asm __volatile("1: ldrex %0, [%3]\n"
 647                             "add %1, %0, %4\n"
 648                             "strex %2, %1, [%3]\n"
 649                             "cmp %2, #0\n"
 650                             "it ne\n"
 651                             "bne        1b\n"
 652                            : "+r" (ret), "=&r" (tmp), "+r" (tmp2)
 653                            ,"+r" (p), "+r" (val) : : "cc", "memory");
 654         return (ret);
 655 }
 656
 657 static __inline u_long
 658 atomic_readandclear_long(volatile u_long *p)
 659 {
 660         u_long ret, tmp = 0, tmp2 = 0;
 661
 662         __asm __volatile("1: ldrex %0, [%3]\n"
 663                          "mov %1, #0\n"
 664                          "strex %2, %1, [%3]\n"
 665                          "cmp %2, #0\n"
 666                          "it ne\n"
 667                          "bne 1b\n"
 668                          : "=r" (ret), "=&r" (tmp), "+r" (tmp2)
 669                          ,"+r" (p) : : "cc", "memory");
 670         return (ret);
 671 }
 672
 673 static __inline u_long
 674 atomic_load_acq_long(volatile u_long *p)
 675 {
 676         u_long v;
 677
 678         v = *p;
 679         __do_dmb();
 680         return (v);
 681 }
 682
 683 static __inline void
 684 atomic_store_rel_long(volatile u_long *p, u_long v)
 685 {
 686
 687         __do_dmb();
 688         *p = v;
 689 }
 690 #else /* < armv6 */
 691
 692 #define __with_interrupts_disabled(expr) \
 693         do {                                            \
 694                 u_int cpsr_save, tmp;                   \
 695                                                         \
 696                 __asm __volatile(                       \
 697                         "mrs  %0, cpsr;"                \
 698                         "orr  %1, %0, %2;"              \
 699                         "msr  cpsr_fsxc, %1;"           \
 700                         : "=r" (cpsr_save), "=r" (tmp)  \
 701                         : "I" (PSR_I | PSR_F)           \
 702                         : "cc" );               \
 703                 (expr);                         \
 704                  __asm __volatile(              \
 705                         "msr  cpsr_fsxc, %0"    \
 706                         : /* no output */       \
 707                         : "r" (cpsr_save)       \
 708                         : "cc" );               \
 709         } while(0)
 710
 711 static __inline uint32_t
 712 __swp(uint32_t val, volatile uint32_t *ptr)
 713 {
 714         __asm __volatile("swp   %0, %2, [%3]"
 715             : "=&r" (val), "=m" (*ptr)
 716             : "r" (val), "r" (ptr), "m" (*ptr)
 717             : "memory");
 718         return (val);
 719 }
 720
 721
 722 #ifdef _KERNEL
 723 #define ARM_HAVE_ATOMIC64
 724
 725 static __inline void
 726 atomic_set_32(volatile uint32_t *address, uint32_t setmask)
 727 {
 728         __with_interrupts_disabled(*address |= setmask);
 729 }
 730
 731 static __inline void
 732 atomic_set_64(volatile uint64_t *address, uint64_t setmask)
 733 {
 734         __with_interrupts_disabled(*address |= setmask);
 735 }
 736
 737 static __inline void
 738 atomic_clear_32(volatile uint32_t *address, uint32_t clearmask)
 739 {
 740         __with_interrupts_disabled(*address &= ~clearmask);
 741 }
 742
 743 static __inline void
 744 atomic_clear_64(volatile uint64_t *address, uint64_t clearmask)
 745 {
 746         __with_interrupts_disabled(*address &= ~clearmask);
 747 }
 748
 749 static __inline u_int32_t
 750 atomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval)
 751 {
 752         int ret;
 753
 754         __with_interrupts_disabled(
 755          {
 756                 if (*p == cmpval) {
 757                         *p = newval;
 758                         ret = 1;
 759                 } else {
 760                         ret = 0;
 761                 }
 762         });
 763         return (ret);
 764 }
 765
 766 static __inline u_int64_t
 767 atomic_cmpset_64(volatile u_int64_t *p, volatile u_int64_t cmpval, volatile u_int64_t newval)
 768 {
 769         int ret;
 770
 771         __with_interrupts_disabled(
 772          {
 773                 if (*p == cmpval) {
 774                         *p = newval;
 775                         ret = 1;
 776                 } else {
 777                         ret = 0;
 778                 }
 779         });
 780         return (ret);
 781 }
 782
 783 static __inline void
 784 atomic_add_32(volatile u_int32_t *p, u_int32_t val)
 785 {
 786         __with_interrupts_disabled(*p += val);
 787 }
 788
 789 static __inline void
 790 atomic_add_64(volatile u_int64_t *p, u_int64_t val)
 791 {
 792         __with_interrupts_disabled(*p += val);
 793 }
 794
 795 static __inline void
 796 atomic_subtract_32(volatile u_int32_t *p, u_int32_t val)
 797 {
 798         __with_interrupts_disabled(*p -= val);
 799 }
 800
 801 static __inline void
 802 atomic_subtract_64(volatile u_int64_t *p, u_int64_t val)
 803 {
 804         __with_interrupts_disabled(*p -= val);
 805 }
 806
 807 static __inline uint32_t
 808 atomic_fetchadd_32(volatile uint32_t *p, uint32_t v)
 809 {
 810         uint32_t value;
 811
 812         __with_interrupts_disabled(
 813         {
 814                 value = *p;
 815                 *p += v;
 816         });
 817         return (value);
 818 }
 819
 820 static __inline uint64_t
 821 atomic_fetchadd_64(volatile uint64_t *p, uint64_t v)
 822 {
 823         uint64_t value;
 824
 825         __with_interrupts_disabled(
 826         {
 827                 value = *p;
 828                 *p += v;
 829         });
 830         return (value);
 831 }
 832
 833 static __inline uint64_t
 834 atomic_load_64(volatile uint64_t *p)
 835 {
 836         uint64_t value;
 837
 838         __with_interrupts_disabled(value = *p);
 839         return (value);
 840 }
 841
 842 static __inline void
 843 atomic_store_64(volatile uint64_t *p, uint64_t value)
 844 {
 845         __with_interrupts_disabled(*p = value);
 846 }
 847
 848 #else /* !_KERNEL */
 849
 850 static __inline u_int32_t
 851 atomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval)
 852 {
 853         register int done, ras_start = ARM_RAS_START;
 854
 855         __asm __volatile("1:\n"
 856             "adr        %1, 1b\n"
 857             "str        %1, [%0]\n"
 858             "adr        %1, 2f\n"
 859             "str        %1, [%0, #4]\n"
 860             "ldr        %1, [%2]\n"
 861             "cmp        %1, %3\n"
 862             "streq      %4, [%2]\n"
 863             "2:\n"
 864             "mov        %1, #0\n"
 865             "str        %1, [%0]\n"
 866             "mov        %1, #0xffffffff\n"
 867             "str        %1, [%0, #4]\n"
 868             "moveq      %1, #1\n"
 869             "movne      %1, #0\n"
 870             : "+r" (ras_start), "=r" (done)
 871             ,"+r" (p), "+r" (cmpval), "+r" (newval) : : "cc", "memory");
 872         return (done);
 873 }
 874
 875 static __inline void
 876 atomic_add_32(volatile u_int32_t *p, u_int32_t val)
 877 {
 878         int start, ras_start = ARM_RAS_START;
 879
 880         __asm __volatile("1:\n"
 881             "adr        %1, 1b\n"
 882             "str        %1, [%0]\n"
 883             "adr        %1, 2f\n"
 884             "str        %1, [%0, #4]\n"
 885             "ldr        %1, [%2]\n"
 886             "add        %1, %1, %3\n"
 887             "str        %1, [%2]\n"
 888             "2:\n"
 889             "mov        %1, #0\n"
 890             "str        %1, [%0]\n"
 891             "mov        %1, #0xffffffff\n"
 892             "str        %1, [%0, #4]\n"
 893             : "+r" (ras_start), "=r" (start), "+r" (p), "+r" (val)
 894             : : "memory");
 895 }
 896
 897 static __inline void
 898 atomic_subtract_32(volatile u_int32_t *p, u_int32_t val)
 899 {
 900         int start, ras_start = ARM_RAS_START;
 901
 902         __asm __volatile("1:\n"
 903             "adr        %1, 1b\n"
 904             "str        %1, [%0]\n"
 905             "adr        %1, 2f\n"
 906             "str        %1, [%0, #4]\n"
 907             "ldr        %1, [%2]\n"
 908             "sub        %1, %1, %3\n"
 909             "str        %1, [%2]\n"
 910             "2:\n"
 911             "mov        %1, #0\n"
 912             "str        %1, [%0]\n"
 913             "mov        %1, #0xffffffff\n"
 914             "str        %1, [%0, #4]\n"
 915
 916             : "+r" (ras_start), "=r" (start), "+r" (p), "+r" (val)
 917             : : "memory");
 918 }
 919
 920 static __inline void
 921 atomic_set_32(volatile uint32_t *address, uint32_t setmask)
 922 {
 923         int start, ras_start = ARM_RAS_START;
 924
 925         __asm __volatile("1:\n"
 926             "adr        %1, 1b\n"
 927             "str        %1, [%0]\n"
 928             "adr        %1, 2f\n"
 929             "str        %1, [%0, #4]\n"
 930             "ldr        %1, [%2]\n"
 931             "orr        %1, %1, %3\n"
 932             "str        %1, [%2]\n"
 933             "2:\n"
 934             "mov        %1, #0\n"
 935             "str        %1, [%0]\n"
 936             "mov        %1, #0xffffffff\n"
 937             "str        %1, [%0, #4]\n"
 938
 939             : "+r" (ras_start), "=r" (start), "+r" (address), "+r" (setmask)
 940             : : "memory");
 941 }
 942
 943 static __inline void
 944 atomic_clear_32(volatile uint32_t *address, uint32_t clearmask)
 945 {
 946         int start, ras_start = ARM_RAS_START;
 947
 948         __asm __volatile("1:\n"
 949             "adr        %1, 1b\n"
 950             "str        %1, [%0]\n"
 951             "adr        %1, 2f\n"
 952             "str        %1, [%0, #4]\n"
 953             "ldr        %1, [%2]\n"
 954             "bic        %1, %1, %3\n"
 955             "str        %1, [%2]\n"
 956             "2:\n"
 957             "mov        %1, #0\n"
 958             "str        %1, [%0]\n"
 959             "mov        %1, #0xffffffff\n"
 960             "str        %1, [%0, #4]\n"
 961             : "+r" (ras_start), "=r" (start), "+r" (address), "+r" (clearmask)
 962             : : "memory");
 963
 964 }
 965
 966 static __inline uint32_t
 967 atomic_fetchadd_32(volatile uint32_t *p, uint32_t v)
 968 {
 969         uint32_t start, tmp, ras_start = ARM_RAS_START;
 970
 971         __asm __volatile("1:\n"
 972             "adr        %1, 1b\n"
 973             "str        %1, [%0]\n"
 974             "adr        %1, 2f\n"
 975             "str        %1, [%0, #4]\n"
 976             "ldr        %1, [%3]\n"
 977             "mov        %2, %1\n"
 978             "add        %2, %2, %4\n"
 979             "str        %2, [%3]\n"
 980             "2:\n"
 981             "mov        %2, #0\n"
 982             "str        %2, [%0]\n"
 983             "mov        %2, #0xffffffff\n"
 984             "str        %2, [%0, #4]\n"
 985             : "+r" (ras_start), "=r" (start), "=r" (tmp), "+r" (p), "+r" (v)
 986             : : "memory");
 987         return (start);
 988 }
 989
 990 #endif /* _KERNEL */
 991
 992
 993 static __inline uint32_t
 994 atomic_readandclear_32(volatile u_int32_t *p)
 995 {
 996
 997         return (__swp(0, p));
 998 }
 999
1000 #define atomic_cmpset_rel_32    atomic_cmpset_32
1001 #define atomic_cmpset_acq_32    atomic_cmpset_32
1002 #define atomic_set_rel_32       atomic_set_32
1003 #define atomic_set_acq_32       atomic_set_32
1004 #define atomic_clear_rel_32     atomic_clear_32
1005 #define atomic_clear_acq_32     atomic_clear_32
1006 #define atomic_add_rel_32       atomic_add_32
1007 #define atomic_add_acq_32       atomic_add_32
1008 #define atomic_subtract_rel_32  atomic_subtract_32
1009 #define atomic_subtract_acq_32  atomic_subtract_32
1010 #define atomic_store_rel_32     atomic_store_32
1011 #define atomic_store_rel_long   atomic_store_long
1012 #define atomic_load_acq_32      atomic_load_32
1013 #define atomic_load_acq_long    atomic_load_long
1014 #define atomic_add_acq_long             atomic_add_long
1015 #define atomic_add_rel_long             atomic_add_long
1016 #define atomic_subtract_acq_long        atomic_subtract_long
1017 #define atomic_subtract_rel_long        atomic_subtract_long
1018 #define atomic_clear_acq_long           atomic_clear_long
1019 #define atomic_clear_rel_long           atomic_clear_long
1020 #define atomic_set_acq_long             atomic_set_long
1021 #define atomic_set_rel_long             atomic_set_long
1022 #define atomic_cmpset_acq_long          atomic_cmpset_long
1023 #define atomic_cmpset_rel_long          atomic_cmpset_long
1024 #define atomic_load_acq_long            atomic_load_long
1025 #undef __with_interrupts_disabled
1026
1027 static __inline void
1028 atomic_add_long(volatile u_long *p, u_long v)
1029 {
1030
1031         atomic_add_32((volatile uint32_t *)p, v);
1032 }
1033
1034 static __inline void
1035 atomic_clear_long(volatile u_long *p, u_long v)
1036 {
1037
1038         atomic_clear_32((volatile uint32_t *)p, v);
1039 }
1040
1041 static __inline int
1042 atomic_cmpset_long(volatile u_long *dst, u_long old, u_long newe)
1043 {
1044
1045         return (atomic_cmpset_32((volatile uint32_t *)dst, old, newe));
1046 }
1047
1048 static __inline u_long
1049 atomic_fetchadd_long(volatile u_long *p, u_long v)
1050 {
1051
1052         return (atomic_fetchadd_32((volatile uint32_t *)p, v));
1053 }
1054
1055 static __inline void
1056 atomic_readandclear_long(volatile u_long *p)
1057 {
1058
1059         atomic_readandclear_32((volatile uint32_t *)p);
1060 }
1061
1062 static __inline void
1063 atomic_set_long(volatile u_long *p, u_long v)
1064 {
1065
1066         atomic_set_32((volatile uint32_t *)p, v);
1067 }
1068
1069 static __inline void
1070 atomic_subtract_long(volatile u_long *p, u_long v)
1071 {
1072
1073         atomic_subtract_32((volatile uint32_t *)p, v);
1074 }
1075
1076
1077
1078 #endif /* Arch >= v6 */
1079
1080 static __inline int
1081 atomic_load_32(volatile uint32_t *v)
1082 {
1083
1084         return (*v);
1085 }
1086
1087 static __inline void
1088 atomic_store_32(volatile uint32_t *dst, uint32_t src)
1089 {
1090         *dst = src;
1091 }
1092
1093 static __inline int
1094 atomic_load_long(volatile u_long *v)
1095 {
1096
1097         return (*v);
1098 }
1099
1100 static __inline void
1101 atomic_store_long(volatile u_long *dst, u_long src)
1102 {
1103         *dst = src;
1104 }
1105
1106 #define atomic_clear_ptr                atomic_clear_32
1107 #define atomic_set_ptr                  atomic_set_32
1108 #define atomic_cmpset_ptr               atomic_cmpset_32
1109 #define atomic_cmpset_rel_ptr           atomic_cmpset_rel_32
1110 #define atomic_cmpset_acq_ptr           atomic_cmpset_acq_32
1111 #define atomic_store_ptr                atomic_store_32
1112 #define atomic_store_rel_ptr            atomic_store_rel_32
1113
1114 #define atomic_add_int                  atomic_add_32
1115 #define atomic_add_acq_int              atomic_add_acq_32
1116 #define atomic_add_rel_int              atomic_add_rel_32
1117 #define atomic_subtract_int             atomic_subtract_32
1118 #define atomic_subtract_acq_int         atomic_subtract_acq_32
1119 #define atomic_subtract_rel_int         atomic_subtract_rel_32
1120 #define atomic_clear_int                atomic_clear_32
1121 #define atomic_clear_acq_int            atomic_clear_acq_32
1122 #define atomic_clear_rel_int            atomic_clear_rel_32
1123 #define atomic_set_int                  atomic_set_32
1124 #define atomic_set_acq_int              atomic_set_acq_32
1125 #define atomic_set_rel_int              atomic_set_rel_32
1126 #define atomic_cmpset_int               atomic_cmpset_32
1127 #define atomic_cmpset_acq_int           atomic_cmpset_acq_32
1128 #define atomic_cmpset_rel_int           atomic_cmpset_rel_32
1129 #define atomic_fetchadd_int             atomic_fetchadd_32
1130 #define atomic_readandclear_int         atomic_readandclear_32
1131 #define atomic_load_acq_int             atomic_load_acq_32
1132 #define atomic_store_rel_int            atomic_store_rel_32
1133
1134 #endif /* _MACHINE_ATOMIC_H_ */