sys/riscv/include/atomic.h

   1 /*-
   2  * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
   3  * All rights reserved.
   4  *
   5  * Portions of this software were developed by SRI International and the
   6  * University of Cambridge Computer Laboratory under DARPA/AFRL contract
   7  * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
   8  *
   9  * Portions of this software were developed by the University of Cambridge
  10  * Computer Laboratory as part of the CTSRD Project, with support from the
  11  * UK Higher Education Innovation Fund (HEIF).
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  * 1. Redistributions of source code must retain the above copyright
  17  *    notice, this list of conditions and the following disclaimer.
  18  * 2. Redistributions in binary form must reproduce the above copyright
  19  *    notice, this list of conditions and the following disclaimer in the
  20  *    documentation and/or other materials provided with the distribution.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  *
  34  * $FreeBSD$
  35  */
  36
  37 #ifndef _MACHINE_ATOMIC_H_
  38 #define _MACHINE_ATOMIC_H_
  39
  40 #include <sys/atomic_common.h>
  41
  42 #define fence() __asm __volatile("fence" ::: "memory");
  43 #define mb()    fence()
  44 #define rmb()   fence()
  45 #define wmb()   fence()
  46
  47 static __inline int atomic_cmpset_8(__volatile uint8_t *, uint8_t, uint8_t);
  48 static __inline int atomic_fcmpset_8(__volatile uint8_t *, uint8_t *, uint8_t);
  49 static __inline int atomic_cmpset_16(__volatile uint16_t *, uint16_t, uint16_t);
  50 static __inline int atomic_fcmpset_16(__volatile uint16_t *, uint16_t *,
  51     uint16_t);
  52
  53 #define ATOMIC_ACQ_REL(NAME, WIDTH)                                     \
  54 static __inline  void                                                   \
  55 atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\
  56 {                                                                       \
  57         atomic_##NAME##_##WIDTH(p, v);                                  \
  58         fence();                                                        \
  59 }                                                                       \
  60                                                                         \
  61 static __inline  void                                                   \
  62 atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\
  63 {                                                                       \
  64         fence();                                                        \
  65         atomic_##NAME##_##WIDTH(p, v);                                  \
  66 }
  67
  68 #define ATOMIC_CMPSET_ACQ_REL(WIDTH)                                    \
  69 static __inline  int                                                    \
  70 atomic_cmpset_acq_##WIDTH(__volatile uint##WIDTH##_t *p,                \
  71     uint##WIDTH##_t cmpval, uint##WIDTH##_t newval)                     \
  72 {                                                                       \
  73         int retval;                                                     \
  74                                                                         \
  75         retval = atomic_cmpset_##WIDTH(p, cmpval, newval);              \
  76         fence();                                                        \
  77         return (retval);                                                \
  78 }                                                                       \
  79                                                                         \
  80 static __inline  int                                                    \
  81 atomic_cmpset_rel_##WIDTH(__volatile uint##WIDTH##_t *p,                \
  82     uint##WIDTH##_t cmpval, uint##WIDTH##_t newval)                     \
  83 {                                                                       \
  84         fence();                                                        \
  85         return (atomic_cmpset_##WIDTH(p, cmpval, newval));              \
  86 }
  87
  88 #define ATOMIC_FCMPSET_ACQ_REL(WIDTH)                                   \
  89 static __inline  int                                                    \
  90 atomic_fcmpset_acq_##WIDTH(__volatile uint##WIDTH##_t *p,               \
  91     uint##WIDTH##_t *cmpval, uint##WIDTH##_t newval)                    \
  92 {                                                                       \
  93         int retval;                                                     \
  94                                                                         \
  95         retval = atomic_fcmpset_##WIDTH(p, cmpval, newval);             \
  96         fence();                                                        \
  97         return (retval);                                                \
  98 }                                                                       \
  99                                                                         \
 100 static __inline  int                                                    \
 101 atomic_fcmpset_rel_##WIDTH(__volatile uint##WIDTH##_t *p,               \
 102     uint##WIDTH##_t *cmpval, uint##WIDTH##_t newval)                    \
 103 {                                                                       \
 104         fence();                                                        \
 105         return (atomic_fcmpset_##WIDTH(p, cmpval, newval));             \
 106 }
 107
 108 ATOMIC_CMPSET_ACQ_REL(8);
 109 ATOMIC_FCMPSET_ACQ_REL(8);
 110 ATOMIC_CMPSET_ACQ_REL(16);
 111 ATOMIC_FCMPSET_ACQ_REL(16);
 112
 113 #define atomic_cmpset_char              atomic_cmpset_8
 114 #define atomic_cmpset_acq_char          atomic_cmpset_acq_8
 115 #define atomic_cmpset_rel_char          atomic_cmpset_rel_8
 116 #define atomic_fcmpset_char             atomic_fcmpset_8
 117 #define atomic_fcmpset_acq_char         atomic_fcmpset_acq_8
 118 #define atomic_fcmpset_rel_char         atomic_fcmpset_rel_8
 119
 120 #define atomic_cmpset_short             atomic_cmpset_16
 121 #define atomic_cmpset_acq_short         atomic_cmpset_acq_16
 122 #define atomic_cmpset_rel_short         atomic_cmpset_rel_16
 123 #define atomic_fcmpset_short            atomic_fcmpset_16
 124 #define atomic_fcmpset_acq_short        atomic_fcmpset_acq_16
 125 #define atomic_fcmpset_rel_short        atomic_fcmpset_rel_16
 126
 127 static __inline void
 128 atomic_add_32(volatile uint32_t *p, uint32_t val)
 129 {
 130
 131         __asm __volatile("amoadd.w zero, %1, %0"
 132                         : "+A" (*p)
 133                         : "r" (val)
 134                         : "memory");
 135 }
 136
 137 static __inline void
 138 atomic_subtract_32(volatile uint32_t *p, uint32_t val)
 139 {
 140
 141         __asm __volatile("amoadd.w zero, %1, %0"
 142                         : "+A" (*p)
 143                         : "r" (-val)
 144                         : "memory");
 145 }
 146
 147 static __inline void
 148 atomic_set_32(volatile uint32_t *p, uint32_t val)
 149 {
 150
 151         __asm __volatile("amoor.w zero, %1, %0"
 152                         : "+A" (*p)
 153                         : "r" (val)
 154                         : "memory");
 155 }
 156
 157 static __inline void
 158 atomic_clear_32(volatile uint32_t *p, uint32_t val)
 159 {
 160
 161         __asm __volatile("amoand.w zero, %1, %0"
 162                         : "+A" (*p)
 163                         : "r" (~val)
 164                         : "memory");
 165 }
 166
 167 static __inline int
 168 atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)
 169 {
 170         uint32_t tmp;
 171         int res;
 172
 173         res = 0;
 174
 175         __asm __volatile(
 176                 "0:"
 177                         "li   %1, 1\n" /* Preset to fail */
 178                         "lr.w %0, %2\n"
 179                         "bne  %0, %z3, 1f\n"
 180                         "sc.w %1, %z4, %2\n"
 181                         "bnez %1, 0b\n"
 182                 "1:"
 183                         : "=&r" (tmp), "=&r" (res), "+A" (*p)
 184                         : "rJ" ((long)(int32_t)cmpval), "rJ" (newval)
 185                         : "memory");
 186
 187         return (!res);
 188 }
 189
 190 static __inline int
 191 atomic_fcmpset_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval)
 192 {
 193         uint32_t tmp;
 194         int res;
 195
 196         res = 0;
 197
 198         __asm __volatile(
 199                 "0:"
 200                         "li   %1, 1\n"          /* Preset to fail */
 201                         "lr.w %0, %2\n"         /* Load old value */
 202                         "bne  %0, %z4, 1f\n"    /* Compare */
 203                         "sc.w %1, %z5, %2\n"    /* Try to store new value */
 204                         "j 2f\n"
 205                 "1:"
 206                         "sw   %0, %3\n"         /* Save old value */
 207                 "2:"
 208                         : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval)
 209                         : "rJ" ((long)(int32_t)*cmpval), "rJ" (newval)
 210                         : "memory");
 211
 212         return (!res);
 213 }
 214
 215 static __inline uint32_t
 216 atomic_fetchadd_32(volatile uint32_t *p, uint32_t val)
 217 {
 218         uint32_t ret;
 219
 220         __asm __volatile("amoadd.w %0, %2, %1"
 221                         : "=&r" (ret), "+A" (*p)
 222                         : "r" (val)
 223                         : "memory");
 224
 225         return (ret);
 226 }
 227
 228 static __inline uint32_t
 229 atomic_readandclear_32(volatile uint32_t *p)
 230 {
 231         uint32_t ret;
 232         uint32_t val;
 233
 234         val = 0;
 235
 236         __asm __volatile("amoswap.w %0, %2, %1"
 237                         : "=&r"(ret), "+A" (*p)
 238                         : "r" (val)
 239                         : "memory");
 240
 241         return (ret);
 242 }
 243
 244 #define atomic_add_int          atomic_add_32
 245 #define atomic_clear_int        atomic_clear_32
 246 #define atomic_cmpset_int       atomic_cmpset_32
 247 #define atomic_fcmpset_int      atomic_fcmpset_32
 248 #define atomic_fetchadd_int     atomic_fetchadd_32
 249 #define atomic_readandclear_int atomic_readandclear_32
 250 #define atomic_set_int          atomic_set_32
 251 #define atomic_subtract_int     atomic_subtract_32
 252
 253 ATOMIC_ACQ_REL(set, 32)
 254 ATOMIC_ACQ_REL(clear, 32)
 255 ATOMIC_ACQ_REL(add, 32)
 256 ATOMIC_ACQ_REL(subtract, 32)
 257
 258 ATOMIC_CMPSET_ACQ_REL(32);
 259 ATOMIC_FCMPSET_ACQ_REL(32);
 260
 261 static __inline uint32_t
 262 atomic_load_acq_32(volatile uint32_t *p)
 263 {
 264         uint32_t ret;
 265
 266         ret = *p;
 267
 268         fence();
 269
 270         return (ret);
 271 }
 272
 273 static __inline void
 274 atomic_store_rel_32(volatile uint32_t *p, uint32_t val)
 275 {
 276
 277         fence();
 278
 279         *p = val;
 280 }
 281
 282 #define atomic_add_acq_int      atomic_add_acq_32
 283 #define atomic_clear_acq_int    atomic_clear_acq_32
 284 #define atomic_cmpset_acq_int   atomic_cmpset_acq_32
 285 #define atomic_fcmpset_acq_int  atomic_fcmpset_acq_32
 286 #define atomic_load_acq_int     atomic_load_acq_32
 287 #define atomic_set_acq_int      atomic_set_acq_32
 288 #define atomic_subtract_acq_int atomic_subtract_acq_32
 289
 290 #define atomic_add_rel_int      atomic_add_rel_32
 291 #define atomic_clear_rel_int    atomic_clear_rel_32
 292 #define atomic_cmpset_rel_int   atomic_cmpset_rel_32
 293 #define atomic_fcmpset_rel_int  atomic_fcmpset_rel_32
 294 #define atomic_set_rel_int      atomic_set_rel_32
 295 #define atomic_subtract_rel_int atomic_subtract_rel_32
 296 #define atomic_store_rel_int    atomic_store_rel_32
 297
 298 static __inline void
 299 atomic_add_64(volatile uint64_t *p, uint64_t val)
 300 {
 301
 302         __asm __volatile("amoadd.d zero, %1, %0"
 303                         : "+A" (*p)
 304                         : "r" (val)
 305                         : "memory");
 306 }
 307
 308 static __inline void
 309 atomic_subtract_64(volatile uint64_t *p, uint64_t val)
 310 {
 311
 312         __asm __volatile("amoadd.d zero, %1, %0"
 313                         : "+A" (*p)
 314                         : "r" (-val)
 315                         : "memory");
 316 }
 317
 318 static __inline void
 319 atomic_set_64(volatile uint64_t *p, uint64_t val)
 320 {
 321
 322         __asm __volatile("amoor.d zero, %1, %0"
 323                         : "+A" (*p)
 324                         : "r" (val)
 325                         : "memory");
 326 }
 327
 328 static __inline void
 329 atomic_clear_64(volatile uint64_t *p, uint64_t val)
 330 {
 331
 332         __asm __volatile("amoand.d zero, %1, %0"
 333                         : "+A" (*p)
 334                         : "r" (~val)
 335                         : "memory");
 336 }
 337
 338 static __inline int
 339 atomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval)
 340 {
 341         uint64_t tmp;
 342         int res;
 343
 344         res = 0;
 345
 346         __asm __volatile(
 347                 "0:"
 348                         "li   %1, 1\n" /* Preset to fail */
 349                         "lr.d %0, %2\n"
 350                         "bne  %0, %z3, 1f\n"
 351                         "sc.d %1, %z4, %2\n"
 352                         "bnez %1, 0b\n"
 353                 "1:"
 354                         : "=&r" (tmp), "=&r" (res), "+A" (*p)
 355                         : "rJ" (cmpval), "rJ" (newval)
 356                         : "memory");
 357
 358         return (!res);
 359 }
 360
 361 static __inline int
 362 atomic_fcmpset_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval)
 363 {
 364         uint64_t tmp;
 365         int res;
 366
 367         res = 0;
 368
 369         __asm __volatile(
 370                 "0:"
 371                         "li   %1, 1\n"          /* Preset to fail */
 372                         "lr.d %0, %2\n"         /* Load old value */
 373                         "bne  %0, %z4, 1f\n"    /* Compare */
 374                         "sc.d %1, %z5, %2\n"    /* Try to store new value */
 375                         "j 2f\n"
 376                 "1:"
 377                         "sd   %0, %3\n"         /* Save old value */
 378                 "2:"
 379                         : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval)
 380                         : "rJ" (*cmpval), "rJ" (newval)
 381                         : "memory");
 382
 383         return (!res);
 384 }
 385
 386 static __inline uint64_t
 387 atomic_fetchadd_64(volatile uint64_t *p, uint64_t val)
 388 {
 389         uint64_t ret;
 390
 391         __asm __volatile("amoadd.d %0, %2, %1"
 392                         : "=&r" (ret), "+A" (*p)
 393                         : "r" (val)
 394                         : "memory");
 395
 396         return (ret);
 397 }
 398
 399 static __inline uint64_t
 400 atomic_readandclear_64(volatile uint64_t *p)
 401 {
 402         uint64_t ret;
 403         uint64_t val;
 404
 405         val = 0;
 406
 407         __asm __volatile("amoswap.d %0, %2, %1"
 408                         : "=&r"(ret), "+A" (*p)
 409                         : "r" (val)
 410                         : "memory");
 411
 412         return (ret);
 413 }
 414
 415 static __inline uint32_t
 416 atomic_swap_32(volatile uint32_t *p, uint32_t val)
 417 {
 418         uint32_t old;
 419
 420         __asm __volatile("amoswap.w %0, %2, %1"
 421                         : "=&r"(old), "+A" (*p)
 422                         : "r" (val)
 423                         : "memory");
 424
 425         return (old);
 426 }
 427
 428 static __inline uint64_t
 429 atomic_swap_64(volatile uint64_t *p, uint64_t val)
 430 {
 431         uint64_t old;
 432
 433         __asm __volatile("amoswap.d %0, %2, %1"
 434                         : "=&r"(old), "+A" (*p)
 435                         : "r" (val)
 436                         : "memory");
 437
 438         return (old);
 439 }
 440
 441 #define atomic_swap_int                 atomic_swap_32
 442
 443 #define atomic_add_long                 atomic_add_64
 444 #define atomic_clear_long               atomic_clear_64
 445 #define atomic_cmpset_long              atomic_cmpset_64
 446 #define atomic_fcmpset_long             atomic_fcmpset_64
 447 #define atomic_fetchadd_long            atomic_fetchadd_64
 448 #define atomic_readandclear_long        atomic_readandclear_64
 449 #define atomic_set_long                 atomic_set_64
 450 #define atomic_subtract_long            atomic_subtract_64
 451 #define atomic_swap_long                atomic_swap_64
 452
 453 #define atomic_add_ptr                  atomic_add_64
 454 #define atomic_clear_ptr                atomic_clear_64
 455 #define atomic_cmpset_ptr               atomic_cmpset_64
 456 #define atomic_fcmpset_ptr              atomic_fcmpset_64
 457 #define atomic_fetchadd_ptr             atomic_fetchadd_64
 458 #define atomic_readandclear_ptr         atomic_readandclear_64
 459 #define atomic_set_ptr                  atomic_set_64
 460 #define atomic_subtract_ptr             atomic_subtract_64
 461 #define atomic_swap_ptr                 atomic_swap_64
 462
 463 ATOMIC_ACQ_REL(set, 64)
 464 ATOMIC_ACQ_REL(clear, 64)
 465 ATOMIC_ACQ_REL(add, 64)
 466 ATOMIC_ACQ_REL(subtract, 64)
 467
 468 ATOMIC_CMPSET_ACQ_REL(64);
 469 ATOMIC_FCMPSET_ACQ_REL(64);
 470
 471 static __inline uint64_t
 472 atomic_load_acq_64(volatile uint64_t *p)
 473 {
 474         uint64_t ret;
 475
 476         ret = *p;
 477
 478         fence();
 479
 480         return (ret);
 481 }
 482
 483 static __inline void
 484 atomic_store_rel_64(volatile uint64_t *p, uint64_t val)
 485 {
 486
 487         fence();
 488
 489         *p = val;
 490 }
 491
 492 #define atomic_add_acq_long             atomic_add_acq_64
 493 #define atomic_clear_acq_long           atomic_clear_acq_64
 494 #define atomic_cmpset_acq_long          atomic_cmpset_acq_64
 495 #define atomic_fcmpset_acq_long         atomic_fcmpset_acq_64
 496 #define atomic_load_acq_long            atomic_load_acq_64
 497 #define atomic_set_acq_long             atomic_set_acq_64
 498 #define atomic_subtract_acq_long        atomic_subtract_acq_64
 499
 500 #define atomic_add_acq_ptr              atomic_add_acq_64
 501 #define atomic_clear_acq_ptr            atomic_clear_acq_64
 502 #define atomic_cmpset_acq_ptr           atomic_cmpset_acq_64
 503 #define atomic_fcmpset_acq_ptr          atomic_fcmpset_acq_64
 504 #define atomic_load_acq_ptr             atomic_load_acq_64
 505 #define atomic_set_acq_ptr              atomic_set_acq_64
 506 #define atomic_subtract_acq_ptr         atomic_subtract_acq_64
 507
 508 #undef ATOMIC_ACQ_REL
 509
 510 static __inline void
 511 atomic_thread_fence_acq(void)
 512 {
 513
 514         fence();
 515 }
 516
 517 static __inline void
 518 atomic_thread_fence_rel(void)
 519 {
 520
 521         fence();
 522 }
 523
 524 static __inline void
 525 atomic_thread_fence_acq_rel(void)
 526 {
 527
 528         fence();
 529 }
 530
 531 static __inline void
 532 atomic_thread_fence_seq_cst(void)
 533 {
 534
 535         fence();
 536 }
 537
 538 #define atomic_add_rel_long             atomic_add_rel_64
 539 #define atomic_clear_rel_long           atomic_clear_rel_64
 540
 541 #define atomic_add_rel_long             atomic_add_rel_64
 542 #define atomic_clear_rel_long           atomic_clear_rel_64
 543 #define atomic_cmpset_rel_long          atomic_cmpset_rel_64
 544 #define atomic_fcmpset_rel_long         atomic_fcmpset_rel_64
 545 #define atomic_set_rel_long             atomic_set_rel_64
 546 #define atomic_subtract_rel_long        atomic_subtract_rel_64
 547 #define atomic_store_rel_long           atomic_store_rel_64
 548
 549 #define atomic_add_rel_ptr              atomic_add_rel_64
 550 #define atomic_clear_rel_ptr            atomic_clear_rel_64
 551 #define atomic_cmpset_rel_ptr           atomic_cmpset_rel_64
 552 #define atomic_fcmpset_rel_ptr          atomic_fcmpset_rel_64
 553 #define atomic_set_rel_ptr              atomic_set_rel_64
 554 #define atomic_subtract_rel_ptr         atomic_subtract_rel_64
 555 #define atomic_store_rel_ptr            atomic_store_rel_64
 556
 557 #include <sys/_atomic_subword.h>
 558
 559 #endif /* _MACHINE_ATOMIC_H_ */