sys/riscv/include/atomic.h

   1 /*-
   2  * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
   3  * All rights reserved.
   4  *
   5  * Portions of this software were developed by SRI International and the
   6  * University of Cambridge Computer Laboratory under DARPA/AFRL contract
   7  * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
   8  *
   9  * Portions of this software were developed by the University of Cambridge
  10  * Computer Laboratory as part of the CTSRD Project, with support from the
  11  * UK Higher Education Innovation Fund (HEIF).
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  * 1. Redistributions of source code must retain the above copyright
  17  *    notice, this list of conditions and the following disclaimer.
  18  * 2. Redistributions in binary form must reproduce the above copyright
  19  *    notice, this list of conditions and the following disclaimer in the
  20  *    documentation and/or other materials provided with the distribution.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  *
  34  * $FreeBSD$
  35  */
  36
  37 #ifndef _MACHINE_ATOMIC_H_
  38 #define _MACHINE_ATOMIC_H_
  39
  40 #include <sys/atomic_common.h>
  41
  42 #define fence() __asm __volatile("fence" ::: "memory");
  43 #define mb()    fence()
  44 #define rmb()   fence()
  45 #define wmb()   fence()
  46
  47 static __inline int atomic_cmpset_8(__volatile uint8_t *, uint8_t, uint8_t);
  48 static __inline int atomic_fcmpset_8(__volatile uint8_t *, uint8_t *, uint8_t);
  49 static __inline int atomic_cmpset_16(__volatile uint16_t *, uint16_t, uint16_t);
  50 static __inline int atomic_fcmpset_16(__volatile uint16_t *, uint16_t *,
  51     uint16_t);
  52
  53 #define ATOMIC_ACQ_REL(NAME, WIDTH)                                     \
  54 static __inline  void                                                   \
  55 atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\
  56 {                                                                       \
  57         atomic_##NAME##_##WIDTH(p, v);                                  \
  58         fence();                                                        \
  59 }                                                                       \
  60                                                                         \
  61 static __inline  void                                                   \
  62 atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\
  63 {                                                                       \
  64         fence();                                                        \
  65         atomic_##NAME##_##WIDTH(p, v);                                  \
  66 }
  67
  68 #define ATOMIC_CMPSET_ACQ_REL(WIDTH)                                    \
  69 static __inline  int                                                    \
  70 atomic_cmpset_acq_##WIDTH(__volatile uint##WIDTH##_t *p,                \
  71     uint##WIDTH##_t cmpval, uint##WIDTH##_t newval)                     \
  72 {                                                                       \
  73         int retval;                                                     \
  74                                                                         \
  75         retval = atomic_cmpset_##WIDTH(p, cmpval, newval);              \
  76         fence();                                                        \
  77         return (retval);                                                \
  78 }                                                                       \
  79                                                                         \
  80 static __inline  int                                                    \
  81 atomic_cmpset_rel_##WIDTH(__volatile uint##WIDTH##_t *p,                \
  82     uint##WIDTH##_t cmpval, uint##WIDTH##_t newval)                     \
  83 {                                                                       \
  84         fence();                                                        \
  85         return (atomic_cmpset_##WIDTH(p, cmpval, newval));              \
  86 }
  87
  88 #define ATOMIC_FCMPSET_ACQ_REL(WIDTH)                                   \
  89 static __inline  int                                                    \
  90 atomic_fcmpset_acq_##WIDTH(__volatile uint##WIDTH##_t *p,               \
  91     uint##WIDTH##_t *cmpval, uint##WIDTH##_t newval)                    \
  92 {                                                                       \
  93         int retval;                                                     \
  94                                                                         \
  95         retval = atomic_fcmpset_##WIDTH(p, cmpval, newval);             \
  96         fence();                                                        \
  97         return (retval);                                                \
  98 }                                                                       \
  99                                                                         \
 100 static __inline  int                                                    \
 101 atomic_fcmpset_rel_##WIDTH(__volatile uint##WIDTH##_t *p,               \
 102     uint##WIDTH##_t *cmpval, uint##WIDTH##_t newval)                    \
 103 {                                                                       \
 104         fence();                                                        \
 105         return (atomic_fcmpset_##WIDTH(p, cmpval, newval));             \
 106 }
 107
 108 ATOMIC_CMPSET_ACQ_REL(8);
 109 ATOMIC_FCMPSET_ACQ_REL(8);
 110 ATOMIC_CMPSET_ACQ_REL(16);
 111 ATOMIC_FCMPSET_ACQ_REL(16);
 112
 113 #define atomic_cmpset_char              atomic_cmpset_8
 114 #define atomic_cmpset_acq_char          atomic_cmpset_acq_8
 115 #define atomic_cmpset_rel_char          atomic_cmpset_rel_8
 116 #define atomic_fcmpset_char             atomic_fcmpset_8
 117 #define atomic_fcmpset_acq_char         atomic_fcmpset_acq_8
 118 #define atomic_fcmpset_rel_char         atomic_fcmpset_rel_8
 119
 120
 121 #define atomic_cmpset_short             atomic_cmpset_16
 122 #define atomic_cmpset_acq_short         atomic_cmpset_acq_16
 123 #define atomic_cmpset_rel_short         atomic_cmpset_rel_16
 124 #define atomic_fcmpset_short            atomic_fcmpset_16
 125 #define atomic_fcmpset_acq_short        atomic_fcmpset_acq_16
 126 #define atomic_fcmpset_rel_short        atomic_fcmpset_rel_16
 127
 128 static __inline void
 129 atomic_add_32(volatile uint32_t *p, uint32_t val)
 130 {
 131
 132         __asm __volatile("amoadd.w zero, %1, %0"
 133                         : "+A" (*p)
 134                         : "r" (val)
 135                         : "memory");
 136 }
 137
 138 static __inline void
 139 atomic_subtract_32(volatile uint32_t *p, uint32_t val)
 140 {
 141
 142         __asm __volatile("amoadd.w zero, %1, %0"
 143                         : "+A" (*p)
 144                         : "r" (-val)
 145                         : "memory");
 146 }
 147
 148 static __inline void
 149 atomic_set_32(volatile uint32_t *p, uint32_t val)
 150 {
 151
 152         __asm __volatile("amoor.w zero, %1, %0"
 153                         : "+A" (*p)
 154                         : "r" (val)
 155                         : "memory");
 156 }
 157
 158 static __inline void
 159 atomic_clear_32(volatile uint32_t *p, uint32_t val)
 160 {
 161
 162         __asm __volatile("amoand.w zero, %1, %0"
 163                         : "+A" (*p)
 164                         : "r" (~val)
 165                         : "memory");
 166 }
 167
 168 static __inline int
 169 atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)
 170 {
 171         uint32_t tmp;
 172         int res;
 173
 174         res = 0;
 175
 176         __asm __volatile(
 177                 "0:"
 178                         "li   %1, 1\n" /* Preset to fail */
 179                         "lr.w %0, %2\n"
 180                         "bne  %0, %z3, 1f\n"
 181                         "sc.w %1, %z4, %2\n"
 182                         "bnez %1, 0b\n"
 183                 "1:"
 184                         : "=&r" (tmp), "=&r" (res), "+A" (*p)
 185                         : "rJ" ((long)(int32_t)cmpval), "rJ" (newval)
 186                         : "memory");
 187
 188         return (!res);
 189 }
 190
 191 static __inline int
 192 atomic_fcmpset_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval)
 193 {
 194         uint32_t tmp;
 195         int res;
 196
 197         res = 0;
 198
 199         __asm __volatile(
 200                 "0:"
 201                         "li   %1, 1\n"          /* Preset to fail */
 202                         "lr.w %0, %2\n"         /* Load old value */
 203                         "bne  %0, %z4, 1f\n"    /* Compare */
 204                         "sc.w %1, %z5, %2\n"    /* Try to store new value */
 205                         "j 2f\n"
 206                 "1:"
 207                         "sw   %0, %3\n"         /* Save old value */
 208                 "2:"
 209                         : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval)
 210                         : "rJ" ((long)(int32_t)*cmpval), "rJ" (newval)
 211                         : "memory");
 212
 213         return (!res);
 214 }
 215
 216 static __inline uint32_t
 217 atomic_fetchadd_32(volatile uint32_t *p, uint32_t val)
 218 {
 219         uint32_t ret;
 220
 221         __asm __volatile("amoadd.w %0, %2, %1"
 222                         : "=&r" (ret), "+A" (*p)
 223                         : "r" (val)
 224                         : "memory");
 225
 226         return (ret);
 227 }
 228
 229 static __inline uint32_t
 230 atomic_readandclear_32(volatile uint32_t *p)
 231 {
 232         uint32_t ret;
 233         uint32_t val;
 234
 235         val = 0;
 236
 237         __asm __volatile("amoswap.w %0, %2, %1"
 238                         : "=&r"(ret), "+A" (*p)
 239                         : "r" (val)
 240                         : "memory");
 241
 242         return (ret);
 243 }
 244
 245 #define atomic_add_int          atomic_add_32
 246 #define atomic_clear_int        atomic_clear_32
 247 #define atomic_cmpset_int       atomic_cmpset_32
 248 #define atomic_fcmpset_int      atomic_fcmpset_32
 249 #define atomic_fetchadd_int     atomic_fetchadd_32
 250 #define atomic_readandclear_int atomic_readandclear_32
 251 #define atomic_set_int          atomic_set_32
 252 #define atomic_subtract_int     atomic_subtract_32
 253
 254 ATOMIC_ACQ_REL(set, 32)
 255 ATOMIC_ACQ_REL(clear, 32)
 256 ATOMIC_ACQ_REL(add, 32)
 257 ATOMIC_ACQ_REL(subtract, 32)
 258
 259 ATOMIC_CMPSET_ACQ_REL(32);
 260 ATOMIC_FCMPSET_ACQ_REL(32);
 261
 262 static __inline uint32_t
 263 atomic_load_acq_32(volatile uint32_t *p)
 264 {
 265         uint32_t ret;
 266
 267         ret = *p;
 268
 269         fence();
 270
 271         return (ret);
 272 }
 273
 274 static __inline void
 275 atomic_store_rel_32(volatile uint32_t *p, uint32_t val)
 276 {
 277
 278         fence();
 279
 280         *p = val;
 281 }
 282
 283 #define atomic_add_acq_int      atomic_add_acq_32
 284 #define atomic_clear_acq_int    atomic_clear_acq_32
 285 #define atomic_cmpset_acq_int   atomic_cmpset_acq_32
 286 #define atomic_fcmpset_acq_int  atomic_fcmpset_acq_32
 287 #define atomic_load_acq_int     atomic_load_acq_32
 288 #define atomic_set_acq_int      atomic_set_acq_32
 289 #define atomic_subtract_acq_int atomic_subtract_acq_32
 290
 291 #define atomic_add_rel_int      atomic_add_rel_32
 292 #define atomic_clear_rel_int    atomic_add_rel_32
 293 #define atomic_cmpset_rel_int   atomic_cmpset_rel_32
 294 #define atomic_fcmpset_rel_int  atomic_fcmpset_rel_32
 295 #define atomic_set_rel_int      atomic_set_rel_32
 296 #define atomic_subtract_rel_int atomic_subtract_rel_32
 297 #define atomic_store_rel_int    atomic_store_rel_32
 298
 299 static __inline void
 300 atomic_add_64(volatile uint64_t *p, uint64_t val)
 301 {
 302
 303         __asm __volatile("amoadd.d zero, %1, %0"
 304                         : "+A" (*p)
 305                         : "r" (val)
 306                         : "memory");
 307 }
 308
 309 static __inline void
 310 atomic_subtract_64(volatile uint64_t *p, uint64_t val)
 311 {
 312
 313         __asm __volatile("amoadd.d zero, %1, %0"
 314                         : "+A" (*p)
 315                         : "r" (-val)
 316                         : "memory");
 317 }
 318
 319 static __inline void
 320 atomic_set_64(volatile uint64_t *p, uint64_t val)
 321 {
 322
 323         __asm __volatile("amoor.d zero, %1, %0"
 324                         : "+A" (*p)
 325                         : "r" (val)
 326                         : "memory");
 327 }
 328
 329 static __inline void
 330 atomic_clear_64(volatile uint64_t *p, uint64_t val)
 331 {
 332
 333         __asm __volatile("amoand.d zero, %1, %0"
 334                         : "+A" (*p)
 335                         : "r" (~val)
 336                         : "memory");
 337 }
 338
 339 static __inline int
 340 atomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval)
 341 {
 342         uint64_t tmp;
 343         int res;
 344
 345         res = 0;
 346
 347         __asm __volatile(
 348                 "0:"
 349                         "li   %1, 1\n" /* Preset to fail */
 350                         "lr.d %0, %2\n"
 351                         "bne  %0, %z3, 1f\n"
 352                         "sc.d %1, %z4, %2\n"
 353                         "bnez %1, 0b\n"
 354                 "1:"
 355                         : "=&r" (tmp), "=&r" (res), "+A" (*p)
 356                         : "rJ" (cmpval), "rJ" (newval)
 357                         : "memory");
 358
 359         return (!res);
 360 }
 361
 362 static __inline int
 363 atomic_fcmpset_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval)
 364 {
 365         uint64_t tmp;
 366         int res;
 367
 368         res = 0;
 369
 370         __asm __volatile(
 371                 "0:"
 372                         "li   %1, 1\n"          /* Preset to fail */
 373                         "lr.d %0, %2\n"         /* Load old value */
 374                         "bne  %0, %z4, 1f\n"    /* Compare */
 375                         "sc.d %1, %z5, %2\n"    /* Try to store new value */
 376                         "j 2f\n"
 377                 "1:"
 378                         "sd   %0, %3\n"         /* Save old value */
 379                 "2:"
 380                         : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval)
 381                         : "rJ" (*cmpval), "rJ" (newval)
 382                         : "memory");
 383
 384         return (!res);
 385 }
 386
 387 static __inline uint64_t
 388 atomic_fetchadd_64(volatile uint64_t *p, uint64_t val)
 389 {
 390         uint64_t ret;
 391
 392         __asm __volatile("amoadd.d %0, %2, %1"
 393                         : "=&r" (ret), "+A" (*p)
 394                         : "r" (val)
 395                         : "memory");
 396
 397         return (ret);
 398 }
 399
 400 static __inline uint64_t
 401 atomic_readandclear_64(volatile uint64_t *p)
 402 {
 403         uint64_t ret;
 404         uint64_t val;
 405
 406         val = 0;
 407
 408         __asm __volatile("amoswap.d %0, %2, %1"
 409                         : "=&r"(ret), "+A" (*p)
 410                         : "r" (val)
 411                         : "memory");
 412
 413         return (ret);
 414 }
 415
 416 static __inline uint32_t
 417 atomic_swap_32(volatile uint32_t *p, uint32_t val)
 418 {
 419         uint32_t old;
 420
 421         __asm __volatile("amoswap.w %0, %2, %1"
 422                         : "=&r"(old), "+A" (*p)
 423                         : "r" (val)
 424                         : "memory");
 425
 426         return (old);
 427 }
 428
 429 static __inline uint64_t
 430 atomic_swap_64(volatile uint64_t *p, uint64_t val)
 431 {
 432         uint64_t old;
 433
 434         __asm __volatile("amoswap.d %0, %2, %1"
 435                         : "=&r"(old), "+A" (*p)
 436                         : "r" (val)
 437                         : "memory");
 438
 439         return (old);
 440 }
 441
 442 #define atomic_swap_int                 atomic_swap_32
 443
 444 #define atomic_add_long                 atomic_add_64
 445 #define atomic_clear_long               atomic_clear_64
 446 #define atomic_cmpset_long              atomic_cmpset_64
 447 #define atomic_fcmpset_long             atomic_fcmpset_64
 448 #define atomic_fetchadd_long            atomic_fetchadd_64
 449 #define atomic_readandclear_long        atomic_readandclear_64
 450 #define atomic_set_long                 atomic_set_64
 451 #define atomic_subtract_long            atomic_subtract_64
 452 #define atomic_swap_long                atomic_swap_64
 453
 454 #define atomic_add_ptr                  atomic_add_64
 455 #define atomic_clear_ptr                atomic_clear_64
 456 #define atomic_cmpset_ptr               atomic_cmpset_64
 457 #define atomic_fcmpset_ptr              atomic_fcmpset_64
 458 #define atomic_fetchadd_ptr             atomic_fetchadd_64
 459 #define atomic_readandclear_ptr         atomic_readandclear_64
 460 #define atomic_set_ptr                  atomic_set_64
 461 #define atomic_subtract_ptr             atomic_subtract_64
 462 #define atomic_swap_ptr                 atomic_swap_64
 463
 464 ATOMIC_ACQ_REL(set, 64)
 465 ATOMIC_ACQ_REL(clear, 64)
 466 ATOMIC_ACQ_REL(add, 64)
 467 ATOMIC_ACQ_REL(subtract, 64)
 468
 469 ATOMIC_CMPSET_ACQ_REL(64);
 470 ATOMIC_FCMPSET_ACQ_REL(64);
 471
 472 static __inline uint64_t
 473 atomic_load_acq_64(volatile uint64_t *p)
 474 {
 475         uint64_t ret;
 476
 477         ret = *p;
 478
 479         fence();
 480
 481         return (ret);
 482 }
 483
 484 static __inline void
 485 atomic_store_rel_64(volatile uint64_t *p, uint64_t val)
 486 {
 487
 488         fence();
 489
 490         *p = val;
 491 }
 492
 493 #define atomic_add_acq_long             atomic_add_acq_64
 494 #define atomic_clear_acq_long           atomic_add_acq_64
 495 #define atomic_cmpset_acq_long          atomic_cmpset_acq_64
 496 #define atomic_fcmpset_acq_long         atomic_fcmpset_acq_64
 497 #define atomic_load_acq_long            atomic_load_acq_64
 498 #define atomic_set_acq_long             atomic_set_acq_64
 499 #define atomic_subtract_acq_long        atomic_subtract_acq_64
 500
 501 #define atomic_add_acq_ptr              atomic_add_acq_64
 502 #define atomic_clear_acq_ptr            atomic_add_acq_64
 503 #define atomic_cmpset_acq_ptr           atomic_cmpset_acq_64
 504 #define atomic_fcmpset_acq_ptr          atomic_fcmpset_acq_64
 505 #define atomic_load_acq_ptr             atomic_load_acq_64
 506 #define atomic_set_acq_ptr              atomic_set_acq_64
 507 #define atomic_subtract_acq_ptr         atomic_subtract_acq_64
 508
 509 #undef ATOMIC_ACQ_REL
 510
 511 static __inline void
 512 atomic_thread_fence_acq(void)
 513 {
 514
 515         fence();
 516 }
 517
 518 static __inline void
 519 atomic_thread_fence_rel(void)
 520 {
 521
 522         fence();
 523 }
 524
 525 static __inline void
 526 atomic_thread_fence_acq_rel(void)
 527 {
 528
 529         fence();
 530 }
 531
 532 static __inline void
 533 atomic_thread_fence_seq_cst(void)
 534 {
 535
 536         fence();
 537 }
 538
 539 #define atomic_add_rel_long             atomic_add_rel_64
 540 #define atomic_clear_rel_long           atomic_clear_rel_64
 541
 542 #define atomic_add_rel_long             atomic_add_rel_64
 543 #define atomic_clear_rel_long           atomic_clear_rel_64
 544 #define atomic_cmpset_rel_long          atomic_cmpset_rel_64
 545 #define atomic_fcmpset_rel_long         atomic_fcmpset_rel_64
 546 #define atomic_set_rel_long             atomic_set_rel_64
 547 #define atomic_subtract_rel_long        atomic_subtract_rel_64
 548 #define atomic_store_rel_long           atomic_store_rel_64
 549
 550 #define atomic_add_rel_ptr              atomic_add_rel_64
 551 #define atomic_clear_rel_ptr            atomic_clear_rel_64
 552 #define atomic_cmpset_rel_ptr           atomic_cmpset_rel_64
 553 #define atomic_fcmpset_rel_ptr          atomic_fcmpset_rel_64
 554 #define atomic_set_rel_ptr              atomic_set_rel_64
 555 #define atomic_subtract_rel_ptr         atomic_subtract_rel_64
 556 #define atomic_store_rel_ptr            atomic_store_rel_64
 557
 558 #include <sys/_atomic_subword.h>
 559
 560 #endif /* _MACHINE_ATOMIC_H_ */