lib/libspl/atomic.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License, Version 1.0 only
   6  * (the "License").  You may not use this file except in compliance
   7  * with the License.
   8  *
   9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  10  * or https://opensource.org/licenses/CDDL-1.0.
  11  * See the License for the specific language governing permissions
  12  * and limitations under the License.
  13  *
  14  * When distributing Covered Code, include this CDDL HEADER in each
  15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  16  * If applicable, add the following below this CDDL HEADER, with the
  17  * fields enclosed by brackets "[]" replaced with your own identifying
  18  * information: Portions Copyright [yyyy] [name of copyright owner]
  19  *
  20  * CDDL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2009 by Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26
  27 #include <atomic.h>
  28
  29 /*
  30  * These are the void returning variants
  31  */
  32 #define ATOMIC_INC(name, type) \
  33         void atomic_inc_##name(volatile type *target)                   \
  34         {                                                               \
  35                 (void) __atomic_add_fetch(target, 1, __ATOMIC_SEQ_CST); \
  36         }
  37
  38 /* BEGIN CSTYLED */
  39 ATOMIC_INC(8, uint8_t)
  40 ATOMIC_INC(16, uint16_t)
  41 ATOMIC_INC(32, uint32_t)
  42 ATOMIC_INC(64, uint64_t)
  43 ATOMIC_INC(uchar, uchar_t)
  44 ATOMIC_INC(ushort, ushort_t)
  45 ATOMIC_INC(uint, uint_t)
  46 ATOMIC_INC(ulong, ulong_t)
  47 /* END CSTYLED */
  48
  49
  50 #define ATOMIC_DEC(name, type) \
  51         void atomic_dec_##name(volatile type *target)                   \
  52         {                                                               \
  53                 (void) __atomic_sub_fetch(target, 1, __ATOMIC_SEQ_CST); \
  54         }
  55
  56 /* BEGIN CSTYLED */
  57 ATOMIC_DEC(8, uint8_t)
  58 ATOMIC_DEC(16, uint16_t)
  59 ATOMIC_DEC(32, uint32_t)
  60 ATOMIC_DEC(64, uint64_t)
  61 ATOMIC_DEC(uchar, uchar_t)
  62 ATOMIC_DEC(ushort, ushort_t)
  63 ATOMIC_DEC(uint, uint_t)
  64 ATOMIC_DEC(ulong, ulong_t)
  65 /* END CSTYLED */
  66
  67
  68 #define ATOMIC_ADD(name, type1, type2) \
  69         void atomic_add_##name(volatile type1 *target, type2 bits)      \
  70         {                                                               \
  71                 (void) __atomic_add_fetch(target, bits, __ATOMIC_SEQ_CST); \
  72         }
  73
  74 void
  75 atomic_add_ptr(volatile void *target, ssize_t bits)
  76 {
  77         (void) __atomic_add_fetch((void **)target, bits, __ATOMIC_SEQ_CST);
  78 }
  79
  80 /* BEGIN CSTYLED */
  81 ATOMIC_ADD(8, uint8_t, int8_t)
  82 ATOMIC_ADD(16, uint16_t, int16_t)
  83 ATOMIC_ADD(32, uint32_t, int32_t)
  84 ATOMIC_ADD(64, uint64_t, int64_t)
  85 ATOMIC_ADD(char, uchar_t, signed char)
  86 ATOMIC_ADD(short, ushort_t, short)
  87 ATOMIC_ADD(int, uint_t, int)
  88 ATOMIC_ADD(long, ulong_t, long)
  89 /* END CSTYLED */
  90
  91
  92 #define ATOMIC_SUB(name, type1, type2) \
  93         void atomic_sub_##name(volatile type1 *target, type2 bits)      \
  94         {                                                               \
  95                 (void) __atomic_sub_fetch(target, bits, __ATOMIC_SEQ_CST); \
  96         }
  97
  98 void
  99 atomic_sub_ptr(volatile void *target, ssize_t bits)
 100 {
 101         (void) __atomic_sub_fetch((void **)target, bits, __ATOMIC_SEQ_CST);
 102 }
 103
 104 /* BEGIN CSTYLED */
 105 ATOMIC_SUB(8, uint8_t, int8_t)
 106 ATOMIC_SUB(16, uint16_t, int16_t)
 107 ATOMIC_SUB(32, uint32_t, int32_t)
 108 ATOMIC_SUB(64, uint64_t, int64_t)
 109 ATOMIC_SUB(char, uchar_t, signed char)
 110 ATOMIC_SUB(short, ushort_t, short)
 111 ATOMIC_SUB(int, uint_t, int)
 112 ATOMIC_SUB(long, ulong_t, long)
 113 /* END CSTYLED */
 114
 115
 116 #define ATOMIC_OR(name, type) \
 117         void atomic_or_##name(volatile type *target, type bits)         \
 118         {                                                               \
 119                 (void) __atomic_or_fetch(target, bits, __ATOMIC_SEQ_CST); \
 120         }
 121
 122 /* BEGIN CSTYLED */
 123 ATOMIC_OR(8, uint8_t)
 124 ATOMIC_OR(16, uint16_t)
 125 ATOMIC_OR(32, uint32_t)
 126 ATOMIC_OR(64, uint64_t)
 127 ATOMIC_OR(uchar, uchar_t)
 128 ATOMIC_OR(ushort, ushort_t)
 129 ATOMIC_OR(uint, uint_t)
 130 ATOMIC_OR(ulong, ulong_t)
 131 /* END CSTYLED */
 132
 133
 134 #define ATOMIC_AND(name, type) \
 135         void atomic_and_##name(volatile type *target, type bits)        \
 136         {                                                               \
 137                 (void) __atomic_and_fetch(target, bits, __ATOMIC_SEQ_CST); \
 138         }
 139
 140 /* BEGIN CSTYLED */
 141 ATOMIC_AND(8, uint8_t)
 142 ATOMIC_AND(16, uint16_t)
 143 ATOMIC_AND(32, uint32_t)
 144 ATOMIC_AND(64, uint64_t)
 145 ATOMIC_AND(uchar, uchar_t)
 146 ATOMIC_AND(ushort, ushort_t)
 147 ATOMIC_AND(uint, uint_t)
 148 ATOMIC_AND(ulong, ulong_t)
 149 /* END CSTYLED */
 150
 151
 152 /*
 153  * New value returning variants
 154  */
 155
 156 #define ATOMIC_INC_NV(name, type) \
 157         type atomic_inc_##name##_nv(volatile type *target)              \
 158         {                                                               \
 159                 return (__atomic_add_fetch(target, 1, __ATOMIC_SEQ_CST)); \
 160         }
 161
 162 /* BEGIN CSTYLED */
 163 ATOMIC_INC_NV(8, uint8_t)
 164 ATOMIC_INC_NV(16, uint16_t)
 165 ATOMIC_INC_NV(32, uint32_t)
 166 ATOMIC_INC_NV(64, uint64_t)
 167 ATOMIC_INC_NV(uchar, uchar_t)
 168 ATOMIC_INC_NV(ushort, ushort_t)
 169 ATOMIC_INC_NV(uint, uint_t)
 170 ATOMIC_INC_NV(ulong, ulong_t)
 171 /* END CSTYLED */
 172
 173
 174 #define ATOMIC_DEC_NV(name, type) \
 175         type atomic_dec_##name##_nv(volatile type *target)              \
 176         {                                                               \
 177                 return (__atomic_sub_fetch(target, 1, __ATOMIC_SEQ_CST)); \
 178         }
 179
 180 /* BEGIN CSTYLED */
 181 ATOMIC_DEC_NV(8, uint8_t)
 182 ATOMIC_DEC_NV(16, uint16_t)
 183 ATOMIC_DEC_NV(32, uint32_t)
 184 ATOMIC_DEC_NV(64, uint64_t)
 185 ATOMIC_DEC_NV(uchar, uchar_t)
 186 ATOMIC_DEC_NV(ushort, ushort_t)
 187 ATOMIC_DEC_NV(uint, uint_t)
 188 ATOMIC_DEC_NV(ulong, ulong_t)
 189 /* END CSTYLED */
 190
 191
 192 #define ATOMIC_ADD_NV(name, type1, type2) \
 193         type1 atomic_add_##name##_nv(volatile type1 *target, type2 bits) \
 194         {                                                               \
 195                 return (__atomic_add_fetch(target, bits, __ATOMIC_SEQ_CST)); \
 196         }
 197
 198 void *
 199 atomic_add_ptr_nv(volatile void *target, ssize_t bits)
 200 {
 201         return (__atomic_add_fetch((void **)target, bits, __ATOMIC_SEQ_CST));
 202 }
 203
 204 /* BEGIN CSTYLED */
 205 ATOMIC_ADD_NV(8, uint8_t, int8_t)
 206 ATOMIC_ADD_NV(16, uint16_t, int16_t)
 207 ATOMIC_ADD_NV(32, uint32_t, int32_t)
 208 ATOMIC_ADD_NV(64, uint64_t, int64_t)
 209 ATOMIC_ADD_NV(char, uchar_t, signed char)
 210 ATOMIC_ADD_NV(short, ushort_t, short)
 211 ATOMIC_ADD_NV(int, uint_t, int)
 212 ATOMIC_ADD_NV(long, ulong_t, long)
 213 /* END CSTYLED */
 214
 215
 216 #define ATOMIC_SUB_NV(name, type1, type2) \
 217         type1 atomic_sub_##name##_nv(volatile type1 *target, type2 bits) \
 218         {                                                               \
 219                 return (__atomic_sub_fetch(target, bits, __ATOMIC_SEQ_CST)); \
 220         }
 221
 222 void *
 223 atomic_sub_ptr_nv(volatile void *target, ssize_t bits)
 224 {
 225         return (__atomic_sub_fetch((void **)target, bits, __ATOMIC_SEQ_CST));
 226 }
 227
 228 /* BEGIN CSTYLED */
 229 ATOMIC_SUB_NV(8, uint8_t, int8_t)
 230 ATOMIC_SUB_NV(char, uchar_t, signed char)
 231 ATOMIC_SUB_NV(16, uint16_t, int16_t)
 232 ATOMIC_SUB_NV(short, ushort_t, short)
 233 ATOMIC_SUB_NV(32, uint32_t, int32_t)
 234 ATOMIC_SUB_NV(int, uint_t, int)
 235 ATOMIC_SUB_NV(long, ulong_t, long)
 236 ATOMIC_SUB_NV(64, uint64_t, int64_t)
 237 /* END CSTYLED */
 238
 239
 240 #define ATOMIC_OR_NV(name, type) \
 241         type atomic_or_##name##_nv(volatile type *target, type bits)    \
 242         {                                                               \
 243                 return (__atomic_or_fetch(target, bits, __ATOMIC_SEQ_CST)); \
 244         }
 245
 246 /* BEGIN CSTYLED */
 247 ATOMIC_OR_NV(8, uint8_t)
 248 ATOMIC_OR_NV(16, uint16_t)
 249 ATOMIC_OR_NV(32, uint32_t)
 250 ATOMIC_OR_NV(64, uint64_t)
 251 ATOMIC_OR_NV(uchar, uchar_t)
 252 ATOMIC_OR_NV(ushort, ushort_t)
 253 ATOMIC_OR_NV(uint, uint_t)
 254 ATOMIC_OR_NV(ulong, ulong_t)
 255 /* END CSTYLED */
 256
 257
 258 #define ATOMIC_AND_NV(name, type) \
 259         type atomic_and_##name##_nv(volatile type *target, type bits)   \
 260         {                                                               \
 261                 return (__atomic_and_fetch(target, bits, __ATOMIC_SEQ_CST)); \
 262         }
 263
 264 /* BEGIN CSTYLED */
 265 ATOMIC_AND_NV(8, uint8_t)
 266 ATOMIC_AND_NV(16, uint16_t)
 267 ATOMIC_AND_NV(32, uint32_t)
 268 ATOMIC_AND_NV(64, uint64_t)
 269 ATOMIC_AND_NV(uchar, uchar_t)
 270 ATOMIC_AND_NV(ushort, ushort_t)
 271 ATOMIC_AND_NV(uint, uint_t)
 272 ATOMIC_AND_NV(ulong, ulong_t)
 273 /* END CSTYLED */
 274
 275
 276 /*
 277  * If *tgt == exp, set *tgt = des; return old value
 278  *
 279  * This may not look right on the first pass (or the sixteenth), but,
 280  * from https://gcc.gnu.org/onlinedocs/gcc/_005f_005fatomic-Builtins.html:
 281  * > If they are not equal, the operation is a read
 282  * > and the current contents of *ptr are written into *expected.
 283  * And, in the converse case, exp is already *target by definition.
 284  */
 285
 286 #define ATOMIC_CAS(name, type) \
 287         type atomic_cas_##name(volatile type *target, type exp, type des) \
 288         {                                                               \
 289                 __atomic_compare_exchange_n(target, &exp, des, B_FALSE, \
 290                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);                \
 291                 return (exp);                                           \
 292         }
 293
 294 void *
 295 atomic_cas_ptr(volatile void *target, void *exp, void *des)
 296 {
 297
 298         __atomic_compare_exchange_n((void **)target, &exp, des, B_FALSE,
 299             __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
 300         return (exp);
 301 }
 302
 303 /* BEGIN CSTYLED */
 304 ATOMIC_CAS(8, uint8_t)
 305 ATOMIC_CAS(16, uint16_t)
 306 ATOMIC_CAS(32, uint32_t)
 307 ATOMIC_CAS(64, uint64_t)
 308 ATOMIC_CAS(uchar, uchar_t)
 309 ATOMIC_CAS(ushort, ushort_t)
 310 ATOMIC_CAS(uint, uint_t)
 311 ATOMIC_CAS(ulong, ulong_t)
 312 /* END CSTYLED */
 313
 314
 315 /*
 316  * Swap target and return old value
 317  */
 318
 319 #define ATOMIC_SWAP(name, type) \
 320         type atomic_swap_##name(volatile type *target, type bits)       \
 321         {                                                               \
 322                 return (__atomic_exchange_n(target, bits, __ATOMIC_SEQ_CST)); \
 323         }
 324
 325 /* BEGIN CSTYLED */
 326 ATOMIC_SWAP(8, uint8_t)
 327 ATOMIC_SWAP(16, uint16_t)
 328 ATOMIC_SWAP(32, uint32_t)
 329 ATOMIC_SWAP(64, uint64_t)
 330 ATOMIC_SWAP(uchar, uchar_t)
 331 ATOMIC_SWAP(ushort, ushort_t)
 332 ATOMIC_SWAP(uint, uint_t)
 333 ATOMIC_SWAP(ulong, ulong_t)
 334 /* END CSTYLED */
 335
 336 void *
 337 atomic_swap_ptr(volatile void *target, void *bits)
 338 {
 339         return (__atomic_exchange_n((void **)target, bits, __ATOMIC_SEQ_CST));
 340 }
 341
 342 #ifndef _LP64
 343 uint64_t
 344 atomic_load_64(volatile uint64_t *target)
 345 {
 346         return (__atomic_load_n(target, __ATOMIC_RELAXED));
 347 }
 348
 349 void
 350 atomic_store_64(volatile uint64_t *target, uint64_t bits)
 351 {
 352         return (__atomic_store_n(target, bits, __ATOMIC_RELAXED));
 353 }
 354 #endif
 355
 356 int
 357 atomic_set_long_excl(volatile ulong_t *target, uint_t value)
 358 {
 359         ulong_t bit = 1UL << value;
 360         ulong_t old = __atomic_fetch_or(target, bit, __ATOMIC_SEQ_CST);
 361         return ((old & bit) ? -1 : 0);
 362 }
 363
 364 int
 365 atomic_clear_long_excl(volatile ulong_t *target, uint_t value)
 366 {
 367         ulong_t bit = 1UL << value;
 368         ulong_t old = __atomic_fetch_and(target, ~bit, __ATOMIC_SEQ_CST);
 369         return ((old & bit) ? 0 : -1);
 370 }
 371
 372 void
 373 membar_enter(void)
 374 {
 375         __atomic_thread_fence(__ATOMIC_SEQ_CST);
 376 }
 377
 378 void
 379 membar_exit(void)
 380 {
 381         __atomic_thread_fence(__ATOMIC_SEQ_CST);
 382 }
 383
 384 void
 385 membar_sync(void)
 386 {
 387         __atomic_thread_fence(__ATOMIC_SEQ_CST);
 388 }
 389
 390 void
 391 membar_producer(void)
 392 {
 393         __atomic_thread_fence(__ATOMIC_RELEASE);
 394 }
 395
 396 void
 397 membar_consumer(void)
 398 {
 399         __atomic_thread_fence(__ATOMIC_ACQUIRE);
 400 }