sys/contrib/ck/include/ck_pr.h

   1 /*
   2  * Copyright 2009-2015 Samy Al Bahra.
   3  * Copyright 2011 David Joseph.
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #ifndef CK_PR_H
  29 #define CK_PR_H
  30
  31 #include <ck_cc.h>
  32 #include <ck_limits.h>
  33 #include <ck_md.h>
  34 #include <ck_stdint.h>
  35 #include <ck_stdbool.h>
  36
  37 #ifndef CK_USE_CC_BUILTINS
  38 #if defined(__x86_64__)
  39 #include "gcc/x86_64/ck_pr.h"
  40 #elif defined(__x86__)
  41 #include "gcc/x86/ck_pr.h"
  42 #elif defined(__sparcv9__)
  43 #include "gcc/sparcv9/ck_pr.h"
  44 #elif defined(__ppc64__)
  45 #include "gcc/ppc64/ck_pr.h"
  46 #elif defined(__s390x__)
  47 #include "gcc/s390x/ck_pr.h"
  48 #elif defined(__ppc__)
  49 #include "gcc/ppc/ck_pr.h"
  50 #elif defined(__arm__)
  51 #if __ARM_ARCH >= 6
  52 #include "gcc/arm/ck_pr.h"
  53 #else
  54 #include "gcc/arm/ck_pr_armv4.h"
  55 #endif
  56 #elif defined(__aarch64__)
  57 #include "gcc/aarch64/ck_pr.h"
  58 #elif !defined(__GNUC__)
  59 #error Your platform is unsupported
  60 #endif
  61 #endif /* !CK_USE_CC_BUILTINS */
  62
  63 #if defined(__GNUC__)
  64 #include "gcc/ck_pr.h"
  65 #endif
  66
  67 #define CK_PR_FENCE_EMIT(T)                     \
  68         CK_CC_INLINE static void                \
  69         ck_pr_fence_##T(void)                   \
  70         {                                       \
  71                 ck_pr_fence_strict_##T();       \
  72                 return;                         \
  73         }
  74 #define CK_PR_FENCE_NOOP(T)                     \
  75         CK_CC_INLINE static void                \
  76         ck_pr_fence_##T(void)                   \
  77         {                                       \
  78                 ck_pr_barrier();                \
  79                 return;                         \
  80         }
  81
  82 /*
  83  * None of the currently supported platforms allow for data-dependent
  84  * load ordering.
  85  */
  86 CK_PR_FENCE_NOOP(load_depends)
  87 #define ck_pr_fence_strict_load_depends ck_pr_fence_load_depends
  88
  89 /*
  90  * In memory models where atomic operations do not have serializing
  91  * effects, atomic read-modify-write operations are modeled as stores.
  92  */
  93 #if defined(CK_MD_RMO)
  94 /*
  95  * Only stores to the same location have a global
  96  * ordering.
  97  */
  98 CK_PR_FENCE_EMIT(atomic)
  99 CK_PR_FENCE_EMIT(atomic_load)
 100 CK_PR_FENCE_EMIT(atomic_store)
 101 CK_PR_FENCE_EMIT(store_atomic)
 102 CK_PR_FENCE_EMIT(load_atomic)
 103 CK_PR_FENCE_EMIT(load_store)
 104 CK_PR_FENCE_EMIT(store_load)
 105 CK_PR_FENCE_EMIT(load)
 106 CK_PR_FENCE_EMIT(store)
 107 CK_PR_FENCE_EMIT(memory)
 108 CK_PR_FENCE_EMIT(acquire)
 109 CK_PR_FENCE_EMIT(release)
 110 CK_PR_FENCE_EMIT(acqrel)
 111 CK_PR_FENCE_EMIT(lock)
 112 CK_PR_FENCE_EMIT(unlock)
 113 #elif defined(CK_MD_PSO)
 114 /*
 115  * Anything can be re-ordered with respect to stores.
 116  * Otherwise, loads are executed in-order.
 117  */
 118 CK_PR_FENCE_EMIT(atomic)
 119 CK_PR_FENCE_NOOP(atomic_load)
 120 CK_PR_FENCE_EMIT(atomic_store)
 121 CK_PR_FENCE_EMIT(store_atomic)
 122 CK_PR_FENCE_NOOP(load_atomic)
 123 CK_PR_FENCE_EMIT(load_store)
 124 CK_PR_FENCE_EMIT(store_load)
 125 CK_PR_FENCE_NOOP(load)
 126 CK_PR_FENCE_EMIT(store)
 127 CK_PR_FENCE_EMIT(memory)
 128 CK_PR_FENCE_EMIT(acquire)
 129 CK_PR_FENCE_EMIT(release)
 130 CK_PR_FENCE_EMIT(acqrel)
 131 CK_PR_FENCE_EMIT(lock)
 132 CK_PR_FENCE_EMIT(unlock)
 133 #elif defined(CK_MD_TSO)
 134 /*
 135  * Only loads are re-ordered and only with respect to
 136  * prior stores. Atomic operations are serializing.
 137  */
 138 CK_PR_FENCE_NOOP(atomic)
 139 CK_PR_FENCE_NOOP(atomic_load)
 140 CK_PR_FENCE_NOOP(atomic_store)
 141 CK_PR_FENCE_NOOP(store_atomic)
 142 CK_PR_FENCE_NOOP(load_atomic)
 143 CK_PR_FENCE_NOOP(load_store)
 144 CK_PR_FENCE_EMIT(store_load)
 145 CK_PR_FENCE_NOOP(load)
 146 CK_PR_FENCE_NOOP(store)
 147 CK_PR_FENCE_EMIT(memory)
 148 CK_PR_FENCE_NOOP(acquire)
 149 CK_PR_FENCE_NOOP(release)
 150 CK_PR_FENCE_NOOP(acqrel)
 151 CK_PR_FENCE_NOOP(lock)
 152 CK_PR_FENCE_NOOP(unlock)
 153 #else
 154 #error "No memory model has been defined."
 155 #endif /* CK_MD_TSO */
 156
 157 #undef CK_PR_FENCE_EMIT
 158 #undef CK_PR_FENCE_NOOP
 159
 160 #ifndef CK_F_PR_RFO
 161 #define CK_F_PR_RFO
 162 CK_CC_INLINE static void
 163 ck_pr_rfo(const void *m)
 164 {
 165
 166         (void)m;
 167         return;
 168 }
 169 #endif /* CK_F_PR_RFO */
 170
 171 #define CK_PR_STORE_SAFE(DST, VAL, TYPE)                        \
 172     ck_pr_md_store_##TYPE(                                      \
 173         ((void)sizeof(*(DST) = (VAL)), (DST)),                  \
 174         (VAL))
 175
 176 #define ck_pr_store_ptr(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), ptr)
 177 #define ck_pr_store_char(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), char)
 178 #ifndef CK_PR_DISABLE_DOUBLE
 179 #define ck_pr_store_double(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), double)
 180 #endif
 181 #define ck_pr_store_uint(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), uint)
 182 #define ck_pr_store_int(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), int)
 183 #define ck_pr_store_32(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 32)
 184 #define ck_pr_store_16(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 16)
 185 #define ck_pr_store_8(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 8)
 186
 187 #define ck_pr_store_ptr_unsafe(DST, VAL) ck_pr_md_store_ptr((DST), (VAL))
 188
 189 #ifdef CK_F_PR_LOAD_64
 190 #define ck_pr_store_64(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 64)
 191 #endif /* CK_F_PR_LOAD_64 */
 192
 193 #define CK_PR_LOAD_PTR_SAFE(SRC) (CK_CC_TYPEOF(*(SRC), (void *)))ck_pr_md_load_ptr((SRC))
 194 #define ck_pr_load_ptr(SRC) CK_PR_LOAD_PTR_SAFE((SRC))
 195
 196 #define CK_PR_LOAD_SAFE(SRC, TYPE) ck_pr_md_load_##TYPE((SRC))
 197 #define ck_pr_load_char(SRC) CK_PR_LOAD_SAFE((SRC), char)
 198 #ifndef CK_PR_DISABLE_DOUBLE
 199 #define ck_pr_load_double(SRC) CK_PR_LOAD_SAFE((SRC), double)
 200 #endif
 201 #define ck_pr_load_uint(SRC) CK_PR_LOAD_SAFE((SRC), uint)
 202 #define ck_pr_load_int(SRC) CK_PR_LOAD_SAFE((SRC), int)
 203 #define ck_pr_load_32(SRC) CK_PR_LOAD_SAFE((SRC), 32)
 204 #define ck_pr_load_16(SRC) CK_PR_LOAD_SAFE((SRC), 16)
 205 #define ck_pr_load_8(SRC) CK_PR_LOAD_SAFE((SRC), 8)
 206
 207 #ifdef CK_F_PR_LOAD_64
 208 #define ck_pr_load_64(SRC) CK_PR_LOAD_SAFE((SRC), 64)
 209 #endif /* CK_F_PR_LOAD_64 */
 210
 211 #define CK_PR_BIN(K, S, M, T, P, C)                                     \
 212         CK_CC_INLINE static void                                        \
 213         ck_pr_##K##_##S(M *target, T value)                             \
 214         {                                                               \
 215                 T previous;                                             \
 216                 C punt;                                                 \
 217                 punt = ck_pr_md_load_##S(target);                       \
 218                 previous = (T)punt;                                     \
 219                 while (ck_pr_cas_##S##_value(target,                    \
 220                                              (C)previous,               \
 221                                              (C)(previous P value),     \
 222                                              &previous) == false)       \
 223                         ck_pr_stall();                                  \
 224                                                                         \
 225                 return;                                                 \
 226         }
 227
 228 #define CK_PR_BIN_S(K, S, T, P) CK_PR_BIN(K, S, T, T, P, T)
 229
 230 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE)
 231
 232 #ifndef CK_F_PR_ADD_CHAR
 233 #define CK_F_PR_ADD_CHAR
 234 CK_PR_BIN_S(add, char, char, +)
 235 #endif /* CK_F_PR_ADD_CHAR */
 236
 237 #ifndef CK_F_PR_SUB_CHAR
 238 #define CK_F_PR_SUB_CHAR
 239 CK_PR_BIN_S(sub, char, char, -)
 240 #endif /* CK_F_PR_SUB_CHAR */
 241
 242 #ifndef CK_F_PR_AND_CHAR
 243 #define CK_F_PR_AND_CHAR
 244 CK_PR_BIN_S(and, char, char, &)
 245 #endif /* CK_F_PR_AND_CHAR */
 246
 247 #ifndef CK_F_PR_XOR_CHAR
 248 #define CK_F_PR_XOR_CHAR
 249 CK_PR_BIN_S(xor, char, char, ^)
 250 #endif /* CK_F_PR_XOR_CHAR */
 251
 252 #ifndef CK_F_PR_OR_CHAR
 253 #define CK_F_PR_OR_CHAR
 254 CK_PR_BIN_S(or, char, char, |)
 255 #endif /* CK_F_PR_OR_CHAR */
 256
 257 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */
 258
 259 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
 260
 261 #ifndef CK_F_PR_ADD_INT
 262 #define CK_F_PR_ADD_INT
 263 CK_PR_BIN_S(add, int, int, +)
 264 #endif /* CK_F_PR_ADD_INT */
 265
 266 #ifndef CK_F_PR_SUB_INT
 267 #define CK_F_PR_SUB_INT
 268 CK_PR_BIN_S(sub, int, int, -)
 269 #endif /* CK_F_PR_SUB_INT */
 270
 271 #ifndef CK_F_PR_AND_INT
 272 #define CK_F_PR_AND_INT
 273 CK_PR_BIN_S(and, int, int, &)
 274 #endif /* CK_F_PR_AND_INT */
 275
 276 #ifndef CK_F_PR_XOR_INT
 277 #define CK_F_PR_XOR_INT
 278 CK_PR_BIN_S(xor, int, int, ^)
 279 #endif /* CK_F_PR_XOR_INT */
 280
 281 #ifndef CK_F_PR_OR_INT
 282 #define CK_F_PR_OR_INT
 283 CK_PR_BIN_S(or, int, int, |)
 284 #endif /* CK_F_PR_OR_INT */
 285
 286 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
 287
 288 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \
 289             !defined(CK_PR_DISABLE_DOUBLE)
 290
 291 #ifndef CK_F_PR_ADD_DOUBLE
 292 #define CK_F_PR_ADD_DOUBLE
 293 CK_PR_BIN_S(add, double, double, +)
 294 #endif /* CK_F_PR_ADD_DOUBLE */
 295
 296 #ifndef CK_F_PR_SUB_DOUBLE
 297 #define CK_F_PR_SUB_DOUBLE
 298 CK_PR_BIN_S(sub, double, double, -)
 299 #endif /* CK_F_PR_SUB_DOUBLE */
 300
 301 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */
 302
 303 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
 304
 305 #ifndef CK_F_PR_ADD_UINT
 306 #define CK_F_PR_ADD_UINT
 307 CK_PR_BIN_S(add, uint, unsigned int, +)
 308 #endif /* CK_F_PR_ADD_UINT */
 309
 310 #ifndef CK_F_PR_SUB_UINT
 311 #define CK_F_PR_SUB_UINT
 312 CK_PR_BIN_S(sub, uint, unsigned int, -)
 313 #endif /* CK_F_PR_SUB_UINT */
 314
 315 #ifndef CK_F_PR_AND_UINT
 316 #define CK_F_PR_AND_UINT
 317 CK_PR_BIN_S(and, uint, unsigned int, &)
 318 #endif /* CK_F_PR_AND_UINT */
 319
 320 #ifndef CK_F_PR_XOR_UINT
 321 #define CK_F_PR_XOR_UINT
 322 CK_PR_BIN_S(xor, uint, unsigned int, ^)
 323 #endif /* CK_F_PR_XOR_UINT */
 324
 325 #ifndef CK_F_PR_OR_UINT
 326 #define CK_F_PR_OR_UINT
 327 CK_PR_BIN_S(or, uint, unsigned int, |)
 328 #endif /* CK_F_PR_OR_UINT */
 329
 330 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
 331
 332 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
 333
 334 #ifndef CK_F_PR_ADD_PTR
 335 #define CK_F_PR_ADD_PTR
 336 CK_PR_BIN(add, ptr, void, uintptr_t, +, void *)
 337 #endif /* CK_F_PR_ADD_PTR */
 338
 339 #ifndef CK_F_PR_SUB_PTR
 340 #define CK_F_PR_SUB_PTR
 341 CK_PR_BIN(sub, ptr, void, uintptr_t, -, void *)
 342 #endif /* CK_F_PR_SUB_PTR */
 343
 344 #ifndef CK_F_PR_AND_PTR
 345 #define CK_F_PR_AND_PTR
 346 CK_PR_BIN(and, ptr, void, uintptr_t, &, void *)
 347 #endif /* CK_F_PR_AND_PTR */
 348
 349 #ifndef CK_F_PR_XOR_PTR
 350 #define CK_F_PR_XOR_PTR
 351 CK_PR_BIN(xor, ptr, void, uintptr_t, ^, void *)
 352 #endif /* CK_F_PR_XOR_PTR */
 353
 354 #ifndef CK_F_PR_OR_PTR
 355 #define CK_F_PR_OR_PTR
 356 CK_PR_BIN(or, ptr, void, uintptr_t, |, void *)
 357 #endif /* CK_F_PR_OR_PTR */
 358
 359 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
 360
 361 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
 362
 363 #ifndef CK_F_PR_ADD_64
 364 #define CK_F_PR_ADD_64
 365 CK_PR_BIN_S(add, 64, uint64_t, +)
 366 #endif /* CK_F_PR_ADD_64 */
 367
 368 #ifndef CK_F_PR_SUB_64
 369 #define CK_F_PR_SUB_64
 370 CK_PR_BIN_S(sub, 64, uint64_t, -)
 371 #endif /* CK_F_PR_SUB_64 */
 372
 373 #ifndef CK_F_PR_AND_64
 374 #define CK_F_PR_AND_64
 375 CK_PR_BIN_S(and, 64, uint64_t, &)
 376 #endif /* CK_F_PR_AND_64 */
 377
 378 #ifndef CK_F_PR_XOR_64
 379 #define CK_F_PR_XOR_64
 380 CK_PR_BIN_S(xor, 64, uint64_t, ^)
 381 #endif /* CK_F_PR_XOR_64 */
 382
 383 #ifndef CK_F_PR_OR_64
 384 #define CK_F_PR_OR_64
 385 CK_PR_BIN_S(or, 64, uint64_t, |)
 386 #endif /* CK_F_PR_OR_64 */
 387
 388 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
 389
 390 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
 391
 392 #ifndef CK_F_PR_ADD_32
 393 #define CK_F_PR_ADD_32
 394 CK_PR_BIN_S(add, 32, uint32_t, +)
 395 #endif /* CK_F_PR_ADD_32 */
 396
 397 #ifndef CK_F_PR_SUB_32
 398 #define CK_F_PR_SUB_32
 399 CK_PR_BIN_S(sub, 32, uint32_t, -)
 400 #endif /* CK_F_PR_SUB_32 */
 401
 402 #ifndef CK_F_PR_AND_32
 403 #define CK_F_PR_AND_32
 404 CK_PR_BIN_S(and, 32, uint32_t, &)
 405 #endif /* CK_F_PR_AND_32 */
 406
 407 #ifndef CK_F_PR_XOR_32
 408 #define CK_F_PR_XOR_32
 409 CK_PR_BIN_S(xor, 32, uint32_t, ^)
 410 #endif /* CK_F_PR_XOR_32 */
 411
 412 #ifndef CK_F_PR_OR_32
 413 #define CK_F_PR_OR_32
 414 CK_PR_BIN_S(or, 32, uint32_t, |)
 415 #endif /* CK_F_PR_OR_32 */
 416
 417 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
 418
 419 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
 420
 421 #ifndef CK_F_PR_ADD_16
 422 #define CK_F_PR_ADD_16
 423 CK_PR_BIN_S(add, 16, uint16_t, +)
 424 #endif /* CK_F_PR_ADD_16 */
 425
 426 #ifndef CK_F_PR_SUB_16
 427 #define CK_F_PR_SUB_16
 428 CK_PR_BIN_S(sub, 16, uint16_t, -)
 429 #endif /* CK_F_PR_SUB_16 */
 430
 431 #ifndef CK_F_PR_AND_16
 432 #define CK_F_PR_AND_16
 433 CK_PR_BIN_S(and, 16, uint16_t, &)
 434 #endif /* CK_F_PR_AND_16 */
 435
 436 #ifndef CK_F_PR_XOR_16
 437 #define CK_F_PR_XOR_16
 438 CK_PR_BIN_S(xor, 16, uint16_t, ^)
 439 #endif /* CK_F_PR_XOR_16 */
 440
 441 #ifndef CK_F_PR_OR_16
 442 #define CK_F_PR_OR_16
 443 CK_PR_BIN_S(or, 16, uint16_t, |)
 444 #endif /* CK_F_PR_OR_16 */
 445
 446 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
 447
 448 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE)
 449
 450 #ifndef CK_F_PR_ADD_8
 451 #define CK_F_PR_ADD_8
 452 CK_PR_BIN_S(add, 8, uint8_t, +)
 453 #endif /* CK_F_PR_ADD_8 */
 454
 455 #ifndef CK_F_PR_SUB_8
 456 #define CK_F_PR_SUB_8
 457 CK_PR_BIN_S(sub, 8, uint8_t, -)
 458 #endif /* CK_F_PR_SUB_8 */
 459
 460 #ifndef CK_F_PR_AND_8
 461 #define CK_F_PR_AND_8
 462 CK_PR_BIN_S(and, 8, uint8_t, &)
 463 #endif /* CK_F_PR_AND_8 */
 464
 465 #ifndef CK_F_PR_XOR_8
 466 #define CK_F_PR_XOR_8
 467 CK_PR_BIN_S(xor, 8, uint8_t, ^)
 468 #endif /* CK_F_PR_XOR_8 */
 469
 470 #ifndef CK_F_PR_OR_8
 471 #define CK_F_PR_OR_8
 472 CK_PR_BIN_S(or, 8, uint8_t, |)
 473 #endif /* CK_F_PR_OR_8 */
 474
 475 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */
 476
 477 #undef CK_PR_BIN_S
 478 #undef CK_PR_BIN
 479
 480 #define CK_PR_BTX(K, S, M, T, P, C, R)                                             \
 481         CK_CC_INLINE static bool                                                   \
 482         ck_pr_##K##_##S(M *target, unsigned int offset)                            \
 483         {                                                                          \
 484                 T previous;                                                        \
 485                 C punt;                                                            \
 486                 punt = ck_pr_md_load_##S(target);                                  \
 487                 previous = (T)punt;                                                \
 488                 while (ck_pr_cas_##S##_value(target, (C)previous,                  \
 489                         (C)(previous P (R ((T)1 << offset))), &previous) == false) \
 490                                 ck_pr_stall();                                     \
 491                 return ((previous >> offset) & 1);                                 \
 492         }
 493
 494 #define CK_PR_BTX_S(K, S, T, P, R) CK_PR_BTX(K, S, T, T, P, T, R)
 495
 496 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
 497
 498 #ifndef CK_F_PR_BTC_INT
 499 #define CK_F_PR_BTC_INT
 500 CK_PR_BTX_S(btc, int, int, ^,)
 501 #endif /* CK_F_PR_BTC_INT */
 502
 503 #ifndef CK_F_PR_BTR_INT
 504 #define CK_F_PR_BTR_INT
 505 CK_PR_BTX_S(btr, int, int, &, ~)
 506 #endif /* CK_F_PR_BTR_INT */
 507
 508 #ifndef CK_F_PR_BTS_INT
 509 #define CK_F_PR_BTS_INT
 510 CK_PR_BTX_S(bts, int, int, |,)
 511 #endif /* CK_F_PR_BTS_INT */
 512
 513 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
 514
 515 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
 516
 517 #ifndef CK_F_PR_BTC_UINT
 518 #define CK_F_PR_BTC_UINT
 519 CK_PR_BTX_S(btc, uint, unsigned int, ^,)
 520 #endif /* CK_F_PR_BTC_UINT */
 521
 522 #ifndef CK_F_PR_BTR_UINT
 523 #define CK_F_PR_BTR_UINT
 524 CK_PR_BTX_S(btr, uint, unsigned int, &, ~)
 525 #endif /* CK_F_PR_BTR_UINT */
 526
 527 #ifndef CK_F_PR_BTS_UINT
 528 #define CK_F_PR_BTS_UINT
 529 CK_PR_BTX_S(bts, uint, unsigned int, |,)
 530 #endif /* CK_F_PR_BTS_UINT */
 531
 532 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
 533
 534 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
 535
 536 #ifndef CK_F_PR_BTC_PTR
 537 #define CK_F_PR_BTC_PTR
 538 CK_PR_BTX(btc, ptr, void, uintptr_t, ^, void *,)
 539 #endif /* CK_F_PR_BTC_PTR */
 540
 541 #ifndef CK_F_PR_BTR_PTR
 542 #define CK_F_PR_BTR_PTR
 543 CK_PR_BTX(btr, ptr, void, uintptr_t, &, void *, ~)
 544 #endif /* CK_F_PR_BTR_PTR */
 545
 546 #ifndef CK_F_PR_BTS_PTR
 547 #define CK_F_PR_BTS_PTR
 548 CK_PR_BTX(bts, ptr, void, uintptr_t, |, void *,)
 549 #endif /* CK_F_PR_BTS_PTR */
 550
 551 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
 552
 553 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
 554
 555 #ifndef CK_F_PR_BTC_64
 556 #define CK_F_PR_BTC_64
 557 CK_PR_BTX_S(btc, 64, uint64_t, ^,)
 558 #endif /* CK_F_PR_BTC_64 */
 559
 560 #ifndef CK_F_PR_BTR_64
 561 #define CK_F_PR_BTR_64
 562 CK_PR_BTX_S(btr, 64, uint64_t, &, ~)
 563 #endif /* CK_F_PR_BTR_64 */
 564
 565 #ifndef CK_F_PR_BTS_64
 566 #define CK_F_PR_BTS_64
 567 CK_PR_BTX_S(bts, 64, uint64_t, |,)
 568 #endif /* CK_F_PR_BTS_64 */
 569
 570 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
 571
 572 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
 573
 574 #ifndef CK_F_PR_BTC_32
 575 #define CK_F_PR_BTC_32
 576 CK_PR_BTX_S(btc, 32, uint32_t, ^,)
 577 #endif /* CK_F_PR_BTC_32 */
 578
 579 #ifndef CK_F_PR_BTR_32
 580 #define CK_F_PR_BTR_32
 581 CK_PR_BTX_S(btr, 32, uint32_t, &, ~)
 582 #endif /* CK_F_PR_BTR_32 */
 583
 584 #ifndef CK_F_PR_BTS_32
 585 #define CK_F_PR_BTS_32
 586 CK_PR_BTX_S(bts, 32, uint32_t, |,)
 587 #endif /* CK_F_PR_BTS_32 */
 588
 589 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
 590
 591 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
 592
 593 #ifndef CK_F_PR_BTC_16
 594 #define CK_F_PR_BTC_16
 595 CK_PR_BTX_S(btc, 16, uint16_t, ^,)
 596 #endif /* CK_F_PR_BTC_16 */
 597
 598 #ifndef CK_F_PR_BTR_16
 599 #define CK_F_PR_BTR_16
 600 CK_PR_BTX_S(btr, 16, uint16_t, &, ~)
 601 #endif /* CK_F_PR_BTR_16 */
 602
 603 #ifndef CK_F_PR_BTS_16
 604 #define CK_F_PR_BTS_16
 605 CK_PR_BTX_S(bts, 16, uint16_t, |,)
 606 #endif /* CK_F_PR_BTS_16 */
 607
 608 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
 609
 610 #undef CK_PR_BTX_S
 611 #undef CK_PR_BTX
 612
 613 #define CK_PR_UNARY(K, X, S, M, T)                                      \
 614         CK_CC_INLINE static void                                        \
 615         ck_pr_##K##_##S(M *target)                                      \
 616         {                                                               \
 617                 ck_pr_##X##_##S(target, (T)1);                          \
 618                 return;                                                 \
 619         }
 620
 621 #define CK_PR_UNARY_Z(K, S, M, T, P, C, Z)                              \
 622         CK_CC_INLINE static bool                                        \
 623         ck_pr_##K##_##S##_is_zero(M *target)                            \
 624         {                                                               \
 625                 T previous;                                             \
 626                 C punt;                                                 \
 627                 punt = (C)ck_pr_md_load_##S(target);                    \
 628                 previous = (T)punt;                                     \
 629                 while (ck_pr_cas_##S##_value(target,                    \
 630                                              (C)previous,               \
 631                                              (C)(previous P 1),         \
 632                                              &previous) == false)       \
 633                         ck_pr_stall();                                  \
 634                 return previous == (T)Z;                                \
 635         }
 636
 637 #define CK_PR_UNARY_Z_STUB(K, S, M)                                     \
 638         CK_CC_INLINE static void                                        \
 639         ck_pr_##K##_##S##_zero(M *target, bool *zero)                   \
 640         {                                                               \
 641                 *zero = ck_pr_##K##_##S##_is_zero(target);              \
 642                 return;                                                 \
 643         }
 644
 645 #define CK_PR_UNARY_S(K, X, S, M) CK_PR_UNARY(K, X, S, M, M)
 646 #define CK_PR_UNARY_Z_S(K, S, M, P, Z)          \
 647         CK_PR_UNARY_Z(K, S, M, M, P, M, Z)      \
 648         CK_PR_UNARY_Z_STUB(K, S, M)
 649
 650 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE)
 651
 652 #ifndef CK_F_PR_INC_CHAR
 653 #define CK_F_PR_INC_CHAR
 654 CK_PR_UNARY_S(inc, add, char, char)
 655 #endif /* CK_F_PR_INC_CHAR */
 656
 657 #ifndef CK_F_PR_INC_CHAR_ZERO
 658 #define CK_F_PR_INC_CHAR_ZERO
 659 CK_PR_UNARY_Z_S(inc, char, char, +, -1)
 660 #else
 661 CK_PR_UNARY_Z_STUB(inc, char, char)
 662 #endif /* CK_F_PR_INC_CHAR_ZERO */
 663
 664 #ifndef CK_F_PR_DEC_CHAR
 665 #define CK_F_PR_DEC_CHAR
 666 CK_PR_UNARY_S(dec, sub, char, char)
 667 #endif /* CK_F_PR_DEC_CHAR */
 668
 669 #ifndef CK_F_PR_DEC_CHAR_ZERO
 670 #define CK_F_PR_DEC_CHAR_ZERO
 671 CK_PR_UNARY_Z_S(dec, char, char, -, 1)
 672 #else
 673 CK_PR_UNARY_Z_STUB(dec, char, char)
 674 #endif /* CK_F_PR_DEC_CHAR_ZERO */
 675
 676 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */
 677
 678 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
 679
 680 #ifndef CK_F_PR_INC_INT
 681 #define CK_F_PR_INC_INT
 682 CK_PR_UNARY_S(inc, add, int, int)
 683 #endif /* CK_F_PR_INC_INT */
 684
 685 #ifndef CK_F_PR_INC_INT_ZERO
 686 #define CK_F_PR_INC_INT_ZERO
 687 CK_PR_UNARY_Z_S(inc, int, int, +, -1)
 688 #else
 689 CK_PR_UNARY_Z_STUB(inc, int, int)
 690 #endif /* CK_F_PR_INC_INT_ZERO */
 691
 692 #ifndef CK_F_PR_DEC_INT
 693 #define CK_F_PR_DEC_INT
 694 CK_PR_UNARY_S(dec, sub, int, int)
 695 #endif /* CK_F_PR_DEC_INT */
 696
 697 #ifndef CK_F_PR_DEC_INT_ZERO
 698 #define CK_F_PR_DEC_INT_ZERO
 699 CK_PR_UNARY_Z_S(dec, int, int, -, 1)
 700 #else
 701 CK_PR_UNARY_Z_STUB(dec, int, int)
 702 #endif /* CK_F_PR_DEC_INT_ZERO */
 703
 704 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
 705
 706 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \
 707             !defined(CK_PR_DISABLE_DOUBLE)
 708
 709 #ifndef CK_F_PR_INC_DOUBLE
 710 #define CK_F_PR_INC_DOUBLE
 711 CK_PR_UNARY_S(inc, add, double, double)
 712 #endif /* CK_F_PR_INC_DOUBLE */
 713
 714 #ifndef CK_F_PR_DEC_DOUBLE
 715 #define CK_F_PR_DEC_DOUBLE
 716 CK_PR_UNARY_S(dec, sub, double, double)
 717 #endif /* CK_F_PR_DEC_DOUBLE */
 718
 719 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */
 720
 721 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
 722
 723 #ifndef CK_F_PR_INC_UINT
 724 #define CK_F_PR_INC_UINT
 725 CK_PR_UNARY_S(inc, add, uint, unsigned int)
 726 #endif /* CK_F_PR_INC_UINT */
 727
 728 #ifndef CK_F_PR_INC_UINT_ZERO
 729 #define CK_F_PR_INC_UINT_ZERO
 730 CK_PR_UNARY_Z_S(inc, uint, unsigned int, +, UINT_MAX)
 731 #else
 732 CK_PR_UNARY_Z_STUB(inc, uint, unsigned int)
 733 #endif /* CK_F_PR_INC_UINT_ZERO */
 734
 735 #ifndef CK_F_PR_DEC_UINT
 736 #define CK_F_PR_DEC_UINT
 737 CK_PR_UNARY_S(dec, sub, uint, unsigned int)
 738 #endif /* CK_F_PR_DEC_UINT */
 739
 740 #ifndef CK_F_PR_DEC_UINT_ZERO
 741 #define CK_F_PR_DEC_UINT_ZERO
 742 CK_PR_UNARY_Z_S(dec, uint, unsigned int, -, 1)
 743 #else
 744 CK_PR_UNARY_Z_STUB(dec, uint, unsigned int)
 745 #endif /* CK_F_PR_DEC_UINT_ZERO */
 746
 747 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
 748
 749 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
 750
 751 #ifndef CK_F_PR_INC_PTR
 752 #define CK_F_PR_INC_PTR
 753 CK_PR_UNARY(inc, add, ptr, void, uintptr_t)
 754 #endif /* CK_F_PR_INC_PTR */
 755
 756 #ifndef CK_F_PR_INC_PTR_ZERO
 757 #define CK_F_PR_INC_PTR_ZERO
 758 CK_PR_UNARY_Z(inc, ptr, void, uintptr_t, +, void *, UINT_MAX)
 759 #else
 760 CK_PR_UNARY_Z_STUB(inc, ptr, void)
 761 #endif /* CK_F_PR_INC_PTR_ZERO */
 762
 763 #ifndef CK_F_PR_DEC_PTR
 764 #define CK_F_PR_DEC_PTR
 765 CK_PR_UNARY(dec, sub, ptr, void, uintptr_t)
 766 #endif /* CK_F_PR_DEC_PTR */
 767
 768 #ifndef CK_F_PR_DEC_PTR_ZERO
 769 #define CK_F_PR_DEC_PTR_ZERO
 770 CK_PR_UNARY_Z(dec, ptr, void, uintptr_t, -, void *, 1)
 771 #else
 772 CK_PR_UNARY_Z_STUB(dec, ptr, void)
 773 #endif /* CK_F_PR_DEC_PTR_ZERO */
 774
 775 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
 776
 777 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
 778
 779 #ifndef CK_F_PR_INC_64
 780 #define CK_F_PR_INC_64
 781 CK_PR_UNARY_S(inc, add, 64, uint64_t)
 782 #endif /* CK_F_PR_INC_64 */
 783
 784 #ifndef CK_F_PR_INC_64_ZERO
 785 #define CK_F_PR_INC_64_ZERO
 786 CK_PR_UNARY_Z_S(inc, 64, uint64_t, +, UINT64_MAX)
 787 #else
 788 CK_PR_UNARY_Z_STUB(inc, 64, uint64_t)
 789 #endif /* CK_F_PR_INC_64_ZERO */
 790
 791 #ifndef CK_F_PR_DEC_64
 792 #define CK_F_PR_DEC_64
 793 CK_PR_UNARY_S(dec, sub, 64, uint64_t)
 794 #endif /* CK_F_PR_DEC_64 */
 795
 796 #ifndef CK_F_PR_DEC_64_ZERO
 797 #define CK_F_PR_DEC_64_ZERO
 798 CK_PR_UNARY_Z_S(dec, 64, uint64_t, -, 1)
 799 #else
 800 CK_PR_UNARY_Z_STUB(dec, 64, uint64_t)
 801 #endif /* CK_F_PR_DEC_64_ZERO */
 802
 803 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
 804
 805 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
 806
 807 #ifndef CK_F_PR_INC_32
 808 #define CK_F_PR_INC_32
 809 CK_PR_UNARY_S(inc, add, 32, uint32_t)
 810 #endif /* CK_F_PR_INC_32 */
 811
 812 #ifndef CK_F_PR_INC_32_ZERO
 813 #define CK_F_PR_INC_32_ZERO
 814 CK_PR_UNARY_Z_S(inc, 32, uint32_t, +, UINT32_MAX)
 815 #else
 816 CK_PR_UNARY_Z_STUB(inc, 32, uint32_t)
 817 #endif /* CK_F_PR_INC_32_ZERO */
 818
 819 #ifndef CK_F_PR_DEC_32
 820 #define CK_F_PR_DEC_32
 821 CK_PR_UNARY_S(dec, sub, 32, uint32_t)
 822 #endif /* CK_F_PR_DEC_32 */
 823
 824 #ifndef CK_F_PR_DEC_32_ZERO
 825 #define CK_F_PR_DEC_32_ZERO
 826 CK_PR_UNARY_Z_S(dec, 32, uint32_t, -, 1)
 827 #else
 828 CK_PR_UNARY_Z_STUB(dec, 32, uint32_t)
 829 #endif /* CK_F_PR_DEC_32_ZERO */
 830
 831 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
 832
 833 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
 834
 835 #ifndef CK_F_PR_INC_16
 836 #define CK_F_PR_INC_16
 837 CK_PR_UNARY_S(inc, add, 16, uint16_t)
 838 #endif /* CK_F_PR_INC_16 */
 839
 840 #ifndef CK_F_PR_INC_16_ZERO
 841 #define CK_F_PR_INC_16_ZERO
 842 CK_PR_UNARY_Z_S(inc, 16, uint16_t, +, UINT16_MAX)
 843 #else
 844 CK_PR_UNARY_Z_STUB(inc, 16, uint16_t)
 845 #endif /* CK_F_PR_INC_16_ZERO */
 846
 847 #ifndef CK_F_PR_DEC_16
 848 #define CK_F_PR_DEC_16
 849 CK_PR_UNARY_S(dec, sub, 16, uint16_t)
 850 #endif /* CK_F_PR_DEC_16 */
 851
 852 #ifndef CK_F_PR_DEC_16_ZERO
 853 #define CK_F_PR_DEC_16_ZERO
 854 CK_PR_UNARY_Z_S(dec, 16, uint16_t, -, 1)
 855 #else
 856 CK_PR_UNARY_Z_STUB(dec, 16, uint16_t)
 857 #endif /* CK_F_PR_DEC_16_ZERO */
 858
 859 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
 860
 861 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE)
 862
 863 #ifndef CK_F_PR_INC_8
 864 #define CK_F_PR_INC_8
 865 CK_PR_UNARY_S(inc, add, 8, uint8_t)
 866 #endif /* CK_F_PR_INC_8 */
 867
 868 #ifndef CK_F_PR_INC_8_ZERO
 869 #define CK_F_PR_INC_8_ZERO
 870 CK_PR_UNARY_Z_S(inc, 8, uint8_t, +, UINT8_MAX)
 871 #else
 872 CK_PR_UNARY_Z_STUB(inc, 8, uint8_t)
 873 #endif /* CK_F_PR_INC_8_ZERO */
 874
 875 #ifndef CK_F_PR_DEC_8
 876 #define CK_F_PR_DEC_8
 877 CK_PR_UNARY_S(dec, sub, 8, uint8_t)
 878 #endif /* CK_F_PR_DEC_8 */
 879
 880 #ifndef CK_F_PR_DEC_8_ZERO
 881 #define CK_F_PR_DEC_8_ZERO
 882 CK_PR_UNARY_Z_S(dec, 8, uint8_t, -, 1)
 883 #else
 884 CK_PR_UNARY_Z_STUB(dec, 8, uint8_t)
 885 #endif /* CK_F_PR_DEC_8_ZERO */
 886
 887 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */
 888
 889 #undef CK_PR_UNARY_Z_S
 890 #undef CK_PR_UNARY_S
 891 #undef CK_PR_UNARY_Z
 892 #undef CK_PR_UNARY
 893
 894 #define CK_PR_N(K, S, M, T, P, C)                                       \
 895         CK_CC_INLINE static void                                        \
 896         ck_pr_##K##_##S(M *target)                                      \
 897         {                                                               \
 898                 T previous;                                             \
 899                 C punt;                                                 \
 900                 punt = (C)ck_pr_md_load_##S(target);                    \
 901                 previous = (T)punt;                                     \
 902                 while (ck_pr_cas_##S##_value(target,                    \
 903                                              (C)previous,               \
 904                                              (C)(P previous),           \
 905                                              &previous) == false)       \
 906                         ck_pr_stall();                                  \
 907                                                                         \
 908                 return;                                                 \
 909         }
 910
 911 #define CK_PR_N_Z(S, M, T, C)                                           \
 912         CK_CC_INLINE static void                                        \
 913         ck_pr_neg_##S##_zero(M *target, bool *zero)                     \
 914         {                                                               \
 915                 T previous;                                             \
 916                 C punt;                                                 \
 917                 punt = (C)ck_pr_md_load_##S(target);                    \
 918                 previous = (T)punt;                                     \
 919                 while (ck_pr_cas_##S##_value(target,                    \
 920                                              (C)previous,               \
 921                                              (C)(-previous),            \
 922                                              &previous) == false)       \
 923                         ck_pr_stall();                                  \
 924                                                                         \
 925                 *zero = previous == 0;                                  \
 926                 return;                                                 \
 927         }
 928
 929 #define CK_PR_N_S(K, S, M, P)   CK_PR_N(K, S, M, M, P, M)
 930 #define CK_PR_N_Z_S(S, M)       CK_PR_N_Z(S, M, M, M)
 931
 932 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE)
 933
 934 #ifndef CK_F_PR_NOT_CHAR
 935 #define CK_F_PR_NOT_CHAR
 936 CK_PR_N_S(not, char, char, ~)
 937 #endif /* CK_F_PR_NOT_CHAR */
 938
 939 #ifndef CK_F_PR_NEG_CHAR
 940 #define CK_F_PR_NEG_CHAR
 941 CK_PR_N_S(neg, char, char, -)
 942 #endif /* CK_F_PR_NEG_CHAR */
 943
 944 #ifndef CK_F_PR_NEG_CHAR_ZERO
 945 #define CK_F_PR_NEG_CHAR_ZERO
 946 CK_PR_N_Z_S(char, char)
 947 #endif /* CK_F_PR_NEG_CHAR_ZERO */
 948
 949 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */
 950
 951 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
 952
 953 #ifndef CK_F_PR_NOT_INT
 954 #define CK_F_PR_NOT_INT
 955 CK_PR_N_S(not, int, int, ~)
 956 #endif /* CK_F_PR_NOT_INT */
 957
 958 #ifndef CK_F_PR_NEG_INT
 959 #define CK_F_PR_NEG_INT
 960 CK_PR_N_S(neg, int, int, -)
 961 #endif /* CK_F_PR_NEG_INT */
 962
 963 #ifndef CK_F_PR_NEG_INT_ZERO
 964 #define CK_F_PR_NEG_INT_ZERO
 965 CK_PR_N_Z_S(int, int)
 966 #endif /* CK_F_PR_NEG_INT_ZERO */
 967
 968 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
 969
 970 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \
 971             !defined(CK_PR_DISABLE_DOUBLE)
 972
 973 #ifndef CK_F_PR_NEG_DOUBLE
 974 #define CK_F_PR_NEG_DOUBLE
 975 CK_PR_N_S(neg, double, double, -)
 976 #endif /* CK_F_PR_NEG_DOUBLE */
 977
 978 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */
 979
 980 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
 981
 982 #ifndef CK_F_PR_NOT_UINT
 983 #define CK_F_PR_NOT_UINT
 984 CK_PR_N_S(not, uint, unsigned int, ~)
 985 #endif /* CK_F_PR_NOT_UINT */
 986
 987 #ifndef CK_F_PR_NEG_UINT
 988 #define CK_F_PR_NEG_UINT
 989 CK_PR_N_S(neg, uint, unsigned int, -)
 990 #endif /* CK_F_PR_NEG_UINT */
 991
 992 #ifndef CK_F_PR_NEG_UINT_ZERO
 993 #define CK_F_PR_NEG_UINT_ZERO
 994 CK_PR_N_Z_S(uint, unsigned int)
 995 #endif /* CK_F_PR_NEG_UINT_ZERO */
 996
 997 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
 998
 999 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
1000
1001 #ifndef CK_F_PR_NOT_PTR
1002 #define CK_F_PR_NOT_PTR
1003 CK_PR_N(not, ptr, void, uintptr_t, ~, void *)
1004 #endif /* CK_F_PR_NOT_PTR */
1005
1006 #ifndef CK_F_PR_NEG_PTR
1007 #define CK_F_PR_NEG_PTR
1008 CK_PR_N(neg, ptr, void, uintptr_t, -, void *)
1009 #endif /* CK_F_PR_NEG_PTR */
1010
1011 #ifndef CK_F_PR_NEG_PTR_ZERO
1012 #define CK_F_PR_NEG_PTR_ZERO
1013 CK_PR_N_Z(ptr, void, uintptr_t, void *)
1014 #endif /* CK_F_PR_NEG_PTR_ZERO */
1015
1016 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
1017
1018 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
1019
1020 #ifndef CK_F_PR_NOT_64
1021 #define CK_F_PR_NOT_64
1022 CK_PR_N_S(not, 64, uint64_t, ~)
1023 #endif /* CK_F_PR_NOT_64 */
1024
1025 #ifndef CK_F_PR_NEG_64
1026 #define CK_F_PR_NEG_64
1027 CK_PR_N_S(neg, 64, uint64_t, -)
1028 #endif /* CK_F_PR_NEG_64 */
1029
1030 #ifndef CK_F_PR_NEG_64_ZERO
1031 #define CK_F_PR_NEG_64_ZERO
1032 CK_PR_N_Z_S(64, uint64_t)
1033 #endif /* CK_F_PR_NEG_64_ZERO */
1034
1035 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
1036
1037 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
1038
1039 #ifndef CK_F_PR_NOT_32
1040 #define CK_F_PR_NOT_32
1041 CK_PR_N_S(not, 32, uint32_t, ~)
1042 #endif /* CK_F_PR_NOT_32 */
1043
1044 #ifndef CK_F_PR_NEG_32
1045 #define CK_F_PR_NEG_32
1046 CK_PR_N_S(neg, 32, uint32_t, -)
1047 #endif /* CK_F_PR_NEG_32 */
1048
1049 #ifndef CK_F_PR_NEG_32_ZERO
1050 #define CK_F_PR_NEG_32_ZERO
1051 CK_PR_N_Z_S(32, uint32_t)
1052 #endif /* CK_F_PR_NEG_32_ZERO */
1053
1054 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
1055
1056 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
1057
1058 #ifndef CK_F_PR_NOT_16
1059 #define CK_F_PR_NOT_16
1060 CK_PR_N_S(not, 16, uint16_t, ~)
1061 #endif /* CK_F_PR_NOT_16 */
1062
1063 #ifndef CK_F_PR_NEG_16
1064 #define CK_F_PR_NEG_16
1065 CK_PR_N_S(neg, 16, uint16_t, -)
1066 #endif /* CK_F_PR_NEG_16 */
1067
1068 #ifndef CK_F_PR_NEG_16_ZERO
1069 #define CK_F_PR_NEG_16_ZERO
1070 CK_PR_N_Z_S(16, uint16_t)
1071 #endif /* CK_F_PR_NEG_16_ZERO */
1072
1073 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
1074
1075 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE)
1076
1077 #ifndef CK_F_PR_NOT_8
1078 #define CK_F_PR_NOT_8
1079 CK_PR_N_S(not, 8, uint8_t, ~)
1080 #endif /* CK_F_PR_NOT_8 */
1081
1082 #ifndef CK_F_PR_NEG_8
1083 #define CK_F_PR_NEG_8
1084 CK_PR_N_S(neg, 8, uint8_t, -)
1085 #endif /* CK_F_PR_NEG_8 */
1086
1087 #ifndef CK_F_PR_NEG_8_ZERO
1088 #define CK_F_PR_NEG_8_ZERO
1089 CK_PR_N_Z_S(8, uint8_t)
1090 #endif /* CK_F_PR_NEG_8_ZERO */
1091
1092 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */
1093
1094 #undef CK_PR_N_Z_S
1095 #undef CK_PR_N_S
1096 #undef CK_PR_N_Z
1097 #undef CK_PR_N
1098
1099 #define CK_PR_FAA(S, M, T, C)                                           \
1100         CK_CC_INLINE static C                                           \
1101         ck_pr_faa_##S(M *target, T delta)                               \
1102         {                                                               \
1103                 T previous;                                             \
1104                 C punt;                                                 \
1105                 punt = (C)ck_pr_md_load_##S(target);                    \
1106                 previous = (T)punt;                                     \
1107                 while (ck_pr_cas_##S##_value(target,                    \
1108                                              (C)previous,               \
1109                                              (C)(previous + delta),     \
1110                                              &previous) == false)       \
1111                         ck_pr_stall();                                  \
1112                                                                         \
1113                 return ((C)previous);                                   \
1114         }
1115
1116 #define CK_PR_FAS(S, M, C)                                              \
1117         CK_CC_INLINE static C                                           \
1118         ck_pr_fas_##S(M *target, C update)                              \
1119         {                                                               \
1120                 C previous;                                             \
1121                 previous = ck_pr_md_load_##S(target);                   \
1122                 while (ck_pr_cas_##S##_value(target,                    \
1123                                              previous,                  \
1124                                              update,                    \
1125                                              &previous) == false)       \
1126                         ck_pr_stall();                                  \
1127                                                                         \
1128                 return (previous);                                      \
1129         }
1130
1131 #define CK_PR_FAA_S(S, M) CK_PR_FAA(S, M, M, M)
1132 #define CK_PR_FAS_S(S, M) CK_PR_FAS(S, M, M)
1133
1134 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE)
1135
1136 #ifndef CK_F_PR_FAA_CHAR
1137 #define CK_F_PR_FAA_CHAR
1138 CK_PR_FAA_S(char, char)
1139 #endif /* CK_F_PR_FAA_CHAR */
1140
1141 #ifndef CK_F_PR_FAS_CHAR
1142 #define CK_F_PR_FAS_CHAR
1143 CK_PR_FAS_S(char, char)
1144 #endif /* CK_F_PR_FAS_CHAR */
1145
1146 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */
1147
1148 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
1149
1150 #ifndef CK_F_PR_FAA_INT
1151 #define CK_F_PR_FAA_INT
1152 CK_PR_FAA_S(int, int)
1153 #endif /* CK_F_PR_FAA_INT */
1154
1155 #ifndef CK_F_PR_FAS_INT
1156 #define CK_F_PR_FAS_INT
1157 CK_PR_FAS_S(int, int)
1158 #endif /* CK_F_PR_FAS_INT */
1159
1160 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
1161
1162 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \
1163             !defined(CK_PR_DISABLE_DOUBLE)
1164
1165 #ifndef CK_F_PR_FAA_DOUBLE
1166 #define CK_F_PR_FAA_DOUBLE
1167 CK_PR_FAA_S(double, double)
1168 #endif /* CK_F_PR_FAA_DOUBLE */
1169
1170 #ifndef CK_F_PR_FAS_DOUBLE
1171 #define CK_F_PR_FAS_DOUBLE
1172 CK_PR_FAS_S(double, double)
1173 #endif /* CK_F_PR_FAS_DOUBLE */
1174
1175 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */
1176
1177 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
1178
1179 #ifndef CK_F_PR_FAA_UINT
1180 #define CK_F_PR_FAA_UINT
1181 CK_PR_FAA_S(uint, unsigned int)
1182 #endif /* CK_F_PR_FAA_UINT */
1183
1184 #ifndef CK_F_PR_FAS_UINT
1185 #define CK_F_PR_FAS_UINT
1186 CK_PR_FAS_S(uint, unsigned int)
1187 #endif /* CK_F_PR_FAS_UINT */
1188
1189 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
1190
1191 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
1192
1193 #ifndef CK_F_PR_FAA_PTR
1194 #define CK_F_PR_FAA_PTR
1195 CK_PR_FAA(ptr, void, uintptr_t, void *)
1196 #endif /* CK_F_PR_FAA_PTR */
1197
1198 #ifndef CK_F_PR_FAS_PTR
1199 #define CK_F_PR_FAS_PTR
1200 CK_PR_FAS(ptr, void, void *)
1201 #endif /* CK_F_PR_FAS_PTR */
1202
1203 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
1204
1205 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
1206
1207 #ifndef CK_F_PR_FAA_64
1208 #define CK_F_PR_FAA_64
1209 CK_PR_FAA_S(64, uint64_t)
1210 #endif /* CK_F_PR_FAA_64 */
1211
1212 #ifndef CK_F_PR_FAS_64
1213 #define CK_F_PR_FAS_64
1214 CK_PR_FAS_S(64, uint64_t)
1215 #endif /* CK_F_PR_FAS_64 */
1216
1217 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
1218
1219 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
1220
1221 #ifndef CK_F_PR_FAA_32
1222 #define CK_F_PR_FAA_32
1223 CK_PR_FAA_S(32, uint32_t)
1224 #endif /* CK_F_PR_FAA_32 */
1225
1226 #ifndef CK_F_PR_FAS_32
1227 #define CK_F_PR_FAS_32
1228 CK_PR_FAS_S(32, uint32_t)
1229 #endif /* CK_F_PR_FAS_32 */
1230
1231 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
1232
1233 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
1234
1235 #ifndef CK_F_PR_FAA_16
1236 #define CK_F_PR_FAA_16
1237 CK_PR_FAA_S(16, uint16_t)
1238 #endif /* CK_F_PR_FAA_16 */
1239
1240 #ifndef CK_F_PR_FAS_16
1241 #define CK_F_PR_FAS_16
1242 CK_PR_FAS_S(16, uint16_t)
1243 #endif /* CK_F_PR_FAS_16 */
1244
1245 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
1246
1247 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE)
1248
1249 #ifndef CK_F_PR_FAA_8
1250 #define CK_F_PR_FAA_8
1251 CK_PR_FAA_S(8, uint8_t)
1252 #endif /* CK_F_PR_FAA_8 */
1253
1254 #ifndef CK_F_PR_FAS_8
1255 #define CK_F_PR_FAS_8
1256 CK_PR_FAS_S(8, uint8_t)
1257 #endif /* CK_F_PR_FAS_8 */
1258
1259 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */
1260
1261 #undef CK_PR_FAA_S
1262 #undef CK_PR_FAS_S
1263 #undef CK_PR_FAA
1264 #undef CK_PR_FAS
1265
1266 #endif /* CK_PR_H */