sys/contrib/ck/include/ck_pr.h

   1 /*
   2  * Copyright 2009-2015 Samy Al Bahra.
   3  * Copyright 2011 David Joseph.
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #ifndef CK_PR_H
  29 #define CK_PR_H
  30
  31 #include <ck_cc.h>
  32 #include <ck_limits.h>
  33 #include <ck_md.h>
  34 #include <ck_stdint.h>
  35 #include <ck_stdbool.h>
  36
  37 #ifndef CK_USE_CC_BUILTINS
  38 #if defined(__x86_64__)
  39 #include "gcc/x86_64/ck_pr.h"
  40 #elif defined(__x86__)
  41 #include "gcc/x86/ck_pr.h"
  42 #elif defined(__sparcv9__)
  43 #include "gcc/sparcv9/ck_pr.h"
  44 #elif defined(__ppc64__)
  45 #include "gcc/ppc64/ck_pr.h"
  46 #elif defined(__s390x__)
  47 #include "gcc/s390x/ck_pr.h"
  48 #elif defined(__ppc__)
  49 #include "gcc/ppc/ck_pr.h"
  50 #elif defined(__arm__)
  51 #if __ARM_ARCH >= 6
  52 #include "gcc/arm/ck_pr.h"
  53 #else
  54 #include "gcc/arm/ck_pr_armv4.h"
  55 #endif
  56 #elif defined(__aarch64__)
  57 #include "gcc/aarch64/ck_pr.h"
  58 #elif !defined(__GNUC__)
  59 #error Your platform is unsupported
  60 #endif
  61 #endif /* !CK_USE_CC_BUILTINS */
  62
  63 #if defined(__GNUC__)
  64 #include "gcc/ck_pr.h"
  65 #endif
  66
  67 #define CK_PR_FENCE_EMIT(T)                     \
  68         CK_CC_INLINE static void                \
  69         ck_pr_fence_##T(void)                   \
  70         {                                       \
  71                 ck_pr_fence_strict_##T();       \
  72                 return;                         \
  73         }
  74 #define CK_PR_FENCE_NOOP(T)                     \
  75         CK_CC_INLINE static void                \
  76         ck_pr_fence_##T(void)                   \
  77         {                                       \
  78                 ck_pr_barrier();                \
  79                 return;                         \
  80         }
  81
  82 /*
  83  * None of the currently supported platforms allow for data-dependent
  84  * load ordering.
  85  */
  86 CK_PR_FENCE_NOOP(load_depends)
  87 #define ck_pr_fence_strict_load_depends ck_pr_fence_load_depends
  88
  89 /*
  90  * In memory models where atomic operations do not have serializing
  91  * effects, atomic read-modify-write operations are modeled as stores.
  92  */
  93 #if defined(CK_MD_RMO)
  94 /*
  95  * Only stores to the same location have a global
  96  * ordering.
  97  */
  98 CK_PR_FENCE_EMIT(atomic)
  99 CK_PR_FENCE_EMIT(atomic_load)
 100 CK_PR_FENCE_EMIT(atomic_store)
 101 CK_PR_FENCE_EMIT(store_atomic)
 102 CK_PR_FENCE_EMIT(load_atomic)
 103 CK_PR_FENCE_EMIT(load_store)
 104 CK_PR_FENCE_EMIT(store_load)
 105 CK_PR_FENCE_EMIT(load)
 106 CK_PR_FENCE_EMIT(store)
 107 CK_PR_FENCE_EMIT(memory)
 108 CK_PR_FENCE_EMIT(acquire)
 109 CK_PR_FENCE_EMIT(release)
 110 CK_PR_FENCE_EMIT(acqrel)
 111 CK_PR_FENCE_EMIT(lock)
 112 CK_PR_FENCE_EMIT(unlock)
 113 #elif defined(CK_MD_PSO)
 114 /*
 115  * Anything can be re-ordered with respect to stores.
 116  * Otherwise, loads are executed in-order.
 117  */
 118 CK_PR_FENCE_EMIT(atomic)
 119 CK_PR_FENCE_NOOP(atomic_load)
 120 CK_PR_FENCE_EMIT(atomic_store)
 121 CK_PR_FENCE_EMIT(store_atomic)
 122 CK_PR_FENCE_NOOP(load_atomic)
 123 CK_PR_FENCE_EMIT(load_store)
 124 CK_PR_FENCE_EMIT(store_load)
 125 CK_PR_FENCE_NOOP(load)
 126 CK_PR_FENCE_EMIT(store)
 127 CK_PR_FENCE_EMIT(memory)
 128 CK_PR_FENCE_EMIT(acquire)
 129 CK_PR_FENCE_EMIT(release)
 130 CK_PR_FENCE_EMIT(acqrel)
 131 CK_PR_FENCE_EMIT(lock)
 132 CK_PR_FENCE_EMIT(unlock)
 133 #elif defined(CK_MD_TSO)
 134 /*
 135  * Only loads are re-ordered and only with respect to
 136  * prior stores. Atomic operations are serializing.
 137  */
 138 CK_PR_FENCE_NOOP(atomic)
 139 CK_PR_FENCE_NOOP(atomic_load)
 140 CK_PR_FENCE_NOOP(atomic_store)
 141 CK_PR_FENCE_NOOP(store_atomic)
 142 CK_PR_FENCE_NOOP(load_atomic)
 143 CK_PR_FENCE_NOOP(load_store)
 144 CK_PR_FENCE_EMIT(store_load)
 145 CK_PR_FENCE_NOOP(load)
 146 CK_PR_FENCE_NOOP(store)
 147 CK_PR_FENCE_EMIT(memory)
 148 CK_PR_FENCE_NOOP(acquire)
 149 CK_PR_FENCE_NOOP(release)
 150 CK_PR_FENCE_NOOP(acqrel)
 151 CK_PR_FENCE_NOOP(lock)
 152 CK_PR_FENCE_NOOP(unlock)
 153 #else
 154 #error "No memory model has been defined."
 155 #endif /* CK_MD_TSO */
 156
 157 #undef CK_PR_FENCE_EMIT
 158 #undef CK_PR_FENCE_NOOP
 159
 160 #ifndef CK_F_PR_RFO
 161 #define CK_F_PR_RFO
 162 CK_CC_INLINE static void
 163 ck_pr_rfo(const void *m)
 164 {
 165
 166         (void)m;
 167         return;
 168 }
 169 #endif /* CK_F_PR_RFO */
 170
 171 #define CK_PR_STORE_SAFE(DST, VAL, TYPE)                        \
 172     ck_pr_md_store_##TYPE(                                      \
 173         ((void)sizeof(*(DST) = (VAL)), (DST)),                  \
 174         (VAL))
 175
 176 #define ck_pr_store_ptr(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), ptr)
 177 #define ck_pr_store_char(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), char)
 178 #ifndef CK_PR_DISABLE_DOUBLE
 179 #define ck_pr_store_double(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), double)
 180 #endif
 181 #define ck_pr_store_uint(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), uint)
 182 #define ck_pr_store_int(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), int)
 183 #define ck_pr_store_32(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 32)
 184 #define ck_pr_store_16(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 16)
 185 #define ck_pr_store_8(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 8)
 186
 187 #define ck_pr_store_ptr_unsafe(DST, VAL) ck_pr_md_store_ptr((DST), (VAL))
 188
 189 #ifdef CK_F_PR_LOAD_64
 190 #define ck_pr_store_64(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 64)
 191 #endif /* CK_F_PR_LOAD_64 */
 192
 193 #define CK_PR_LOAD_PTR_SAFE(SRC) (CK_CC_TYPEOF(*(SRC), (void *)))ck_pr_md_load_ptr((SRC))
 194 #define ck_pr_load_ptr(SRC) CK_PR_LOAD_PTR_SAFE((SRC))
 195
 196 #define CK_PR_LOAD_SAFE(SRC, TYPE) ck_pr_md_load_##TYPE((SRC))
 197 #define ck_pr_load_char(SRC) CK_PR_LOAD_SAFE((SRC), char)
 198 #ifndef CK_PR_DISABLE_DOUBLE
 199 #define ck_pr_load_double(SRC) CK_PR_LOAD_SAFE((SRC), double)
 200 #endif
 201 #define ck_pr_load_uint(SRC) CK_PR_LOAD_SAFE((SRC), uint)
 202 #define ck_pr_load_int(SRC) CK_PR_LOAD_SAFE((SRC), int)
 203 #define ck_pr_load_32(SRC) CK_PR_LOAD_SAFE((SRC), 32)
 204 #define ck_pr_load_16(SRC) CK_PR_LOAD_SAFE((SRC), 16)
 205 #define ck_pr_load_8(SRC) CK_PR_LOAD_SAFE((SRC), 8)
 206
 207 #ifdef CK_F_PR_LOAD_64
 208 #define ck_pr_load_64(SRC) CK_PR_LOAD_SAFE((SRC), 64)
 209 #endif /* CK_F_PR_LOAD_64 */
 210
 211 #define CK_PR_BIN(K, S, M, T, P, C)                                     \
 212         CK_CC_INLINE static void                                        \
 213         ck_pr_##K##_##S(M *target, T value)                             \
 214         {                                                               \
 215                 T previous;                                             \
 216                 C punt;                                                 \
 217                 punt = ck_pr_md_load_##S(target);                       \
 218                 previous = (T)punt;                                     \
 219                 while (ck_pr_cas_##S##_value(target,                    \
 220                                              (C)previous,               \
 221                                              (C)(previous P value),     \
 222                                              &previous) == false)       \
 223                         ck_pr_stall();                                  \
 224                                                                         \
 225                 return;                                                 \
 226         }
 227
 228 #define CK_PR_BIN_S(K, S, T, P) CK_PR_BIN(K, S, T, T, P, T)
 229
 230 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE)
 231
 232 #ifndef CK_F_PR_ADD_CHAR
 233 #define CK_F_PR_ADD_CHAR
 234 CK_PR_BIN_S(add, char, char, +)
 235 #endif /* CK_F_PR_ADD_CHAR */
 236
 237 #ifndef CK_F_PR_SUB_CHAR
 238 #define CK_F_PR_SUB_CHAR
 239 CK_PR_BIN_S(sub, char, char, -)
 240 #endif /* CK_F_PR_SUB_CHAR */
 241
 242 #ifndef CK_F_PR_AND_CHAR
 243 #define CK_F_PR_AND_CHAR
 244 CK_PR_BIN_S(and, char, char, &)
 245 #endif /* CK_F_PR_AND_CHAR */
 246
 247 #ifndef CK_F_PR_XOR_CHAR
 248 #define CK_F_PR_XOR_CHAR
 249 CK_PR_BIN_S(xor, char, char, ^)
 250 #endif /* CK_F_PR_XOR_CHAR */
 251
 252 #ifndef CK_F_PR_OR_CHAR
 253 #define CK_F_PR_OR_CHAR
 254 CK_PR_BIN_S(or, char, char, |)
 255 #endif /* CK_F_PR_OR_CHAR */
 256
 257 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */
 258
 259 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
 260
 261 #ifndef CK_F_PR_ADD_INT
 262 #define CK_F_PR_ADD_INT
 263 CK_PR_BIN_S(add, int, int, +)
 264 #endif /* CK_F_PR_ADD_INT */
 265
 266 #ifndef CK_F_PR_SUB_INT
 267 #define CK_F_PR_SUB_INT
 268 CK_PR_BIN_S(sub, int, int, -)
 269 #endif /* CK_F_PR_SUB_INT */
 270
 271 #ifndef CK_F_PR_AND_INT
 272 #define CK_F_PR_AND_INT
 273 CK_PR_BIN_S(and, int, int, &)
 274 #endif /* CK_F_PR_AND_INT */
 275
 276 #ifndef CK_F_PR_XOR_INT
 277 #define CK_F_PR_XOR_INT
 278 CK_PR_BIN_S(xor, int, int, ^)
 279 #endif /* CK_F_PR_XOR_INT */
 280
 281 #ifndef CK_F_PR_OR_INT
 282 #define CK_F_PR_OR_INT
 283 CK_PR_BIN_S(or, int, int, |)
 284 #endif /* CK_F_PR_OR_INT */
 285
 286 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
 287
 288 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \
 289             !defined(CK_PR_DISABLE_DOUBLE)
 290
 291 #ifndef CK_F_PR_ADD_DOUBLE
 292 #define CK_F_PR_ADD_DOUBLE
 293 CK_PR_BIN_S(add, double, double, +)
 294 #endif /* CK_F_PR_ADD_DOUBLE */
 295
 296 #ifndef CK_F_PR_SUB_DOUBLE
 297 #define CK_F_PR_SUB_DOUBLE
 298 CK_PR_BIN_S(sub, double, double, -)
 299 #endif /* CK_F_PR_SUB_DOUBLE */
 300
 301 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */
 302
 303 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
 304
 305 #ifndef CK_F_PR_ADD_UINT
 306 #define CK_F_PR_ADD_UINT
 307 CK_PR_BIN_S(add, uint, unsigned int, +)
 308 #endif /* CK_F_PR_ADD_UINT */
 309
 310 #ifndef CK_F_PR_SUB_UINT
 311 #define CK_F_PR_SUB_UINT
 312 CK_PR_BIN_S(sub, uint, unsigned int, -)
 313 #endif /* CK_F_PR_SUB_UINT */
 314
 315 #ifndef CK_F_PR_AND_UINT
 316 #define CK_F_PR_AND_UINT
 317 CK_PR_BIN_S(and, uint, unsigned int, &)
 318 #endif /* CK_F_PR_AND_UINT */
 319
 320 #ifndef CK_F_PR_XOR_UINT
 321 #define CK_F_PR_XOR_UINT
 322 CK_PR_BIN_S(xor, uint, unsigned int, ^)
 323 #endif /* CK_F_PR_XOR_UINT */
 324
 325 #ifndef CK_F_PR_OR_UINT
 326 #define CK_F_PR_OR_UINT
 327 CK_PR_BIN_S(or, uint, unsigned int, |)
 328 #endif /* CK_F_PR_OR_UINT */
 329
 330 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
 331
 332 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
 333
 334 #ifndef CK_F_PR_ADD_PTR
 335 #define CK_F_PR_ADD_PTR
 336 CK_PR_BIN(add, ptr, void, uintptr_t, +, void *)
 337 #endif /* CK_F_PR_ADD_PTR */
 338
 339 #ifndef CK_F_PR_SUB_PTR
 340 #define CK_F_PR_SUB_PTR
 341 CK_PR_BIN(sub, ptr, void, uintptr_t, -, void *)
 342 #endif /* CK_F_PR_SUB_PTR */
 343
 344 #ifndef CK_F_PR_AND_PTR
 345 #define CK_F_PR_AND_PTR
 346 CK_PR_BIN(and, ptr, void, uintptr_t, &, void *)
 347 #endif /* CK_F_PR_AND_PTR */
 348
 349 #ifndef CK_F_PR_XOR_PTR
 350 #define CK_F_PR_XOR_PTR
 351 CK_PR_BIN(xor, ptr, void, uintptr_t, ^, void *)
 352 #endif /* CK_F_PR_XOR_PTR */
 353
 354 #ifndef CK_F_PR_OR_PTR
 355 #define CK_F_PR_OR_PTR
 356 CK_PR_BIN(or, ptr, void, uintptr_t, |, void *)
 357 #endif /* CK_F_PR_OR_PTR */
 358
 359 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
 360
 361 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
 362
 363 #ifndef CK_F_PR_ADD_64
 364 #define CK_F_PR_ADD_64
 365 CK_PR_BIN_S(add, 64, uint64_t, +)
 366 #endif /* CK_F_PR_ADD_64 */
 367
 368 #ifndef CK_F_PR_SUB_64
 369 #define CK_F_PR_SUB_64
 370 CK_PR_BIN_S(sub, 64, uint64_t, -)
 371 #endif /* CK_F_PR_SUB_64 */
 372
 373 #ifndef CK_F_PR_AND_64
 374 #define CK_F_PR_AND_64
 375 CK_PR_BIN_S(and, 64, uint64_t, &)
 376 #endif /* CK_F_PR_AND_64 */
 377
 378 #ifndef CK_F_PR_XOR_64
 379 #define CK_F_PR_XOR_64
 380 CK_PR_BIN_S(xor, 64, uint64_t, ^)
 381 #endif /* CK_F_PR_XOR_64 */
 382
 383 #ifndef CK_F_PR_OR_64
 384 #define CK_F_PR_OR_64
 385 CK_PR_BIN_S(or, 64, uint64_t, |)
 386 #endif /* CK_F_PR_OR_64 */
 387
 388 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
 389
 390 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
 391
 392 #ifndef CK_F_PR_ADD_32
 393 #define CK_F_PR_ADD_32
 394 CK_PR_BIN_S(add, 32, uint32_t, +)
 395 #endif /* CK_F_PR_ADD_32 */
 396
 397 #ifndef CK_F_PR_SUB_32
 398 #define CK_F_PR_SUB_32
 399 CK_PR_BIN_S(sub, 32, uint32_t, -)
 400 #endif /* CK_F_PR_SUB_32 */
 401
 402 #ifndef CK_F_PR_AND_32
 403 #define CK_F_PR_AND_32
 404 CK_PR_BIN_S(and, 32, uint32_t, &)
 405 #endif /* CK_F_PR_AND_32 */
 406
 407 #ifndef CK_F_PR_XOR_32
 408 #define CK_F_PR_XOR_32
 409 CK_PR_BIN_S(xor, 32, uint32_t, ^)
 410 #endif /* CK_F_PR_XOR_32 */
 411
 412 #ifndef CK_F_PR_OR_32
 413 #define CK_F_PR_OR_32
 414 CK_PR_BIN_S(or, 32, uint32_t, |)
 415 #endif /* CK_F_PR_OR_32 */
 416
 417 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
 418
 419 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
 420
 421 #ifndef CK_F_PR_ADD_16
 422 #define CK_F_PR_ADD_16
 423 CK_PR_BIN_S(add, 16, uint16_t, +)
 424 #endif /* CK_F_PR_ADD_16 */
 425
 426 #ifndef CK_F_PR_SUB_16
 427 #define CK_F_PR_SUB_16
 428 CK_PR_BIN_S(sub, 16, uint16_t, -)
 429 #endif /* CK_F_PR_SUB_16 */
 430
 431 #ifndef CK_F_PR_AND_16
 432 #define CK_F_PR_AND_16
 433 CK_PR_BIN_S(and, 16, uint16_t, &)
 434 #endif /* CK_F_PR_AND_16 */
 435
 436 #ifndef CK_F_PR_XOR_16
 437 #define CK_F_PR_XOR_16
 438 CK_PR_BIN_S(xor, 16, uint16_t, ^)
 439 #endif /* CK_F_PR_XOR_16 */
 440
 441 #ifndef CK_F_PR_OR_16
 442 #define CK_F_PR_OR_16
 443 CK_PR_BIN_S(or, 16, uint16_t, |)
 444 #endif /* CK_F_PR_OR_16 */
 445
 446 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
 447
 448 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE)
 449
 450 #ifndef CK_F_PR_ADD_8
 451 #define CK_F_PR_ADD_8
 452 CK_PR_BIN_S(add, 8, uint8_t, +)
 453 #endif /* CK_F_PR_ADD_8 */
 454
 455 #ifndef CK_F_PR_SUB_8
 456 #define CK_F_PR_SUB_8
 457 CK_PR_BIN_S(sub, 8, uint8_t, -)
 458 #endif /* CK_F_PR_SUB_8 */
 459
 460 #ifndef CK_F_PR_AND_8
 461 #define CK_F_PR_AND_8
 462 CK_PR_BIN_S(and, 8, uint8_t, &)
 463 #endif /* CK_F_PR_AND_8 */
 464
 465 #ifndef CK_F_PR_XOR_8
 466 #define CK_F_PR_XOR_8
 467 CK_PR_BIN_S(xor, 8, uint8_t, ^)
 468 #endif /* CK_F_PR_XOR_8 */
 469
 470 #ifndef CK_F_PR_OR_8
 471 #define CK_F_PR_OR_8
 472 CK_PR_BIN_S(or, 8, uint8_t, |)
 473 #endif /* CK_F_PR_OR_8 */
 474
 475 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */
 476
 477 #undef CK_PR_BIN_S
 478 #undef CK_PR_BIN
 479
 480 #define CK_PR_BTX(K, S, M, T, P, C, R)                                             \
 481         CK_CC_INLINE static bool                                                   \
 482         ck_pr_##K##_##S(M *target, unsigned int offset)                            \
 483         {                                                                          \
 484                 T previous;                                                        \
 485                 C punt;                                                            \
 486                 punt = ck_pr_md_load_##S(target);                                  \
 487                 previous = (T)punt;                                                \
 488                 while (ck_pr_cas_##S##_value(target, (C)previous,                  \
 489                         (C)(previous P (R ((T)1 << offset))), &previous) == false) \
 490                                 ck_pr_stall();                                     \
 491                 return ((previous >> offset) & 1);                                 \
 492         }
 493
 494 #define CK_PR_BTX_S(K, S, T, P, R) CK_PR_BTX(K, S, T, T, P, T, R)
 495
 496 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
 497
 498 #ifndef CK_F_PR_BTC_INT
 499 #define CK_F_PR_BTC_INT
 500 CK_PR_BTX_S(btc, int, int, ^,)
 501 #endif /* CK_F_PR_BTC_INT */
 502
 503 #ifndef CK_F_PR_BTR_INT
 504 #define CK_F_PR_BTR_INT
 505 CK_PR_BTX_S(btr, int, int, &, ~)
 506 #endif /* CK_F_PR_BTR_INT */
 507
 508 #ifndef CK_F_PR_BTS_INT
 509 #define CK_F_PR_BTS_INT
 510 CK_PR_BTX_S(bts, int, int, |,)
 511 #endif /* CK_F_PR_BTS_INT */
 512
 513 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
 514
 515 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
 516
 517 #ifndef CK_F_PR_BTC_UINT
 518 #define CK_F_PR_BTC_UINT
 519 CK_PR_BTX_S(btc, uint, unsigned int, ^,)
 520 #endif /* CK_F_PR_BTC_UINT */
 521
 522 #ifndef CK_F_PR_BTR_UINT
 523 #define CK_F_PR_BTR_UINT
 524 CK_PR_BTX_S(btr, uint, unsigned int, &, ~)
 525 #endif /* CK_F_PR_BTR_UINT */
 526
 527 #ifndef CK_F_PR_BTS_UINT
 528 #define CK_F_PR_BTS_UINT
 529 CK_PR_BTX_S(bts, uint, unsigned int, |,)
 530 #endif /* CK_F_PR_BTS_UINT */
 531
 532 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
 533
 534 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
 535
 536 #ifndef CK_F_PR_BTC_PTR
 537 #define CK_F_PR_BTC_PTR
 538 CK_PR_BTX(btc, ptr, void, uintptr_t, ^, void *,)
 539 #endif /* CK_F_PR_BTC_PTR */
 540
 541 #ifndef CK_F_PR_BTR_PTR
 542 #define CK_F_PR_BTR_PTR
 543 CK_PR_BTX(btr, ptr, void, uintptr_t, &, void *, ~)
 544 #endif /* CK_F_PR_BTR_PTR */
 545
 546 #ifndef CK_F_PR_BTS_PTR
 547 #define CK_F_PR_BTS_PTR
 548 CK_PR_BTX(bts, ptr, void, uintptr_t, |, void *,)
 549 #endif /* CK_F_PR_BTS_PTR */
 550
 551 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
 552
 553 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
 554
 555 #ifndef CK_F_PR_BTC_64
 556 #define CK_F_PR_BTC_64
 557 CK_PR_BTX_S(btc, 64, uint64_t, ^,)
 558 #endif /* CK_F_PR_BTC_64 */
 559
 560 #ifndef CK_F_PR_BTR_64
 561 #define CK_F_PR_BTR_64
 562 CK_PR_BTX_S(btr, 64, uint64_t, &, ~)
 563 #endif /* CK_F_PR_BTR_64 */
 564
 565 #ifndef CK_F_PR_BTS_64
 566 #define CK_F_PR_BTS_64
 567 CK_PR_BTX_S(bts, 64, uint64_t, |,)
 568 #endif /* CK_F_PR_BTS_64 */
 569
 570 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
 571
 572 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
 573
 574 #ifndef CK_F_PR_BTC_32
 575 #define CK_F_PR_BTC_32
 576 CK_PR_BTX_S(btc, 32, uint32_t, ^,)
 577 #endif /* CK_F_PR_BTC_32 */
 578
 579 #ifndef CK_F_PR_BTR_32
 580 #define CK_F_PR_BTR_32
 581 CK_PR_BTX_S(btr, 32, uint32_t, &, ~)
 582 #endif /* CK_F_PR_BTR_32 */
 583
 584 #ifndef CK_F_PR_BTS_32
 585 #define CK_F_PR_BTS_32
 586 CK_PR_BTX_S(bts, 32, uint32_t, |,)
 587 #endif /* CK_F_PR_BTS_32 */
 588
 589 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
 590
 591 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
 592
 593 #ifndef CK_F_PR_BTC_16
 594 #define CK_F_PR_BTC_16
 595 CK_PR_BTX_S(btc, 16, uint16_t, ^,)
 596 #endif /* CK_F_PR_BTC_16 */
 597
 598 #ifndef CK_F_PR_BTR_16
 599 #define CK_F_PR_BTR_16
 600 CK_PR_BTX_S(btr, 16, uint16_t, &, ~)
 601 #endif /* CK_F_PR_BTR_16 */
 602
 603 #ifndef CK_F_PR_BTS_16
 604 #define CK_F_PR_BTS_16
 605 CK_PR_BTX_S(bts, 16, uint16_t, |,)
 606 #endif /* CK_F_PR_BTS_16 */
 607
 608 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
 609
 610 #undef CK_PR_BTX_S
 611 #undef CK_PR_BTX
 612
 613 #define CK_PR_UNARY(K, X, S, M, T)                                      \
 614         CK_CC_INLINE static void                                        \
 615         ck_pr_##K##_##S(M *target)                                      \
 616         {                                                               \
 617                 ck_pr_##X##_##S(target, (T)1);                          \
 618                 return;                                                 \
 619         }
 620
 621 #define CK_PR_UNARY_Z(K, S, M, T, P, C, Z)                              \
 622         CK_CC_INLINE static void                                        \
 623         ck_pr_##K##_##S##_zero(M *target, bool *zero)                   \
 624         {                                                               \
 625                 T previous;                                             \
 626                 C punt;                                                 \
 627                 punt = (C)ck_pr_md_load_##S(target);                    \
 628                 previous = (T)punt;                                     \
 629                 while (ck_pr_cas_##S##_value(target,                    \
 630                                              (C)previous,               \
 631                                              (C)(previous P 1),         \
 632                                              &previous) == false)       \
 633                         ck_pr_stall();                                  \
 634                 *zero = previous == (T)Z;                               \
 635                 return;                                                 \
 636         }
 637
 638 #define CK_PR_UNARY_S(K, X, S, M) CK_PR_UNARY(K, X, S, M, M)
 639 #define CK_PR_UNARY_Z_S(K, S, M, P, Z) CK_PR_UNARY_Z(K, S, M, M, P, M, Z)
 640
 641 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE)
 642
 643 #ifndef CK_F_PR_INC_CHAR
 644 #define CK_F_PR_INC_CHAR
 645 CK_PR_UNARY_S(inc, add, char, char)
 646 #endif /* CK_F_PR_INC_CHAR */
 647
 648 #ifndef CK_F_PR_INC_CHAR_ZERO
 649 #define CK_F_PR_INC_CHAR_ZERO
 650 CK_PR_UNARY_Z_S(inc, char, char, +, -1)
 651 #endif /* CK_F_PR_INC_CHAR_ZERO */
 652
 653 #ifndef CK_F_PR_DEC_CHAR
 654 #define CK_F_PR_DEC_CHAR
 655 CK_PR_UNARY_S(dec, sub, char, char)
 656 #endif /* CK_F_PR_DEC_CHAR */
 657
 658 #ifndef CK_F_PR_DEC_CHAR_ZERO
 659 #define CK_F_PR_DEC_CHAR_ZERO
 660 CK_PR_UNARY_Z_S(dec, char, char, -, 1)
 661 #endif /* CK_F_PR_DEC_CHAR_ZERO */
 662
 663 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */
 664
 665 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
 666
 667 #ifndef CK_F_PR_INC_INT
 668 #define CK_F_PR_INC_INT
 669 CK_PR_UNARY_S(inc, add, int, int)
 670 #endif /* CK_F_PR_INC_INT */
 671
 672 #ifndef CK_F_PR_INC_INT_ZERO
 673 #define CK_F_PR_INC_INT_ZERO
 674 CK_PR_UNARY_Z_S(inc, int, int, +, -1)
 675 #endif /* CK_F_PR_INC_INT_ZERO */
 676
 677 #ifndef CK_F_PR_DEC_INT
 678 #define CK_F_PR_DEC_INT
 679 CK_PR_UNARY_S(dec, sub, int, int)
 680 #endif /* CK_F_PR_DEC_INT */
 681
 682 #ifndef CK_F_PR_DEC_INT_ZERO
 683 #define CK_F_PR_DEC_INT_ZERO
 684 CK_PR_UNARY_Z_S(dec, int, int, -, 1)
 685 #endif /* CK_F_PR_DEC_INT_ZERO */
 686
 687 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
 688
 689 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \
 690             !defined(CK_PR_DISABLE_DOUBLE)
 691
 692 #ifndef CK_F_PR_INC_DOUBLE
 693 #define CK_F_PR_INC_DOUBLE
 694 CK_PR_UNARY_S(inc, add, double, double)
 695 #endif /* CK_F_PR_INC_DOUBLE */
 696
 697 #ifndef CK_F_PR_DEC_DOUBLE
 698 #define CK_F_PR_DEC_DOUBLE
 699 CK_PR_UNARY_S(dec, sub, double, double)
 700 #endif /* CK_F_PR_DEC_DOUBLE */
 701
 702 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */
 703
 704 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
 705
 706 #ifndef CK_F_PR_INC_UINT
 707 #define CK_F_PR_INC_UINT
 708 CK_PR_UNARY_S(inc, add, uint, unsigned int)
 709 #endif /* CK_F_PR_INC_UINT */
 710
 711 #ifndef CK_F_PR_INC_UINT_ZERO
 712 #define CK_F_PR_INC_UINT_ZERO
 713 CK_PR_UNARY_Z_S(inc, uint, unsigned int, +, UINT_MAX)
 714 #endif /* CK_F_PR_INC_UINT_ZERO */
 715
 716 #ifndef CK_F_PR_DEC_UINT
 717 #define CK_F_PR_DEC_UINT
 718 CK_PR_UNARY_S(dec, sub, uint, unsigned int)
 719 #endif /* CK_F_PR_DEC_UINT */
 720
 721 #ifndef CK_F_PR_DEC_UINT_ZERO
 722 #define CK_F_PR_DEC_UINT_ZERO
 723 CK_PR_UNARY_Z_S(dec, uint, unsigned int, -, 1)
 724 #endif /* CK_F_PR_DEC_UINT_ZERO */
 725
 726 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
 727
 728 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
 729
 730 #ifndef CK_F_PR_INC_PTR
 731 #define CK_F_PR_INC_PTR
 732 CK_PR_UNARY(inc, add, ptr, void, uintptr_t)
 733 #endif /* CK_F_PR_INC_PTR */
 734
 735 #ifndef CK_F_PR_INC_PTR_ZERO
 736 #define CK_F_PR_INC_PTR_ZERO
 737 CK_PR_UNARY_Z(inc, ptr, void, uintptr_t, +, void *, UINT_MAX)
 738 #endif /* CK_F_PR_INC_PTR_ZERO */
 739
 740 #ifndef CK_F_PR_DEC_PTR
 741 #define CK_F_PR_DEC_PTR
 742 CK_PR_UNARY(dec, sub, ptr, void, uintptr_t)
 743 #endif /* CK_F_PR_DEC_PTR */
 744
 745 #ifndef CK_F_PR_DEC_PTR_ZERO
 746 #define CK_F_PR_DEC_PTR_ZERO
 747 CK_PR_UNARY_Z(dec, ptr, void, uintptr_t, -, void *, 1)
 748 #endif /* CK_F_PR_DEC_PTR_ZERO */
 749
 750 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
 751
 752 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
 753
 754 #ifndef CK_F_PR_INC_64
 755 #define CK_F_PR_INC_64
 756 CK_PR_UNARY_S(inc, add, 64, uint64_t)
 757 #endif /* CK_F_PR_INC_64 */
 758
 759 #ifndef CK_F_PR_INC_64_ZERO
 760 #define CK_F_PR_INC_64_ZERO
 761 CK_PR_UNARY_Z_S(inc, 64, uint64_t, +, UINT64_MAX)
 762 #endif /* CK_F_PR_INC_64_ZERO */
 763
 764 #ifndef CK_F_PR_DEC_64
 765 #define CK_F_PR_DEC_64
 766 CK_PR_UNARY_S(dec, sub, 64, uint64_t)
 767 #endif /* CK_F_PR_DEC_64 */
 768
 769 #ifndef CK_F_PR_DEC_64_ZERO
 770 #define CK_F_PR_DEC_64_ZERO
 771 CK_PR_UNARY_Z_S(dec, 64, uint64_t, -, 1)
 772 #endif /* CK_F_PR_DEC_64_ZERO */
 773
 774 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
 775
 776 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
 777
 778 #ifndef CK_F_PR_INC_32
 779 #define CK_F_PR_INC_32
 780 CK_PR_UNARY_S(inc, add, 32, uint32_t)
 781 #endif /* CK_F_PR_INC_32 */
 782
 783 #ifndef CK_F_PR_INC_32_ZERO
 784 #define CK_F_PR_INC_32_ZERO
 785 CK_PR_UNARY_Z_S(inc, 32, uint32_t, +, UINT32_MAX)
 786 #endif /* CK_F_PR_INC_32_ZERO */
 787
 788 #ifndef CK_F_PR_DEC_32
 789 #define CK_F_PR_DEC_32
 790 CK_PR_UNARY_S(dec, sub, 32, uint32_t)
 791 #endif /* CK_F_PR_DEC_32 */
 792
 793 #ifndef CK_F_PR_DEC_32_ZERO
 794 #define CK_F_PR_DEC_32_ZERO
 795 CK_PR_UNARY_Z_S(dec, 32, uint32_t, -, 1)
 796 #endif /* CK_F_PR_DEC_32_ZERO */
 797
 798 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
 799
 800 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
 801
 802 #ifndef CK_F_PR_INC_16
 803 #define CK_F_PR_INC_16
 804 CK_PR_UNARY_S(inc, add, 16, uint16_t)
 805 #endif /* CK_F_PR_INC_16 */
 806
 807 #ifndef CK_F_PR_INC_16_ZERO
 808 #define CK_F_PR_INC_16_ZERO
 809 CK_PR_UNARY_Z_S(inc, 16, uint16_t, +, UINT16_MAX)
 810 #endif /* CK_F_PR_INC_16_ZERO */
 811
 812 #ifndef CK_F_PR_DEC_16
 813 #define CK_F_PR_DEC_16
 814 CK_PR_UNARY_S(dec, sub, 16, uint16_t)
 815 #endif /* CK_F_PR_DEC_16 */
 816
 817 #ifndef CK_F_PR_DEC_16_ZERO
 818 #define CK_F_PR_DEC_16_ZERO
 819 CK_PR_UNARY_Z_S(dec, 16, uint16_t, -, 1)
 820 #endif /* CK_F_PR_DEC_16_ZERO */
 821
 822 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
 823
 824 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE)
 825
 826 #ifndef CK_F_PR_INC_8
 827 #define CK_F_PR_INC_8
 828 CK_PR_UNARY_S(inc, add, 8, uint8_t)
 829 #endif /* CK_F_PR_INC_8 */
 830
 831 #ifndef CK_F_PR_INC_8_ZERO
 832 #define CK_F_PR_INC_8_ZERO
 833 CK_PR_UNARY_Z_S(inc, 8, uint8_t, +, UINT8_MAX)
 834 #endif /* CK_F_PR_INC_8_ZERO */
 835
 836 #ifndef CK_F_PR_DEC_8
 837 #define CK_F_PR_DEC_8
 838 CK_PR_UNARY_S(dec, sub, 8, uint8_t)
 839 #endif /* CK_F_PR_DEC_8 */
 840
 841 #ifndef CK_F_PR_DEC_8_ZERO
 842 #define CK_F_PR_DEC_8_ZERO
 843 CK_PR_UNARY_Z_S(dec, 8, uint8_t, -, 1)
 844 #endif /* CK_F_PR_DEC_8_ZERO */
 845
 846 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */
 847
 848 #undef CK_PR_UNARY_Z_S
 849 #undef CK_PR_UNARY_S
 850 #undef CK_PR_UNARY_Z
 851 #undef CK_PR_UNARY
 852
 853 #define CK_PR_N(K, S, M, T, P, C)                                       \
 854         CK_CC_INLINE static void                                        \
 855         ck_pr_##K##_##S(M *target)                                      \
 856         {                                                               \
 857                 T previous;                                             \
 858                 C punt;                                                 \
 859                 punt = (C)ck_pr_md_load_##S(target);                    \
 860                 previous = (T)punt;                                     \
 861                 while (ck_pr_cas_##S##_value(target,                    \
 862                                              (C)previous,               \
 863                                              (C)(P previous),           \
 864                                              &previous) == false)       \
 865                         ck_pr_stall();                                  \
 866                                                                         \
 867                 return;                                                 \
 868         }
 869
 870 #define CK_PR_N_Z(S, M, T, C)                                           \
 871         CK_CC_INLINE static void                                        \
 872         ck_pr_neg_##S##_zero(M *target, bool *zero)                     \
 873         {                                                               \
 874                 T previous;                                             \
 875                 C punt;                                                 \
 876                 punt = (C)ck_pr_md_load_##S(target);                    \
 877                 previous = (T)punt;                                     \
 878                 while (ck_pr_cas_##S##_value(target,                    \
 879                                              (C)previous,               \
 880                                              (C)(-previous),            \
 881                                              &previous) == false)       \
 882                         ck_pr_stall();                                  \
 883                                                                         \
 884                 *zero = previous == 0;                                  \
 885                 return;                                                 \
 886         }
 887
 888 #define CK_PR_N_S(K, S, M, P)   CK_PR_N(K, S, M, M, P, M)
 889 #define CK_PR_N_Z_S(S, M)       CK_PR_N_Z(S, M, M, M)
 890
 891 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE)
 892
 893 #ifndef CK_F_PR_NOT_CHAR
 894 #define CK_F_PR_NOT_CHAR
 895 CK_PR_N_S(not, char, char, ~)
 896 #endif /* CK_F_PR_NOT_CHAR */
 897
 898 #ifndef CK_F_PR_NEG_CHAR
 899 #define CK_F_PR_NEG_CHAR
 900 CK_PR_N_S(neg, char, char, -)
 901 #endif /* CK_F_PR_NEG_CHAR */
 902
 903 #ifndef CK_F_PR_NEG_CHAR_ZERO
 904 #define CK_F_PR_NEG_CHAR_ZERO
 905 CK_PR_N_Z_S(char, char)
 906 #endif /* CK_F_PR_NEG_CHAR_ZERO */
 907
 908 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */
 909
 910 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
 911
 912 #ifndef CK_F_PR_NOT_INT
 913 #define CK_F_PR_NOT_INT
 914 CK_PR_N_S(not, int, int, ~)
 915 #endif /* CK_F_PR_NOT_INT */
 916
 917 #ifndef CK_F_PR_NEG_INT
 918 #define CK_F_PR_NEG_INT
 919 CK_PR_N_S(neg, int, int, -)
 920 #endif /* CK_F_PR_NEG_INT */
 921
 922 #ifndef CK_F_PR_NEG_INT_ZERO
 923 #define CK_F_PR_NEG_INT_ZERO
 924 CK_PR_N_Z_S(int, int)
 925 #endif /* CK_F_PR_NEG_INT_ZERO */
 926
 927 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
 928
 929 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \
 930             !defined(CK_PR_DISABLE_DOUBLE)
 931
 932 #ifndef CK_F_PR_NEG_DOUBLE
 933 #define CK_F_PR_NEG_DOUBLE
 934 CK_PR_N_S(neg, double, double, -)
 935 #endif /* CK_F_PR_NEG_DOUBLE */
 936
 937 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */
 938
 939 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
 940
 941 #ifndef CK_F_PR_NOT_UINT
 942 #define CK_F_PR_NOT_UINT
 943 CK_PR_N_S(not, uint, unsigned int, ~)
 944 #endif /* CK_F_PR_NOT_UINT */
 945
 946 #ifndef CK_F_PR_NEG_UINT
 947 #define CK_F_PR_NEG_UINT
 948 CK_PR_N_S(neg, uint, unsigned int, -)
 949 #endif /* CK_F_PR_NEG_UINT */
 950
 951 #ifndef CK_F_PR_NEG_UINT_ZERO
 952 #define CK_F_PR_NEG_UINT_ZERO
 953 CK_PR_N_Z_S(uint, unsigned int)
 954 #endif /* CK_F_PR_NEG_UINT_ZERO */
 955
 956 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
 957
 958 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
 959
 960 #ifndef CK_F_PR_NOT_PTR
 961 #define CK_F_PR_NOT_PTR
 962 CK_PR_N(not, ptr, void, uintptr_t, ~, void *)
 963 #endif /* CK_F_PR_NOT_PTR */
 964
 965 #ifndef CK_F_PR_NEG_PTR
 966 #define CK_F_PR_NEG_PTR
 967 CK_PR_N(neg, ptr, void, uintptr_t, -, void *)
 968 #endif /* CK_F_PR_NEG_PTR */
 969
 970 #ifndef CK_F_PR_NEG_PTR_ZERO
 971 #define CK_F_PR_NEG_PTR_ZERO
 972 CK_PR_N_Z(ptr, void, uintptr_t, void *)
 973 #endif /* CK_F_PR_NEG_PTR_ZERO */
 974
 975 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
 976
 977 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
 978
 979 #ifndef CK_F_PR_NOT_64
 980 #define CK_F_PR_NOT_64
 981 CK_PR_N_S(not, 64, uint64_t, ~)
 982 #endif /* CK_F_PR_NOT_64 */
 983
 984 #ifndef CK_F_PR_NEG_64
 985 #define CK_F_PR_NEG_64
 986 CK_PR_N_S(neg, 64, uint64_t, -)
 987 #endif /* CK_F_PR_NEG_64 */
 988
 989 #ifndef CK_F_PR_NEG_64_ZERO
 990 #define CK_F_PR_NEG_64_ZERO
 991 CK_PR_N_Z_S(64, uint64_t)
 992 #endif /* CK_F_PR_NEG_64_ZERO */
 993
 994 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
 995
 996 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
 997
 998 #ifndef CK_F_PR_NOT_32
 999 #define CK_F_PR_NOT_32
1000 CK_PR_N_S(not, 32, uint32_t, ~)
1001 #endif /* CK_F_PR_NOT_32 */
1002
1003 #ifndef CK_F_PR_NEG_32
1004 #define CK_F_PR_NEG_32
1005 CK_PR_N_S(neg, 32, uint32_t, -)
1006 #endif /* CK_F_PR_NEG_32 */
1007
1008 #ifndef CK_F_PR_NEG_32_ZERO
1009 #define CK_F_PR_NEG_32_ZERO
1010 CK_PR_N_Z_S(32, uint32_t)
1011 #endif /* CK_F_PR_NEG_32_ZERO */
1012
1013 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
1014
1015 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
1016
1017 #ifndef CK_F_PR_NOT_16
1018 #define CK_F_PR_NOT_16
1019 CK_PR_N_S(not, 16, uint16_t, ~)
1020 #endif /* CK_F_PR_NOT_16 */
1021
1022 #ifndef CK_F_PR_NEG_16
1023 #define CK_F_PR_NEG_16
1024 CK_PR_N_S(neg, 16, uint16_t, -)
1025 #endif /* CK_F_PR_NEG_16 */
1026
1027 #ifndef CK_F_PR_NEG_16_ZERO
1028 #define CK_F_PR_NEG_16_ZERO
1029 CK_PR_N_Z_S(16, uint16_t)
1030 #endif /* CK_F_PR_NEG_16_ZERO */
1031
1032 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
1033
1034 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE)
1035
1036 #ifndef CK_F_PR_NOT_8
1037 #define CK_F_PR_NOT_8
1038 CK_PR_N_S(not, 8, uint8_t, ~)
1039 #endif /* CK_F_PR_NOT_8 */
1040
1041 #ifndef CK_F_PR_NEG_8
1042 #define CK_F_PR_NEG_8
1043 CK_PR_N_S(neg, 8, uint8_t, -)
1044 #endif /* CK_F_PR_NEG_8 */
1045
1046 #ifndef CK_F_PR_NEG_8_ZERO
1047 #define CK_F_PR_NEG_8_ZERO
1048 CK_PR_N_Z_S(8, uint8_t)
1049 #endif /* CK_F_PR_NEG_8_ZERO */
1050
1051 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */
1052
1053 #undef CK_PR_N_Z_S
1054 #undef CK_PR_N_S
1055 #undef CK_PR_N_Z
1056 #undef CK_PR_N
1057
1058 #define CK_PR_FAA(S, M, T, C)                                           \
1059         CK_CC_INLINE static C                                           \
1060         ck_pr_faa_##S(M *target, T delta)                               \
1061         {                                                               \
1062                 T previous;                                             \
1063                 C punt;                                                 \
1064                 punt = (C)ck_pr_md_load_##S(target);                    \
1065                 previous = (T)punt;                                     \
1066                 while (ck_pr_cas_##S##_value(target,                    \
1067                                              (C)previous,               \
1068                                              (C)(previous + delta),     \
1069                                              &previous) == false)       \
1070                         ck_pr_stall();                                  \
1071                                                                         \
1072                 return ((C)previous);                                   \
1073         }
1074
1075 #define CK_PR_FAS(S, M, C)                                              \
1076         CK_CC_INLINE static C                                           \
1077         ck_pr_fas_##S(M *target, C update)                              \
1078         {                                                               \
1079                 C previous;                                             \
1080                 previous = ck_pr_md_load_##S(target);                   \
1081                 while (ck_pr_cas_##S##_value(target,                    \
1082                                              previous,                  \
1083                                              update,                    \
1084                                              &previous) == false)       \
1085                         ck_pr_stall();                                  \
1086                                                                         \
1087                 return (previous);                                      \
1088         }
1089
1090 #define CK_PR_FAA_S(S, M) CK_PR_FAA(S, M, M, M)
1091 #define CK_PR_FAS_S(S, M) CK_PR_FAS(S, M, M)
1092
1093 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE)
1094
1095 #ifndef CK_F_PR_FAA_CHAR
1096 #define CK_F_PR_FAA_CHAR
1097 CK_PR_FAA_S(char, char)
1098 #endif /* CK_F_PR_FAA_CHAR */
1099
1100 #ifndef CK_F_PR_FAS_CHAR
1101 #define CK_F_PR_FAS_CHAR
1102 CK_PR_FAS_S(char, char)
1103 #endif /* CK_F_PR_FAS_CHAR */
1104
1105 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */
1106
1107 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
1108
1109 #ifndef CK_F_PR_FAA_INT
1110 #define CK_F_PR_FAA_INT
1111 CK_PR_FAA_S(int, int)
1112 #endif /* CK_F_PR_FAA_INT */
1113
1114 #ifndef CK_F_PR_FAS_INT
1115 #define CK_F_PR_FAS_INT
1116 CK_PR_FAS_S(int, int)
1117 #endif /* CK_F_PR_FAS_INT */
1118
1119 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
1120
1121 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \
1122             !defined(CK_PR_DISABLE_DOUBLE)
1123
1124 #ifndef CK_F_PR_FAA_DOUBLE
1125 #define CK_F_PR_FAA_DOUBLE
1126 CK_PR_FAA_S(double, double)
1127 #endif /* CK_F_PR_FAA_DOUBLE */
1128
1129 #ifndef CK_F_PR_FAS_DOUBLE
1130 #define CK_F_PR_FAS_DOUBLE
1131 CK_PR_FAS_S(double, double)
1132 #endif /* CK_F_PR_FAS_DOUBLE */
1133
1134 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */
1135
1136 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
1137
1138 #ifndef CK_F_PR_FAA_UINT
1139 #define CK_F_PR_FAA_UINT
1140 CK_PR_FAA_S(uint, unsigned int)
1141 #endif /* CK_F_PR_FAA_UINT */
1142
1143 #ifndef CK_F_PR_FAS_UINT
1144 #define CK_F_PR_FAS_UINT
1145 CK_PR_FAS_S(uint, unsigned int)
1146 #endif /* CK_F_PR_FAS_UINT */
1147
1148 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
1149
1150 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
1151
1152 #ifndef CK_F_PR_FAA_PTR
1153 #define CK_F_PR_FAA_PTR
1154 CK_PR_FAA(ptr, void, uintptr_t, void *)
1155 #endif /* CK_F_PR_FAA_PTR */
1156
1157 #ifndef CK_F_PR_FAS_PTR
1158 #define CK_F_PR_FAS_PTR
1159 CK_PR_FAS(ptr, void, void *)
1160 #endif /* CK_F_PR_FAS_PTR */
1161
1162 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
1163
1164 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
1165
1166 #ifndef CK_F_PR_FAA_64
1167 #define CK_F_PR_FAA_64
1168 CK_PR_FAA_S(64, uint64_t)
1169 #endif /* CK_F_PR_FAA_64 */
1170
1171 #ifndef CK_F_PR_FAS_64
1172 #define CK_F_PR_FAS_64
1173 CK_PR_FAS_S(64, uint64_t)
1174 #endif /* CK_F_PR_FAS_64 */
1175
1176 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
1177
1178 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
1179
1180 #ifndef CK_F_PR_FAA_32
1181 #define CK_F_PR_FAA_32
1182 CK_PR_FAA_S(32, uint32_t)
1183 #endif /* CK_F_PR_FAA_32 */
1184
1185 #ifndef CK_F_PR_FAS_32
1186 #define CK_F_PR_FAS_32
1187 CK_PR_FAS_S(32, uint32_t)
1188 #endif /* CK_F_PR_FAS_32 */
1189
1190 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
1191
1192 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
1193
1194 #ifndef CK_F_PR_FAA_16
1195 #define CK_F_PR_FAA_16
1196 CK_PR_FAA_S(16, uint16_t)
1197 #endif /* CK_F_PR_FAA_16 */
1198
1199 #ifndef CK_F_PR_FAS_16
1200 #define CK_F_PR_FAS_16
1201 CK_PR_FAS_S(16, uint16_t)
1202 #endif /* CK_F_PR_FAS_16 */
1203
1204 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
1205
1206 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE)
1207
1208 #ifndef CK_F_PR_FAA_8
1209 #define CK_F_PR_FAA_8
1210 CK_PR_FAA_S(8, uint8_t)
1211 #endif /* CK_F_PR_FAA_8 */
1212
1213 #ifndef CK_F_PR_FAS_8
1214 #define CK_F_PR_FAS_8
1215 CK_PR_FAS_S(8, uint8_t)
1216 #endif /* CK_F_PR_FAS_8 */
1217
1218 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */
1219
1220 #undef CK_PR_FAA_S
1221 #undef CK_PR_FAS_S
1222 #undef CK_PR_FAA
1223 #undef CK_PR_FAS
1224
1225 #endif /* CK_PR_H */