sys/contrib/ck/include/ck_pr.h

   1 /*
   2  * Copyright 2009-2015 Samy Al Bahra.
   3  * Copyright 2011 David Joseph.
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #ifndef CK_PR_H
  29 #define CK_PR_H
  30
  31 #include <ck_cc.h>
  32 #include <ck_limits.h>
  33 #include <ck_md.h>
  34 #include <ck_stdint.h>
  35 #include <ck_stdbool.h>
  36
  37 #ifndef CK_USE_CC_BUILTINS
  38 #if defined(__x86_64__)
  39 #include "gcc/x86_64/ck_pr.h"
  40 #elif defined(__x86__)
  41 #include "gcc/x86/ck_pr.h"
  42 #elif defined(__sparcv9__)
  43 #include "gcc/sparcv9/ck_pr.h"
  44 #elif defined(__ppc64__)
  45 #include "gcc/ppc64/ck_pr.h"
  46 #elif defined(__ppc__)
  47 #include "gcc/ppc/ck_pr.h"
  48 #elif defined(__arm__)
  49 #if __ARM_ARCH >= 6
  50 #include "gcc/arm/ck_pr.h"
  51 #else
  52 #include "gcc/arm/ck_pr_armv4.h"
  53 #endif
  54 #elif defined(__aarch64__)
  55 #include "gcc/aarch64/ck_pr.h"
  56 #elif !defined(__GNUC__)
  57 #error Your platform is unsupported
  58 #endif
  59 #endif /* !CK_USE_CC_BUILTINS */
  60
  61 #if defined(__GNUC__)
  62 #include "gcc/ck_pr.h"
  63 #endif
  64
  65 #define CK_PR_FENCE_EMIT(T)                     \
  66         CK_CC_INLINE static void                \
  67         ck_pr_fence_##T(void)                   \
  68         {                                       \
  69                 ck_pr_fence_strict_##T();       \
  70                 return;                         \
  71         }
  72 #define CK_PR_FENCE_NOOP(T)                     \
  73         CK_CC_INLINE static void                \
  74         ck_pr_fence_##T(void)                   \
  75         {                                       \
  76                 ck_pr_barrier();                \
  77                 return;                         \
  78         }
  79
  80 /*
  81  * None of the currently supported platforms allow for data-dependent
  82  * load ordering.
  83  */
  84 CK_PR_FENCE_NOOP(load_depends)
  85 #define ck_pr_fence_strict_load_depends ck_pr_fence_load_depends
  86
  87 /*
  88  * In memory models where atomic operations do not have serializing
  89  * effects, atomic read-modify-write operations are modeled as stores.
  90  */
  91 #if defined(CK_MD_RMO)
  92 /*
  93  * Only stores to the same location have a global
  94  * ordering.
  95  */
  96 CK_PR_FENCE_EMIT(atomic)
  97 CK_PR_FENCE_EMIT(atomic_load)
  98 CK_PR_FENCE_EMIT(atomic_store)
  99 CK_PR_FENCE_EMIT(store_atomic)
 100 CK_PR_FENCE_EMIT(load_atomic)
 101 CK_PR_FENCE_EMIT(load_store)
 102 CK_PR_FENCE_EMIT(store_load)
 103 CK_PR_FENCE_EMIT(load)
 104 CK_PR_FENCE_EMIT(store)
 105 CK_PR_FENCE_EMIT(memory)
 106 CK_PR_FENCE_EMIT(acquire)
 107 CK_PR_FENCE_EMIT(release)
 108 CK_PR_FENCE_EMIT(acqrel)
 109 CK_PR_FENCE_EMIT(lock)
 110 CK_PR_FENCE_EMIT(unlock)
 111 #elif defined(CK_MD_PSO)
 112 /*
 113  * Anything can be re-ordered with respect to stores.
 114  * Otherwise, loads are executed in-order.
 115  */
 116 CK_PR_FENCE_EMIT(atomic)
 117 CK_PR_FENCE_NOOP(atomic_load)
 118 CK_PR_FENCE_EMIT(atomic_store)
 119 CK_PR_FENCE_EMIT(store_atomic)
 120 CK_PR_FENCE_NOOP(load_atomic)
 121 CK_PR_FENCE_EMIT(load_store)
 122 CK_PR_FENCE_EMIT(store_load)
 123 CK_PR_FENCE_NOOP(load)
 124 CK_PR_FENCE_EMIT(store)
 125 CK_PR_FENCE_EMIT(memory)
 126 CK_PR_FENCE_EMIT(acquire)
 127 CK_PR_FENCE_EMIT(release)
 128 CK_PR_FENCE_EMIT(acqrel)
 129 CK_PR_FENCE_EMIT(lock)
 130 CK_PR_FENCE_EMIT(unlock)
 131 #elif defined(CK_MD_TSO)
 132 /*
 133  * Only loads are re-ordered and only with respect to
 134  * prior stores. Atomic operations are serializing.
 135  */
 136 CK_PR_FENCE_NOOP(atomic)
 137 CK_PR_FENCE_NOOP(atomic_load)
 138 CK_PR_FENCE_NOOP(atomic_store)
 139 CK_PR_FENCE_NOOP(store_atomic)
 140 CK_PR_FENCE_NOOP(load_atomic)
 141 CK_PR_FENCE_NOOP(load_store)
 142 CK_PR_FENCE_EMIT(store_load)
 143 CK_PR_FENCE_NOOP(load)
 144 CK_PR_FENCE_NOOP(store)
 145 CK_PR_FENCE_EMIT(memory)
 146 CK_PR_FENCE_NOOP(acquire)
 147 CK_PR_FENCE_NOOP(release)
 148 CK_PR_FENCE_NOOP(acqrel)
 149 CK_PR_FENCE_NOOP(lock)
 150 CK_PR_FENCE_NOOP(unlock)
 151 #else
 152 #error "No memory model has been defined."
 153 #endif /* CK_MD_TSO */
 154
 155 #undef CK_PR_FENCE_EMIT
 156 #undef CK_PR_FENCE_NOOP
 157
 158 #ifndef CK_F_PR_RFO
 159 #define CK_F_PR_RFO
 160 CK_CC_INLINE static void
 161 ck_pr_rfo(const void *m)
 162 {
 163
 164         (void)m;
 165         return;
 166 }
 167 #endif /* CK_F_PR_RFO */
 168
 169 #define CK_PR_STORE_SAFE(DST, VAL, TYPE)                        \
 170     ck_pr_md_store_##TYPE(                                      \
 171         ((void)sizeof(*(DST) = (VAL)), (DST)),                  \
 172         (VAL))
 173
 174 #define ck_pr_store_ptr(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), ptr)
 175 #define ck_pr_store_char(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), char)
 176 #define ck_pr_store_double(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), double)
 177 #define ck_pr_store_uint(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), uint)
 178 #define ck_pr_store_int(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), int)
 179 #define ck_pr_store_32(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 32)
 180 #define ck_pr_store_16(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 16)
 181 #define ck_pr_store_8(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 8)
 182
 183 #define ck_pr_store_ptr_unsafe(DST, VAL) ck_pr_md_store_ptr((DST), (VAL))
 184
 185 #ifdef CK_F_PR_LOAD_64
 186 #define ck_pr_store_64(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 64)
 187 #endif /* CK_F_PR_LOAD_64 */
 188
 189 #define CK_PR_LOAD_PTR_SAFE(SRC) (CK_CC_TYPEOF(*(SRC), (void *)))ck_pr_md_load_ptr((SRC))
 190 #define ck_pr_load_ptr(SRC) CK_PR_LOAD_PTR_SAFE((SRC))
 191
 192 #define CK_PR_LOAD_SAFE(SRC, TYPE) ck_pr_md_load_##TYPE((SRC))
 193 #define ck_pr_load_char(SRC) CK_PR_LOAD_SAFE((SRC), char)
 194 #define ck_pr_load_double(SRC) CK_PR_LOAD_SAFE((SRC), double)
 195 #define ck_pr_load_uint(SRC) CK_PR_LOAD_SAFE((SRC), uint)
 196 #define ck_pr_load_int(SRC) CK_PR_LOAD_SAFE((SRC), int)
 197 #define ck_pr_load_32(SRC) CK_PR_LOAD_SAFE((SRC), 32)
 198 #define ck_pr_load_16(SRC) CK_PR_LOAD_SAFE((SRC), 16)
 199 #define ck_pr_load_8(SRC) CK_PR_LOAD_SAFE((SRC), 8)
 200
 201 #ifdef CK_F_PR_LOAD_64
 202 #define ck_pr_load_64(SRC) CK_PR_LOAD_SAFE((SRC), 64)
 203 #endif /* CK_F_PR_LOAD_64 */
 204
 205 #define CK_PR_BIN(K, S, M, T, P, C)                                     \
 206         CK_CC_INLINE static void                                        \
 207         ck_pr_##K##_##S(M *target, T value)                             \
 208         {                                                               \
 209                 T previous;                                             \
 210                 C punt;                                                 \
 211                 punt = ck_pr_md_load_##S(target);                       \
 212                 previous = (T)punt;                                     \
 213                 while (ck_pr_cas_##S##_value(target,                    \
 214                                              (C)previous,               \
 215                                              (C)(previous P value),     \
 216                                              &previous) == false)       \
 217                         ck_pr_stall();                                  \
 218                                                                         \
 219                 return;                                                 \
 220         }
 221
 222 #define CK_PR_BIN_S(K, S, T, P) CK_PR_BIN(K, S, T, T, P, T)
 223
 224 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE)
 225
 226 #ifndef CK_F_PR_ADD_CHAR
 227 #define CK_F_PR_ADD_CHAR
 228 CK_PR_BIN_S(add, char, char, +)
 229 #endif /* CK_F_PR_ADD_CHAR */
 230
 231 #ifndef CK_F_PR_SUB_CHAR
 232 #define CK_F_PR_SUB_CHAR
 233 CK_PR_BIN_S(sub, char, char, -)
 234 #endif /* CK_F_PR_SUB_CHAR */
 235
 236 #ifndef CK_F_PR_AND_CHAR
 237 #define CK_F_PR_AND_CHAR
 238 CK_PR_BIN_S(and, char, char, &)
 239 #endif /* CK_F_PR_AND_CHAR */
 240
 241 #ifndef CK_F_PR_XOR_CHAR
 242 #define CK_F_PR_XOR_CHAR
 243 CK_PR_BIN_S(xor, char, char, ^)
 244 #endif /* CK_F_PR_XOR_CHAR */
 245
 246 #ifndef CK_F_PR_OR_CHAR
 247 #define CK_F_PR_OR_CHAR
 248 CK_PR_BIN_S(or, char, char, |)
 249 #endif /* CK_F_PR_OR_CHAR */
 250
 251 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */
 252
 253 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
 254
 255 #ifndef CK_F_PR_ADD_INT
 256 #define CK_F_PR_ADD_INT
 257 CK_PR_BIN_S(add, int, int, +)
 258 #endif /* CK_F_PR_ADD_INT */
 259
 260 #ifndef CK_F_PR_SUB_INT
 261 #define CK_F_PR_SUB_INT
 262 CK_PR_BIN_S(sub, int, int, -)
 263 #endif /* CK_F_PR_SUB_INT */
 264
 265 #ifndef CK_F_PR_AND_INT
 266 #define CK_F_PR_AND_INT
 267 CK_PR_BIN_S(and, int, int, &)
 268 #endif /* CK_F_PR_AND_INT */
 269
 270 #ifndef CK_F_PR_XOR_INT
 271 #define CK_F_PR_XOR_INT
 272 CK_PR_BIN_S(xor, int, int, ^)
 273 #endif /* CK_F_PR_XOR_INT */
 274
 275 #ifndef CK_F_PR_OR_INT
 276 #define CK_F_PR_OR_INT
 277 CK_PR_BIN_S(or, int, int, |)
 278 #endif /* CK_F_PR_OR_INT */
 279
 280 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
 281
 282 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE)
 283
 284 #ifndef CK_F_PR_ADD_DOUBLE
 285 #define CK_F_PR_ADD_DOUBLE
 286 CK_PR_BIN_S(add, double, double, +)
 287 #endif /* CK_F_PR_ADD_DOUBLE */
 288
 289 #ifndef CK_F_PR_SUB_DOUBLE
 290 #define CK_F_PR_SUB_DOUBLE
 291 CK_PR_BIN_S(sub, double, double, -)
 292 #endif /* CK_F_PR_SUB_DOUBLE */
 293
 294 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE */
 295
 296 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
 297
 298 #ifndef CK_F_PR_ADD_UINT
 299 #define CK_F_PR_ADD_UINT
 300 CK_PR_BIN_S(add, uint, unsigned int, +)
 301 #endif /* CK_F_PR_ADD_UINT */
 302
 303 #ifndef CK_F_PR_SUB_UINT
 304 #define CK_F_PR_SUB_UINT
 305 CK_PR_BIN_S(sub, uint, unsigned int, -)
 306 #endif /* CK_F_PR_SUB_UINT */
 307
 308 #ifndef CK_F_PR_AND_UINT
 309 #define CK_F_PR_AND_UINT
 310 CK_PR_BIN_S(and, uint, unsigned int, &)
 311 #endif /* CK_F_PR_AND_UINT */
 312
 313 #ifndef CK_F_PR_XOR_UINT
 314 #define CK_F_PR_XOR_UINT
 315 CK_PR_BIN_S(xor, uint, unsigned int, ^)
 316 #endif /* CK_F_PR_XOR_UINT */
 317
 318 #ifndef CK_F_PR_OR_UINT
 319 #define CK_F_PR_OR_UINT
 320 CK_PR_BIN_S(or, uint, unsigned int, |)
 321 #endif /* CK_F_PR_OR_UINT */
 322
 323 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
 324
 325 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
 326
 327 #ifndef CK_F_PR_ADD_PTR
 328 #define CK_F_PR_ADD_PTR
 329 CK_PR_BIN(add, ptr, void, uintptr_t, +, void *)
 330 #endif /* CK_F_PR_ADD_PTR */
 331
 332 #ifndef CK_F_PR_SUB_PTR
 333 #define CK_F_PR_SUB_PTR
 334 CK_PR_BIN(sub, ptr, void, uintptr_t, -, void *)
 335 #endif /* CK_F_PR_SUB_PTR */
 336
 337 #ifndef CK_F_PR_AND_PTR
 338 #define CK_F_PR_AND_PTR
 339 CK_PR_BIN(and, ptr, void, uintptr_t, &, void *)
 340 #endif /* CK_F_PR_AND_PTR */
 341
 342 #ifndef CK_F_PR_XOR_PTR
 343 #define CK_F_PR_XOR_PTR
 344 CK_PR_BIN(xor, ptr, void, uintptr_t, ^, void *)
 345 #endif /* CK_F_PR_XOR_PTR */
 346
 347 #ifndef CK_F_PR_OR_PTR
 348 #define CK_F_PR_OR_PTR
 349 CK_PR_BIN(or, ptr, void, uintptr_t, |, void *)
 350 #endif /* CK_F_PR_OR_PTR */
 351
 352 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
 353
 354 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
 355
 356 #ifndef CK_F_PR_ADD_64
 357 #define CK_F_PR_ADD_64
 358 CK_PR_BIN_S(add, 64, uint64_t, +)
 359 #endif /* CK_F_PR_ADD_64 */
 360
 361 #ifndef CK_F_PR_SUB_64
 362 #define CK_F_PR_SUB_64
 363 CK_PR_BIN_S(sub, 64, uint64_t, -)
 364 #endif /* CK_F_PR_SUB_64 */
 365
 366 #ifndef CK_F_PR_AND_64
 367 #define CK_F_PR_AND_64
 368 CK_PR_BIN_S(and, 64, uint64_t, &)
 369 #endif /* CK_F_PR_AND_64 */
 370
 371 #ifndef CK_F_PR_XOR_64
 372 #define CK_F_PR_XOR_64
 373 CK_PR_BIN_S(xor, 64, uint64_t, ^)
 374 #endif /* CK_F_PR_XOR_64 */
 375
 376 #ifndef CK_F_PR_OR_64
 377 #define CK_F_PR_OR_64
 378 CK_PR_BIN_S(or, 64, uint64_t, |)
 379 #endif /* CK_F_PR_OR_64 */
 380
 381 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
 382
 383 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
 384
 385 #ifndef CK_F_PR_ADD_32
 386 #define CK_F_PR_ADD_32
 387 CK_PR_BIN_S(add, 32, uint32_t, +)
 388 #endif /* CK_F_PR_ADD_32 */
 389
 390 #ifndef CK_F_PR_SUB_32
 391 #define CK_F_PR_SUB_32
 392 CK_PR_BIN_S(sub, 32, uint32_t, -)
 393 #endif /* CK_F_PR_SUB_32 */
 394
 395 #ifndef CK_F_PR_AND_32
 396 #define CK_F_PR_AND_32
 397 CK_PR_BIN_S(and, 32, uint32_t, &)
 398 #endif /* CK_F_PR_AND_32 */
 399
 400 #ifndef CK_F_PR_XOR_32
 401 #define CK_F_PR_XOR_32
 402 CK_PR_BIN_S(xor, 32, uint32_t, ^)
 403 #endif /* CK_F_PR_XOR_32 */
 404
 405 #ifndef CK_F_PR_OR_32
 406 #define CK_F_PR_OR_32
 407 CK_PR_BIN_S(or, 32, uint32_t, |)
 408 #endif /* CK_F_PR_OR_32 */
 409
 410 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
 411
 412 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
 413
 414 #ifndef CK_F_PR_ADD_16
 415 #define CK_F_PR_ADD_16
 416 CK_PR_BIN_S(add, 16, uint16_t, +)
 417 #endif /* CK_F_PR_ADD_16 */
 418
 419 #ifndef CK_F_PR_SUB_16
 420 #define CK_F_PR_SUB_16
 421 CK_PR_BIN_S(sub, 16, uint16_t, -)
 422 #endif /* CK_F_PR_SUB_16 */
 423
 424 #ifndef CK_F_PR_AND_16
 425 #define CK_F_PR_AND_16
 426 CK_PR_BIN_S(and, 16, uint16_t, &)
 427 #endif /* CK_F_PR_AND_16 */
 428
 429 #ifndef CK_F_PR_XOR_16
 430 #define CK_F_PR_XOR_16
 431 CK_PR_BIN_S(xor, 16, uint16_t, ^)
 432 #endif /* CK_F_PR_XOR_16 */
 433
 434 #ifndef CK_F_PR_OR_16
 435 #define CK_F_PR_OR_16
 436 CK_PR_BIN_S(or, 16, uint16_t, |)
 437 #endif /* CK_F_PR_OR_16 */
 438
 439 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
 440
 441 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE)
 442
 443 #ifndef CK_F_PR_ADD_8
 444 #define CK_F_PR_ADD_8
 445 CK_PR_BIN_S(add, 8, uint8_t, +)
 446 #endif /* CK_F_PR_ADD_8 */
 447
 448 #ifndef CK_F_PR_SUB_8
 449 #define CK_F_PR_SUB_8
 450 CK_PR_BIN_S(sub, 8, uint8_t, -)
 451 #endif /* CK_F_PR_SUB_8 */
 452
 453 #ifndef CK_F_PR_AND_8
 454 #define CK_F_PR_AND_8
 455 CK_PR_BIN_S(and, 8, uint8_t, &)
 456 #endif /* CK_F_PR_AND_8 */
 457
 458 #ifndef CK_F_PR_XOR_8
 459 #define CK_F_PR_XOR_8
 460 CK_PR_BIN_S(xor, 8, uint8_t, ^)
 461 #endif /* CK_F_PR_XOR_8 */
 462
 463 #ifndef CK_F_PR_OR_8
 464 #define CK_F_PR_OR_8
 465 CK_PR_BIN_S(or, 8, uint8_t, |)
 466 #endif /* CK_F_PR_OR_8 */
 467
 468 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */
 469
 470 #undef CK_PR_BIN_S
 471 #undef CK_PR_BIN
 472
 473 #define CK_PR_BTX(K, S, M, T, P, C, R)                                             \
 474         CK_CC_INLINE static bool                                                   \
 475         ck_pr_##K##_##S(M *target, unsigned int offset)                            \
 476         {                                                                          \
 477                 T previous;                                                        \
 478                 C punt;                                                            \
 479                 punt = ck_pr_md_load_##S(target);                                  \
 480                 previous = (T)punt;                                                \
 481                 while (ck_pr_cas_##S##_value(target, (C)previous,                  \
 482                         (C)(previous P (R ((T)1 << offset))), &previous) == false) \
 483                                 ck_pr_stall();                                     \
 484                 return ((previous >> offset) & 1);                                 \
 485         }
 486
 487 #define CK_PR_BTX_S(K, S, T, P, R) CK_PR_BTX(K, S, T, T, P, T, R)
 488
 489 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
 490
 491 #ifndef CK_F_PR_BTC_INT
 492 #define CK_F_PR_BTC_INT
 493 CK_PR_BTX_S(btc, int, int, ^,)
 494 #endif /* CK_F_PR_BTC_INT */
 495
 496 #ifndef CK_F_PR_BTR_INT
 497 #define CK_F_PR_BTR_INT
 498 CK_PR_BTX_S(btr, int, int, &, ~)
 499 #endif /* CK_F_PR_BTR_INT */
 500
 501 #ifndef CK_F_PR_BTS_INT
 502 #define CK_F_PR_BTS_INT
 503 CK_PR_BTX_S(bts, int, int, |,)
 504 #endif /* CK_F_PR_BTS_INT */
 505
 506 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
 507
 508 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
 509
 510 #ifndef CK_F_PR_BTC_UINT
 511 #define CK_F_PR_BTC_UINT
 512 CK_PR_BTX_S(btc, uint, unsigned int, ^,)
 513 #endif /* CK_F_PR_BTC_UINT */
 514
 515 #ifndef CK_F_PR_BTR_UINT
 516 #define CK_F_PR_BTR_UINT
 517 CK_PR_BTX_S(btr, uint, unsigned int, &, ~)
 518 #endif /* CK_F_PR_BTR_UINT */
 519
 520 #ifndef CK_F_PR_BTS_UINT
 521 #define CK_F_PR_BTS_UINT
 522 CK_PR_BTX_S(bts, uint, unsigned int, |,)
 523 #endif /* CK_F_PR_BTS_UINT */
 524
 525 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
 526
 527 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
 528
 529 #ifndef CK_F_PR_BTC_PTR
 530 #define CK_F_PR_BTC_PTR
 531 CK_PR_BTX(btc, ptr, void, uintptr_t, ^, void *,)
 532 #endif /* CK_F_PR_BTC_PTR */
 533
 534 #ifndef CK_F_PR_BTR_PTR
 535 #define CK_F_PR_BTR_PTR
 536 CK_PR_BTX(btr, ptr, void, uintptr_t, &, void *, ~)
 537 #endif /* CK_F_PR_BTR_PTR */
 538
 539 #ifndef CK_F_PR_BTS_PTR
 540 #define CK_F_PR_BTS_PTR
 541 CK_PR_BTX(bts, ptr, void, uintptr_t, |, void *,)
 542 #endif /* CK_F_PR_BTS_PTR */
 543
 544 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
 545
 546 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
 547
 548 #ifndef CK_F_PR_BTC_64
 549 #define CK_F_PR_BTC_64
 550 CK_PR_BTX_S(btc, 64, uint64_t, ^,)
 551 #endif /* CK_F_PR_BTC_64 */
 552
 553 #ifndef CK_F_PR_BTR_64
 554 #define CK_F_PR_BTR_64
 555 CK_PR_BTX_S(btr, 64, uint64_t, &, ~)
 556 #endif /* CK_F_PR_BTR_64 */
 557
 558 #ifndef CK_F_PR_BTS_64
 559 #define CK_F_PR_BTS_64
 560 CK_PR_BTX_S(bts, 64, uint64_t, |,)
 561 #endif /* CK_F_PR_BTS_64 */
 562
 563 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
 564
 565 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
 566
 567 #ifndef CK_F_PR_BTC_32
 568 #define CK_F_PR_BTC_32
 569 CK_PR_BTX_S(btc, 32, uint32_t, ^,)
 570 #endif /* CK_F_PR_BTC_32 */
 571
 572 #ifndef CK_F_PR_BTR_32
 573 #define CK_F_PR_BTR_32
 574 CK_PR_BTX_S(btr, 32, uint32_t, &, ~)
 575 #endif /* CK_F_PR_BTR_32 */
 576
 577 #ifndef CK_F_PR_BTS_32
 578 #define CK_F_PR_BTS_32
 579 CK_PR_BTX_S(bts, 32, uint32_t, |,)
 580 #endif /* CK_F_PR_BTS_32 */
 581
 582 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
 583
 584 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
 585
 586 #ifndef CK_F_PR_BTC_16
 587 #define CK_F_PR_BTC_16
 588 CK_PR_BTX_S(btc, 16, uint16_t, ^,)
 589 #endif /* CK_F_PR_BTC_16 */
 590
 591 #ifndef CK_F_PR_BTR_16
 592 #define CK_F_PR_BTR_16
 593 CK_PR_BTX_S(btr, 16, uint16_t, &, ~)
 594 #endif /* CK_F_PR_BTR_16 */
 595
 596 #ifndef CK_F_PR_BTS_16
 597 #define CK_F_PR_BTS_16
 598 CK_PR_BTX_S(bts, 16, uint16_t, |,)
 599 #endif /* CK_F_PR_BTS_16 */
 600
 601 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
 602
 603 #undef CK_PR_BTX_S
 604 #undef CK_PR_BTX
 605
 606 #define CK_PR_UNARY(K, X, S, M, T)                                      \
 607         CK_CC_INLINE static void                                        \
 608         ck_pr_##K##_##S(M *target)                                      \
 609         {                                                               \
 610                 ck_pr_##X##_##S(target, (T)1);                          \
 611                 return;                                                 \
 612         }
 613
 614 #define CK_PR_UNARY_Z(K, S, M, T, P, C, Z)                              \
 615         CK_CC_INLINE static void                                        \
 616         ck_pr_##K##_##S##_zero(M *target, bool *zero)                   \
 617         {                                                               \
 618                 T previous;                                             \
 619                 C punt;                                                 \
 620                 punt = (C)ck_pr_md_load_##S(target);                    \
 621                 previous = (T)punt;                                     \
 622                 while (ck_pr_cas_##S##_value(target,                    \
 623                                              (C)previous,               \
 624                                              (C)(previous P 1),         \
 625                                              &previous) == false)       \
 626                         ck_pr_stall();                                  \
 627                 *zero = previous == (T)Z;                               \
 628                 return;                                                 \
 629         }
 630
 631 #define CK_PR_UNARY_S(K, X, S, M) CK_PR_UNARY(K, X, S, M, M)
 632 #define CK_PR_UNARY_Z_S(K, S, M, P, Z) CK_PR_UNARY_Z(K, S, M, M, P, M, Z)
 633
 634 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE)
 635
 636 #ifndef CK_F_PR_INC_CHAR
 637 #define CK_F_PR_INC_CHAR
 638 CK_PR_UNARY_S(inc, add, char, char)
 639 #endif /* CK_F_PR_INC_CHAR */
 640
 641 #ifndef CK_F_PR_INC_CHAR_ZERO
 642 #define CK_F_PR_INC_CHAR_ZERO
 643 CK_PR_UNARY_Z_S(inc, char, char, +, -1)
 644 #endif /* CK_F_PR_INC_CHAR_ZERO */
 645
 646 #ifndef CK_F_PR_DEC_CHAR
 647 #define CK_F_PR_DEC_CHAR
 648 CK_PR_UNARY_S(dec, sub, char, char)
 649 #endif /* CK_F_PR_DEC_CHAR */
 650
 651 #ifndef CK_F_PR_DEC_CHAR_ZERO
 652 #define CK_F_PR_DEC_CHAR_ZERO
 653 CK_PR_UNARY_Z_S(dec, char, char, -, 1)
 654 #endif /* CK_F_PR_DEC_CHAR_ZERO */
 655
 656 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */
 657
 658 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
 659
 660 #ifndef CK_F_PR_INC_INT
 661 #define CK_F_PR_INC_INT
 662 CK_PR_UNARY_S(inc, add, int, int)
 663 #endif /* CK_F_PR_INC_INT */
 664
 665 #ifndef CK_F_PR_INC_INT_ZERO
 666 #define CK_F_PR_INC_INT_ZERO
 667 CK_PR_UNARY_Z_S(inc, int, int, +, -1)
 668 #endif /* CK_F_PR_INC_INT_ZERO */
 669
 670 #ifndef CK_F_PR_DEC_INT
 671 #define CK_F_PR_DEC_INT
 672 CK_PR_UNARY_S(dec, sub, int, int)
 673 #endif /* CK_F_PR_DEC_INT */
 674
 675 #ifndef CK_F_PR_DEC_INT_ZERO
 676 #define CK_F_PR_DEC_INT_ZERO
 677 CK_PR_UNARY_Z_S(dec, int, int, -, 1)
 678 #endif /* CK_F_PR_DEC_INT_ZERO */
 679
 680 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
 681
 682 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE)
 683
 684 #ifndef CK_F_PR_INC_DOUBLE
 685 #define CK_F_PR_INC_DOUBLE
 686 CK_PR_UNARY_S(inc, add, double, double)
 687 #endif /* CK_F_PR_INC_DOUBLE */
 688
 689 #ifndef CK_F_PR_DEC_DOUBLE
 690 #define CK_F_PR_DEC_DOUBLE
 691 CK_PR_UNARY_S(dec, sub, double, double)
 692 #endif /* CK_F_PR_DEC_DOUBLE */
 693
 694 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE */
 695
 696 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
 697
 698 #ifndef CK_F_PR_INC_UINT
 699 #define CK_F_PR_INC_UINT
 700 CK_PR_UNARY_S(inc, add, uint, unsigned int)
 701 #endif /* CK_F_PR_INC_UINT */
 702
 703 #ifndef CK_F_PR_INC_UINT_ZERO
 704 #define CK_F_PR_INC_UINT_ZERO
 705 CK_PR_UNARY_Z_S(inc, uint, unsigned int, +, UINT_MAX)
 706 #endif /* CK_F_PR_INC_UINT_ZERO */
 707
 708 #ifndef CK_F_PR_DEC_UINT
 709 #define CK_F_PR_DEC_UINT
 710 CK_PR_UNARY_S(dec, sub, uint, unsigned int)
 711 #endif /* CK_F_PR_DEC_UINT */
 712
 713 #ifndef CK_F_PR_DEC_UINT_ZERO
 714 #define CK_F_PR_DEC_UINT_ZERO
 715 CK_PR_UNARY_Z_S(dec, uint, unsigned int, -, 1)
 716 #endif /* CK_F_PR_DEC_UINT_ZERO */
 717
 718 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
 719
 720 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
 721
 722 #ifndef CK_F_PR_INC_PTR
 723 #define CK_F_PR_INC_PTR
 724 CK_PR_UNARY(inc, add, ptr, void, uintptr_t)
 725 #endif /* CK_F_PR_INC_PTR */
 726
 727 #ifndef CK_F_PR_INC_PTR_ZERO
 728 #define CK_F_PR_INC_PTR_ZERO
 729 CK_PR_UNARY_Z(inc, ptr, void, uintptr_t, +, void *, UINT_MAX)
 730 #endif /* CK_F_PR_INC_PTR_ZERO */
 731
 732 #ifndef CK_F_PR_DEC_PTR
 733 #define CK_F_PR_DEC_PTR
 734 CK_PR_UNARY(dec, sub, ptr, void, uintptr_t)
 735 #endif /* CK_F_PR_DEC_PTR */
 736
 737 #ifndef CK_F_PR_DEC_PTR_ZERO
 738 #define CK_F_PR_DEC_PTR_ZERO
 739 CK_PR_UNARY_Z(dec, ptr, void, uintptr_t, -, void *, 1)
 740 #endif /* CK_F_PR_DEC_PTR_ZERO */
 741
 742 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
 743
 744 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
 745
 746 #ifndef CK_F_PR_INC_64
 747 #define CK_F_PR_INC_64
 748 CK_PR_UNARY_S(inc, add, 64, uint64_t)
 749 #endif /* CK_F_PR_INC_64 */
 750
 751 #ifndef CK_F_PR_INC_64_ZERO
 752 #define CK_F_PR_INC_64_ZERO
 753 CK_PR_UNARY_Z_S(inc, 64, uint64_t, +, UINT64_MAX)
 754 #endif /* CK_F_PR_INC_64_ZERO */
 755
 756 #ifndef CK_F_PR_DEC_64
 757 #define CK_F_PR_DEC_64
 758 CK_PR_UNARY_S(dec, sub, 64, uint64_t)
 759 #endif /* CK_F_PR_DEC_64 */
 760
 761 #ifndef CK_F_PR_DEC_64_ZERO
 762 #define CK_F_PR_DEC_64_ZERO
 763 CK_PR_UNARY_Z_S(dec, 64, uint64_t, -, 1)
 764 #endif /* CK_F_PR_DEC_64_ZERO */
 765
 766 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
 767
 768 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
 769
 770 #ifndef CK_F_PR_INC_32
 771 #define CK_F_PR_INC_32
 772 CK_PR_UNARY_S(inc, add, 32, uint32_t)
 773 #endif /* CK_F_PR_INC_32 */
 774
 775 #ifndef CK_F_PR_INC_32_ZERO
 776 #define CK_F_PR_INC_32_ZERO
 777 CK_PR_UNARY_Z_S(inc, 32, uint32_t, +, UINT32_MAX)
 778 #endif /* CK_F_PR_INC_32_ZERO */
 779
 780 #ifndef CK_F_PR_DEC_32
 781 #define CK_F_PR_DEC_32
 782 CK_PR_UNARY_S(dec, sub, 32, uint32_t)
 783 #endif /* CK_F_PR_DEC_32 */
 784
 785 #ifndef CK_F_PR_DEC_32_ZERO
 786 #define CK_F_PR_DEC_32_ZERO
 787 CK_PR_UNARY_Z_S(dec, 32, uint32_t, -, 1)
 788 #endif /* CK_F_PR_DEC_32_ZERO */
 789
 790 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
 791
 792 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
 793
 794 #ifndef CK_F_PR_INC_16
 795 #define CK_F_PR_INC_16
 796 CK_PR_UNARY_S(inc, add, 16, uint16_t)
 797 #endif /* CK_F_PR_INC_16 */
 798
 799 #ifndef CK_F_PR_INC_16_ZERO
 800 #define CK_F_PR_INC_16_ZERO
 801 CK_PR_UNARY_Z_S(inc, 16, uint16_t, +, UINT16_MAX)
 802 #endif /* CK_F_PR_INC_16_ZERO */
 803
 804 #ifndef CK_F_PR_DEC_16
 805 #define CK_F_PR_DEC_16
 806 CK_PR_UNARY_S(dec, sub, 16, uint16_t)
 807 #endif /* CK_F_PR_DEC_16 */
 808
 809 #ifndef CK_F_PR_DEC_16_ZERO
 810 #define CK_F_PR_DEC_16_ZERO
 811 CK_PR_UNARY_Z_S(dec, 16, uint16_t, -, 1)
 812 #endif /* CK_F_PR_DEC_16_ZERO */
 813
 814 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
 815
 816 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE)
 817
 818 #ifndef CK_F_PR_INC_8
 819 #define CK_F_PR_INC_8
 820 CK_PR_UNARY_S(inc, add, 8, uint8_t)
 821 #endif /* CK_F_PR_INC_8 */
 822
 823 #ifndef CK_F_PR_INC_8_ZERO
 824 #define CK_F_PR_INC_8_ZERO
 825 CK_PR_UNARY_Z_S(inc, 8, uint8_t, +, UINT8_MAX)
 826 #endif /* CK_F_PR_INC_8_ZERO */
 827
 828 #ifndef CK_F_PR_DEC_8
 829 #define CK_F_PR_DEC_8
 830 CK_PR_UNARY_S(dec, sub, 8, uint8_t)
 831 #endif /* CK_F_PR_DEC_8 */
 832
 833 #ifndef CK_F_PR_DEC_8_ZERO
 834 #define CK_F_PR_DEC_8_ZERO
 835 CK_PR_UNARY_Z_S(dec, 8, uint8_t, -, 1)
 836 #endif /* CK_F_PR_DEC_8_ZERO */
 837
 838 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */
 839
 840 #undef CK_PR_UNARY_Z_S
 841 #undef CK_PR_UNARY_S
 842 #undef CK_PR_UNARY_Z
 843 #undef CK_PR_UNARY
 844
 845 #define CK_PR_N(K, S, M, T, P, C)                                       \
 846         CK_CC_INLINE static void                                        \
 847         ck_pr_##K##_##S(M *target)                                      \
 848         {                                                               \
 849                 T previous;                                             \
 850                 C punt;                                                 \
 851                 punt = (C)ck_pr_md_load_##S(target);                    \
 852                 previous = (T)punt;                                     \
 853                 while (ck_pr_cas_##S##_value(target,                    \
 854                                              (C)previous,               \
 855                                              (C)(P previous),           \
 856                                              &previous) == false)       \
 857                         ck_pr_stall();                                  \
 858                                                                         \
 859                 return;                                                 \
 860         }
 861
 862 #define CK_PR_N_Z(S, M, T, C)                                           \
 863         CK_CC_INLINE static void                                        \
 864         ck_pr_neg_##S##_zero(M *target, bool *zero)                     \
 865         {                                                               \
 866                 T previous;                                             \
 867                 C punt;                                                 \
 868                 punt = (C)ck_pr_md_load_##S(target);                    \
 869                 previous = (T)punt;                                     \
 870                 while (ck_pr_cas_##S##_value(target,                    \
 871                                              (C)previous,               \
 872                                              (C)(-previous),            \
 873                                              &previous) == false)       \
 874                         ck_pr_stall();                                  \
 875                                                                         \
 876                 *zero = previous == 0;                                  \
 877                 return;                                                 \
 878         }
 879
 880 #define CK_PR_N_S(K, S, M, P)   CK_PR_N(K, S, M, M, P, M)
 881 #define CK_PR_N_Z_S(S, M)       CK_PR_N_Z(S, M, M, M)
 882
 883 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE)
 884
 885 #ifndef CK_F_PR_NOT_CHAR
 886 #define CK_F_PR_NOT_CHAR
 887 CK_PR_N_S(not, char, char, ~)
 888 #endif /* CK_F_PR_NOT_CHAR */
 889
 890 #ifndef CK_F_PR_NEG_CHAR
 891 #define CK_F_PR_NEG_CHAR
 892 CK_PR_N_S(neg, char, char, -)
 893 #endif /* CK_F_PR_NEG_CHAR */
 894
 895 #ifndef CK_F_PR_NEG_CHAR_ZERO
 896 #define CK_F_PR_NEG_CHAR_ZERO
 897 CK_PR_N_Z_S(char, char)
 898 #endif /* CK_F_PR_NEG_CHAR_ZERO */
 899
 900 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */
 901
 902 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
 903
 904 #ifndef CK_F_PR_NOT_INT
 905 #define CK_F_PR_NOT_INT
 906 CK_PR_N_S(not, int, int, ~)
 907 #endif /* CK_F_PR_NOT_INT */
 908
 909 #ifndef CK_F_PR_NEG_INT
 910 #define CK_F_PR_NEG_INT
 911 CK_PR_N_S(neg, int, int, -)
 912 #endif /* CK_F_PR_NEG_INT */
 913
 914 #ifndef CK_F_PR_NEG_INT_ZERO
 915 #define CK_F_PR_NEG_INT_ZERO
 916 CK_PR_N_Z_S(int, int)
 917 #endif /* CK_F_PR_NEG_INT_ZERO */
 918
 919 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
 920
 921 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE)
 922
 923 #ifndef CK_F_PR_NEG_DOUBLE
 924 #define CK_F_PR_NEG_DOUBLE
 925 CK_PR_N_S(neg, double, double, -)
 926 #endif /* CK_F_PR_NEG_DOUBLE */
 927
 928 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE */
 929
 930 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
 931
 932 #ifndef CK_F_PR_NOT_UINT
 933 #define CK_F_PR_NOT_UINT
 934 CK_PR_N_S(not, uint, unsigned int, ~)
 935 #endif /* CK_F_PR_NOT_UINT */
 936
 937 #ifndef CK_F_PR_NEG_UINT
 938 #define CK_F_PR_NEG_UINT
 939 CK_PR_N_S(neg, uint, unsigned int, -)
 940 #endif /* CK_F_PR_NEG_UINT */
 941
 942 #ifndef CK_F_PR_NEG_UINT_ZERO
 943 #define CK_F_PR_NEG_UINT_ZERO
 944 CK_PR_N_Z_S(uint, unsigned int)
 945 #endif /* CK_F_PR_NEG_UINT_ZERO */
 946
 947 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
 948
 949 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
 950
 951 #ifndef CK_F_PR_NOT_PTR
 952 #define CK_F_PR_NOT_PTR
 953 CK_PR_N(not, ptr, void, uintptr_t, ~, void *)
 954 #endif /* CK_F_PR_NOT_PTR */
 955
 956 #ifndef CK_F_PR_NEG_PTR
 957 #define CK_F_PR_NEG_PTR
 958 CK_PR_N(neg, ptr, void, uintptr_t, -, void *)
 959 #endif /* CK_F_PR_NEG_PTR */
 960
 961 #ifndef CK_F_PR_NEG_PTR_ZERO
 962 #define CK_F_PR_NEG_PTR_ZERO
 963 CK_PR_N_Z(ptr, void, uintptr_t, void *)
 964 #endif /* CK_F_PR_NEG_PTR_ZERO */
 965
 966 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
 967
 968 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
 969
 970 #ifndef CK_F_PR_NOT_64
 971 #define CK_F_PR_NOT_64
 972 CK_PR_N_S(not, 64, uint64_t, ~)
 973 #endif /* CK_F_PR_NOT_64 */
 974
 975 #ifndef CK_F_PR_NEG_64
 976 #define CK_F_PR_NEG_64
 977 CK_PR_N_S(neg, 64, uint64_t, -)
 978 #endif /* CK_F_PR_NEG_64 */
 979
 980 #ifndef CK_F_PR_NEG_64_ZERO
 981 #define CK_F_PR_NEG_64_ZERO
 982 CK_PR_N_Z_S(64, uint64_t)
 983 #endif /* CK_F_PR_NEG_64_ZERO */
 984
 985 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
 986
 987 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
 988
 989 #ifndef CK_F_PR_NOT_32
 990 #define CK_F_PR_NOT_32
 991 CK_PR_N_S(not, 32, uint32_t, ~)
 992 #endif /* CK_F_PR_NOT_32 */
 993
 994 #ifndef CK_F_PR_NEG_32
 995 #define CK_F_PR_NEG_32
 996 CK_PR_N_S(neg, 32, uint32_t, -)
 997 #endif /* CK_F_PR_NEG_32 */
 998
 999 #ifndef CK_F_PR_NEG_32_ZERO
1000 #define CK_F_PR_NEG_32_ZERO
1001 CK_PR_N_Z_S(32, uint32_t)
1002 #endif /* CK_F_PR_NEG_32_ZERO */
1003
1004 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
1005
1006 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
1007
1008 #ifndef CK_F_PR_NOT_16
1009 #define CK_F_PR_NOT_16
1010 CK_PR_N_S(not, 16, uint16_t, ~)
1011 #endif /* CK_F_PR_NOT_16 */
1012
1013 #ifndef CK_F_PR_NEG_16
1014 #define CK_F_PR_NEG_16
1015 CK_PR_N_S(neg, 16, uint16_t, -)
1016 #endif /* CK_F_PR_NEG_16 */
1017
1018 #ifndef CK_F_PR_NEG_16_ZERO
1019 #define CK_F_PR_NEG_16_ZERO
1020 CK_PR_N_Z_S(16, uint16_t)
1021 #endif /* CK_F_PR_NEG_16_ZERO */
1022
1023 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
1024
1025 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE)
1026
1027 #ifndef CK_F_PR_NOT_8
1028 #define CK_F_PR_NOT_8
1029 CK_PR_N_S(not, 8, uint8_t, ~)
1030 #endif /* CK_F_PR_NOT_8 */
1031
1032 #ifndef CK_F_PR_NEG_8
1033 #define CK_F_PR_NEG_8
1034 CK_PR_N_S(neg, 8, uint8_t, -)
1035 #endif /* CK_F_PR_NEG_8 */
1036
1037 #ifndef CK_F_PR_NEG_8_ZERO
1038 #define CK_F_PR_NEG_8_ZERO
1039 CK_PR_N_Z_S(8, uint8_t)
1040 #endif /* CK_F_PR_NEG_8_ZERO */
1041
1042 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */
1043
1044 #undef CK_PR_N_Z_S
1045 #undef CK_PR_N_S
1046 #undef CK_PR_N_Z
1047 #undef CK_PR_N
1048
1049 #define CK_PR_FAA(S, M, T, C)                                           \
1050         CK_CC_INLINE static C                                           \
1051         ck_pr_faa_##S(M *target, T delta)                               \
1052         {                                                               \
1053                 T previous;                                             \
1054                 C punt;                                                 \
1055                 punt = (C)ck_pr_md_load_##S(target);                    \
1056                 previous = (T)punt;                                     \
1057                 while (ck_pr_cas_##S##_value(target,                    \
1058                                              (C)previous,               \
1059                                              (C)(previous + delta),     \
1060                                              &previous) == false)       \
1061                         ck_pr_stall();                                  \
1062                                                                         \
1063                 return ((C)previous);                                   \
1064         }
1065
1066 #define CK_PR_FAS(S, M, C)                                              \
1067         CK_CC_INLINE static C                                           \
1068         ck_pr_fas_##S(M *target, C update)                              \
1069         {                                                               \
1070                 C previous;                                             \
1071                 previous = ck_pr_md_load_##S(target);                   \
1072                 while (ck_pr_cas_##S##_value(target,                    \
1073                                              previous,                  \
1074                                              update,                    \
1075                                              &previous) == false)       \
1076                         ck_pr_stall();                                  \
1077                                                                         \
1078                 return (previous);                                      \
1079         }
1080
1081 #define CK_PR_FAA_S(S, M) CK_PR_FAA(S, M, M, M)
1082 #define CK_PR_FAS_S(S, M) CK_PR_FAS(S, M, M)
1083
1084 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE)
1085
1086 #ifndef CK_F_PR_FAA_CHAR
1087 #define CK_F_PR_FAA_CHAR
1088 CK_PR_FAA_S(char, char)
1089 #endif /* CK_F_PR_FAA_CHAR */
1090
1091 #ifndef CK_F_PR_FAS_CHAR
1092 #define CK_F_PR_FAS_CHAR
1093 CK_PR_FAS_S(char, char)
1094 #endif /* CK_F_PR_FAS_CHAR */
1095
1096 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */
1097
1098 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
1099
1100 #ifndef CK_F_PR_FAA_INT
1101 #define CK_F_PR_FAA_INT
1102 CK_PR_FAA_S(int, int)
1103 #endif /* CK_F_PR_FAA_INT */
1104
1105 #ifndef CK_F_PR_FAS_INT
1106 #define CK_F_PR_FAS_INT
1107 CK_PR_FAS_S(int, int)
1108 #endif /* CK_F_PR_FAS_INT */
1109
1110 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
1111
1112 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE)
1113
1114 #ifndef CK_F_PR_FAA_DOUBLE
1115 #define CK_F_PR_FAA_DOUBLE
1116 CK_PR_FAA_S(double, double)
1117 #endif /* CK_F_PR_FAA_DOUBLE */
1118
1119 #ifndef CK_F_PR_FAS_DOUBLE
1120 #define CK_F_PR_FAS_DOUBLE
1121 CK_PR_FAS_S(double, double)
1122 #endif /* CK_F_PR_FAS_DOUBLE */
1123
1124 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE */
1125
1126 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
1127
1128 #ifndef CK_F_PR_FAA_UINT
1129 #define CK_F_PR_FAA_UINT
1130 CK_PR_FAA_S(uint, unsigned int)
1131 #endif /* CK_F_PR_FAA_UINT */
1132
1133 #ifndef CK_F_PR_FAS_UINT
1134 #define CK_F_PR_FAS_UINT
1135 CK_PR_FAS_S(uint, unsigned int)
1136 #endif /* CK_F_PR_FAS_UINT */
1137
1138 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
1139
1140 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
1141
1142 #ifndef CK_F_PR_FAA_PTR
1143 #define CK_F_PR_FAA_PTR
1144 CK_PR_FAA(ptr, void, uintptr_t, void *)
1145 #endif /* CK_F_PR_FAA_PTR */
1146
1147 #ifndef CK_F_PR_FAS_PTR
1148 #define CK_F_PR_FAS_PTR
1149 CK_PR_FAS(ptr, void, void *)
1150 #endif /* CK_F_PR_FAS_PTR */
1151
1152 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
1153
1154 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
1155
1156 #ifndef CK_F_PR_FAA_64
1157 #define CK_F_PR_FAA_64
1158 CK_PR_FAA_S(64, uint64_t)
1159 #endif /* CK_F_PR_FAA_64 */
1160
1161 #ifndef CK_F_PR_FAS_64
1162 #define CK_F_PR_FAS_64
1163 CK_PR_FAS_S(64, uint64_t)
1164 #endif /* CK_F_PR_FAS_64 */
1165
1166 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
1167
1168 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
1169
1170 #ifndef CK_F_PR_FAA_32
1171 #define CK_F_PR_FAA_32
1172 CK_PR_FAA_S(32, uint32_t)
1173 #endif /* CK_F_PR_FAA_32 */
1174
1175 #ifndef CK_F_PR_FAS_32
1176 #define CK_F_PR_FAS_32
1177 CK_PR_FAS_S(32, uint32_t)
1178 #endif /* CK_F_PR_FAS_32 */
1179
1180 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
1181
1182 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
1183
1184 #ifndef CK_F_PR_FAA_16
1185 #define CK_F_PR_FAA_16
1186 CK_PR_FAA_S(16, uint16_t)
1187 #endif /* CK_F_PR_FAA_16 */
1188
1189 #ifndef CK_F_PR_FAS_16
1190 #define CK_F_PR_FAS_16
1191 CK_PR_FAS_S(16, uint16_t)
1192 #endif /* CK_F_PR_FAS_16 */
1193
1194 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
1195
1196 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE)
1197
1198 #ifndef CK_F_PR_FAA_8
1199 #define CK_F_PR_FAA_8
1200 CK_PR_FAA_S(8, uint8_t)
1201 #endif /* CK_F_PR_FAA_8 */
1202
1203 #ifndef CK_F_PR_FAS_8
1204 #define CK_F_PR_FAS_8
1205 CK_PR_FAS_S(8, uint8_t)
1206 #endif /* CK_F_PR_FAS_8 */
1207
1208 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */
1209
1210 #undef CK_PR_FAA_S
1211 #undef CK_PR_FAS_S
1212 #undef CK_PR_FAA
1213 #undef CK_PR_FAS
1214
1215 #endif /* CK_PR_H */