2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2013 Ed Schouten <ed@FreeBSD.org>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
32 #include <sys/param.h>
33 #include <sys/stdatomic.h>
34 #include <sys/types.h>
36 #include <machine/atomic.h>
37 #include <machine/cpufunc.h>
38 #include <machine/sysarch.h>
41 * Executing statements with interrupts disabled.
44 #if defined(_KERNEL) && !defined(SMP)
45 #define WITHOUT_INTERRUPTS(s) do { \
48 regs = intr_disable(); \
52 #endif /* _KERNEL && !SMP */
57 * It turns out __sync_synchronize() does not emit any code when used
58 * with GCC 4.2. Implement our own version that does work reliably.
60 * Although __sync_lock_test_and_set() should only perform an acquire
61 * barrier, make it do a full barrier like the other functions. This
62 * should make <stdatomic.h>'s atomic_exchange_explicit() work reliably.
65 #if defined(_KERNEL) && !defined(SMP)
70 __asm volatile ("" : : : "memory");
81 #if defined(__CLANG_ATOMICS) || defined(__GNUC_ATOMICS)
84 * New C11 __atomic_* API.
87 /* ARMv6+ systems should be supported by the compiler. */
90 /* Clang doesn't allow us to reimplement builtins without this. */
92 #pragma redefine_extname __sync_synchronize_ext __sync_synchronize
93 #define __sync_synchronize __sync_synchronize_ext
97 __sync_synchronize(void)
104 #error "On SMP systems we should have proper atomic operations."
108 * On uniprocessor systems, we can perform the atomic operations by
109 * disabling interrupts.
112 #define EMIT_LOAD_N(N, uintN_t) \
114 __atomic_load_##N(uintN_t *mem, int model __unused) \
118 WITHOUT_INTERRUPTS({ \
124 #define EMIT_STORE_N(N, uintN_t) \
126 __atomic_store_##N(uintN_t *mem, uintN_t val, int model __unused) \
129 WITHOUT_INTERRUPTS({ \
134 #define EMIT_COMPARE_EXCHANGE_N(N, uintN_t) \
136 __atomic_compare_exchange_##N(uintN_t *mem, uintN_t *expected, \
137 uintN_t desired, int success __unused, int failure __unused) \
141 WITHOUT_INTERRUPTS({ \
142 if (*mem == *expected) { \
153 #define EMIT_FETCH_OP_N(N, uintN_t, name, op) \
155 __atomic_##name##_##N(uintN_t *mem, uintN_t val, int model __unused) \
159 WITHOUT_INTERRUPTS({ \
166 #define EMIT_ALL_OPS_N(N, uintN_t) \
167 EMIT_LOAD_N(N, uintN_t) \
168 EMIT_STORE_N(N, uintN_t) \
169 EMIT_COMPARE_EXCHANGE_N(N, uintN_t) \
170 EMIT_FETCH_OP_N(N, uintN_t, exchange, =) \
171 EMIT_FETCH_OP_N(N, uintN_t, fetch_add, +=) \
172 EMIT_FETCH_OP_N(N, uintN_t, fetch_and, &=) \
173 EMIT_FETCH_OP_N(N, uintN_t, fetch_or, |=) \
174 EMIT_FETCH_OP_N(N, uintN_t, fetch_sub, -=) \
175 EMIT_FETCH_OP_N(N, uintN_t, fetch_xor, ^=)
177 EMIT_ALL_OPS_N(1, uint8_t)
178 EMIT_ALL_OPS_N(2, uint16_t)
179 EMIT_ALL_OPS_N(4, uint32_t)
180 EMIT_ALL_OPS_N(8, uint64_t)
181 #undef EMIT_ALL_OPS_N
186 * For userspace on uniprocessor systems, we can implement the atomic
187 * operations by using a Restartable Atomic Sequence. This makes the
188 * kernel restart the code from the beginning when interrupted.
191 #define EMIT_LOAD_N(N, uintN_t) \
193 __atomic_load_##N(uintN_t *mem, int model __unused) \
199 #define EMIT_STORE_N(N, uintN_t) \
201 __atomic_store_##N(uintN_t *mem, uintN_t val, int model __unused) \
207 #define EMIT_EXCHANGE_N(N, uintN_t, ldr, str) \
209 __atomic_exchange_##N(uintN_t *mem, uintN_t val, int model __unused) \
211 uint32_t old, temp, ras_start; \
213 ras_start = ARM_RAS_START; \
215 /* Set up Restartable Atomic Sequence. */ \
220 "\tstr %2, [%5, #4]\n" \
222 "\t"ldr" %0, %4\n" /* Load old value. */ \
223 "\t"str" %3, %1\n" /* Store new value. */ \
225 /* Tear down Restartable Atomic Sequence. */ \
227 "\tmov %2, #0x00000000\n" \
229 "\tmov %2, #0xffffffff\n" \
230 "\tstr %2, [%5, #4]\n" \
231 : "=&r" (old), "=m" (*mem), "=&r" (temp) \
232 : "r" (val), "m" (*mem), "r" (ras_start)); \
236 #define EMIT_COMPARE_EXCHANGE_N(N, uintN_t, ldr, streq) \
238 __atomic_compare_exchange_##N(uintN_t *mem, uintN_t *pexpected, \
239 uintN_t desired, int success __unused, int failure __unused) \
241 uint32_t expected, old, temp, ras_start; \
243 expected = *pexpected; \
244 ras_start = ARM_RAS_START; \
246 /* Set up Restartable Atomic Sequence. */ \
251 "\tstr %2, [%6, #4]\n" \
253 "\t"ldr" %0, %5\n" /* Load old value. */ \
254 "\tcmp %0, %3\n" /* Compare to expected value. */\
255 "\t"streq" %4, %1\n" /* Store new value. */ \
257 /* Tear down Restartable Atomic Sequence. */ \
259 "\tmov %2, #0x00000000\n" \
261 "\tmov %2, #0xffffffff\n" \
262 "\tstr %2, [%6, #4]\n" \
263 : "=&r" (old), "=m" (*mem), "=&r" (temp) \
264 : "r" (expected), "r" (desired), "m" (*mem), \
266 if (old == expected) { \
274 #define EMIT_FETCH_OP_N(N, uintN_t, ldr, str, name, op, ret) \
276 __atomic_##name##_##N(uintN_t *mem, uintN_t val, int model __unused) \
278 uint32_t old, new, ras_start; \
280 ras_start = ARM_RAS_START; \
282 /* Set up Restartable Atomic Sequence. */ \
287 "\tstr %2, [%5, #4]\n" \
289 "\t"ldr" %0, %4\n" /* Load old value. */ \
290 "\t"op" %2, %0, %3\n" /* Calculate new value. */ \
291 "\t"str" %2, %1\n" /* Store new value. */ \
293 /* Tear down Restartable Atomic Sequence. */ \
295 "\tmov %2, #0x00000000\n" \
297 "\tmov %2, #0xffffffff\n" \
298 "\tstr %2, [%5, #4]\n" \
299 : "=&r" (old), "=m" (*mem), "=&r" (new) \
300 : "r" (val), "m" (*mem), "r" (ras_start)); \
304 #define EMIT_ALL_OPS_N(N, uintN_t, ldr, str, streq) \
305 EMIT_LOAD_N(N, uintN_t) \
306 EMIT_STORE_N(N, uintN_t) \
307 EMIT_EXCHANGE_N(N, uintN_t, ldr, str) \
308 EMIT_COMPARE_EXCHANGE_N(N, uintN_t, ldr, streq) \
309 EMIT_FETCH_OP_N(N, uintN_t, ldr, str, fetch_add, "add", old) \
310 EMIT_FETCH_OP_N(N, uintN_t, ldr, str, fetch_and, "and", old) \
311 EMIT_FETCH_OP_N(N, uintN_t, ldr, str, fetch_or, "orr", old) \
312 EMIT_FETCH_OP_N(N, uintN_t, ldr, str, fetch_sub, "sub", old) \
313 EMIT_FETCH_OP_N(N, uintN_t, ldr, str, fetch_xor, "eor", old) \
314 EMIT_FETCH_OP_N(N, uintN_t, ldr, str, add_fetch, "add", new) \
315 EMIT_FETCH_OP_N(N, uintN_t, ldr, str, and_fetch, "and", new) \
316 EMIT_FETCH_OP_N(N, uintN_t, ldr, str, or_fetch, "orr", new) \
317 EMIT_FETCH_OP_N(N, uintN_t, ldr, str, sub_fetch, "sub", new) \
318 EMIT_FETCH_OP_N(N, uintN_t, ldr, str, xor_fetch, "eor", new)
320 EMIT_ALL_OPS_N(1, uint8_t, "ldrb", "strb", "strbeq")
321 EMIT_ALL_OPS_N(2, uint16_t, "ldrh", "strh", "strheq")
322 EMIT_ALL_OPS_N(4, uint32_t, "ldr", "str", "streq")
323 #undef EMIT_ALL_OPS_N
327 #endif /* __ARM_ARCH */
329 #endif /* __CLANG_ATOMICS || __GNUC_ATOMICS */
331 #if defined(__SYNC_ATOMICS) || defined(EMIT_SYNC_ATOMICS)
334 #pragma redefine_extname __sync_lock_test_and_set_1_c __sync_lock_test_and_set_1
335 #pragma redefine_extname __sync_lock_test_and_set_2_c __sync_lock_test_and_set_2
336 #pragma redefine_extname __sync_lock_test_and_set_4_c __sync_lock_test_and_set_4
337 #pragma redefine_extname __sync_val_compare_and_swap_1_c __sync_val_compare_and_swap_1
338 #pragma redefine_extname __sync_val_compare_and_swap_2_c __sync_val_compare_and_swap_2
339 #pragma redefine_extname __sync_val_compare_and_swap_4_c __sync_val_compare_and_swap_4
340 #pragma redefine_extname __sync_fetch_and_add_1_c __sync_fetch_and_add_1
341 #pragma redefine_extname __sync_fetch_and_add_2_c __sync_fetch_and_add_2
342 #pragma redefine_extname __sync_fetch_and_add_4_c __sync_fetch_and_add_4
343 #pragma redefine_extname __sync_fetch_and_and_1_c __sync_fetch_and_and_1
344 #pragma redefine_extname __sync_fetch_and_and_2_c __sync_fetch_and_and_2
345 #pragma redefine_extname __sync_fetch_and_and_4_c __sync_fetch_and_and_4
346 #pragma redefine_extname __sync_fetch_and_or_1_c __sync_fetch_and_or_1
347 #pragma redefine_extname __sync_fetch_and_or_2_c __sync_fetch_and_or_2
348 #pragma redefine_extname __sync_fetch_and_or_4_c __sync_fetch_and_or_4
349 #pragma redefine_extname __sync_fetch_and_xor_1_c __sync_fetch_and_xor_1
350 #pragma redefine_extname __sync_fetch_and_xor_2_c __sync_fetch_and_xor_2
351 #pragma redefine_extname __sync_fetch_and_xor_4_c __sync_fetch_and_xor_4
352 #pragma redefine_extname __sync_fetch_and_sub_1_c __sync_fetch_and_sub_1
353 #pragma redefine_extname __sync_fetch_and_sub_2_c __sync_fetch_and_sub_2
354 #pragma redefine_extname __sync_fetch_and_sub_4_c __sync_fetch_and_sub_4
363 /* Implementations for old GCC versions, lacking support for atomics. */
371 * Given a memory address pointing to an 8-bit or 16-bit integer, return
372 * the address of the 32-bit word containing it.
375 static inline uint32_t *
376 round_to_word(void *ptr)
379 return ((uint32_t *)((intptr_t)ptr & ~3));
383 * Utility functions for loading and storing 8-bit and 16-bit integers
384 * in 32-bit words at an offset corresponding with the location of the
389 put_1(reg_t *r, const uint8_t *offset_ptr, uint8_t val)
393 offset = (intptr_t)offset_ptr & 3;
397 static inline uint8_t
398 get_1(const reg_t *r, const uint8_t *offset_ptr)
402 offset = (intptr_t)offset_ptr & 3;
403 return (r->v8[offset]);
407 put_2(reg_t *r, const uint16_t *offset_ptr, uint16_t val)
415 offset = (intptr_t)offset_ptr & 3;
417 r->v8[offset] = bytes.out[0];
418 r->v8[offset + 1] = bytes.out[1];
421 static inline uint16_t
422 get_2(const reg_t *r, const uint16_t *offset_ptr)
430 offset = (intptr_t)offset_ptr & 3;
431 bytes.in[0] = r->v8[offset];
432 bytes.in[1] = r->v8[offset + 1];
437 * 8-bit and 16-bit routines.
439 * These operations are not natively supported by the CPU, so we use
440 * some shifting and bitmasking on top of the 32-bit instructions.
443 #define EMIT_LOCK_TEST_AND_SET_N(N, uintN_t) \
445 __sync_lock_test_and_set_##N##_c(uintN_t *mem, uintN_t val) \
448 reg_t val32, negmask, old; \
449 uint32_t temp1, temp2; \
451 mem32 = round_to_word(mem); \
452 val32.v32 = 0x00000000; \
453 put_##N(&val32, mem, val); \
454 negmask.v32 = 0xffffffff; \
455 put_##N(&negmask, mem, 0); \
460 "\tldrex %0, %6\n" /* Load old value. */ \
461 "\tand %2, %5, %0\n" /* Remove the old value. */ \
462 "\torr %2, %2, %4\n" /* Put in the new value. */ \
463 "\tstrex %3, %2, %1\n" /* Attempt to store. */ \
464 "\tcmp %3, #0\n" /* Did it succeed? */ \
465 "\tbne 1b\n" /* Spin if failed. */ \
466 : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp1), \
468 : "r" (val32.v32), "r" (negmask.v32), "m" (*mem32)); \
469 return (get_##N(&old, mem)); \
472 EMIT_LOCK_TEST_AND_SET_N(1, uint8_t)
473 EMIT_LOCK_TEST_AND_SET_N(2, uint16_t)
475 #define EMIT_VAL_COMPARE_AND_SWAP_N(N, uintN_t) \
477 __sync_val_compare_and_swap_##N##_c(uintN_t *mem, uintN_t expected, \
481 reg_t expected32, desired32, posmask, old; \
482 uint32_t negmask, temp1, temp2; \
484 mem32 = round_to_word(mem); \
485 expected32.v32 = 0x00000000; \
486 put_##N(&expected32, mem, expected); \
487 desired32.v32 = 0x00000000; \
488 put_##N(&desired32, mem, desired); \
489 posmask.v32 = 0x00000000; \
490 put_##N(&posmask, mem, ~0); \
491 negmask = ~posmask.v32; \
496 "\tldrex %0, %8\n" /* Load old value. */ \
497 "\tand %2, %6, %0\n" /* Isolate the old value. */ \
498 "\tcmp %2, %4\n" /* Compare to expected value. */\
499 "\tbne 2f\n" /* Values are unequal. */ \
500 "\tand %2, %7, %0\n" /* Remove the old value. */ \
501 "\torr %2, %5\n" /* Put in the new value. */ \
502 "\tstrex %3, %2, %1\n" /* Attempt to store. */ \
503 "\tcmp %3, #0\n" /* Did it succeed? */ \
504 "\tbne 1b\n" /* Spin if failed. */ \
506 : "=&r" (old), "=m" (*mem32), "=&r" (temp1), \
508 : "r" (expected32.v32), "r" (desired32.v32), \
509 "r" (posmask.v32), "r" (negmask), "m" (*mem32)); \
510 return (get_##N(&old, mem)); \
513 EMIT_VAL_COMPARE_AND_SWAP_N(1, uint8_t)
514 EMIT_VAL_COMPARE_AND_SWAP_N(2, uint16_t)
516 #define EMIT_ARITHMETIC_FETCH_AND_OP_N(N, uintN_t, name, op) \
518 __sync_##name##_##N##_c(uintN_t *mem, uintN_t val) \
521 reg_t val32, posmask, old; \
522 uint32_t negmask, temp1, temp2; \
524 mem32 = round_to_word(mem); \
525 val32.v32 = 0x00000000; \
526 put_##N(&val32, mem, val); \
527 posmask.v32 = 0x00000000; \
528 put_##N(&posmask, mem, ~0); \
529 negmask = ~posmask.v32; \
534 "\tldrex %0, %7\n" /* Load old value. */ \
535 "\t"op" %2, %0, %4\n" /* Calculate new value. */ \
536 "\tand %2, %5\n" /* Isolate the new value. */ \
537 "\tand %3, %6, %0\n" /* Remove the old value. */ \
538 "\torr %2, %2, %3\n" /* Put in the new value. */ \
539 "\tstrex %3, %2, %1\n" /* Attempt to store. */ \
540 "\tcmp %3, #0\n" /* Did it succeed? */ \
541 "\tbne 1b\n" /* Spin if failed. */ \
542 : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp1), \
544 : "r" (val32.v32), "r" (posmask.v32), "r" (negmask), \
546 return (get_##N(&old, mem)); \
549 EMIT_ARITHMETIC_FETCH_AND_OP_N(1, uint8_t, fetch_and_add, "add")
550 EMIT_ARITHMETIC_FETCH_AND_OP_N(1, uint8_t, fetch_and_sub, "sub")
551 EMIT_ARITHMETIC_FETCH_AND_OP_N(2, uint16_t, fetch_and_add, "add")
552 EMIT_ARITHMETIC_FETCH_AND_OP_N(2, uint16_t, fetch_and_sub, "sub")
554 #define EMIT_BITWISE_FETCH_AND_OP_N(N, uintN_t, name, op, idempotence) \
556 __sync_##name##_##N##_c(uintN_t *mem, uintN_t val) \
560 uint32_t temp1, temp2; \
562 mem32 = round_to_word(mem); \
563 val32.v32 = idempotence ? 0xffffffff : 0x00000000; \
564 put_##N(&val32, mem, val); \
569 "\tldrex %0, %5\n" /* Load old value. */ \
570 "\t"op" %2, %4, %0\n" /* Calculate new value. */ \
571 "\tstrex %3, %2, %1\n" /* Attempt to store. */ \
572 "\tcmp %3, #0\n" /* Did it succeed? */ \
573 "\tbne 1b\n" /* Spin if failed. */ \
574 : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp1), \
576 : "r" (val32.v32), "m" (*mem32)); \
577 return (get_##N(&old, mem)); \
580 EMIT_BITWISE_FETCH_AND_OP_N(1, uint8_t, fetch_and_and, "and", 1)
581 EMIT_BITWISE_FETCH_AND_OP_N(1, uint8_t, fetch_and_or, "orr", 0)
582 EMIT_BITWISE_FETCH_AND_OP_N(1, uint8_t, fetch_and_xor, "eor", 0)
583 EMIT_BITWISE_FETCH_AND_OP_N(2, uint16_t, fetch_and_and, "and", 1)
584 EMIT_BITWISE_FETCH_AND_OP_N(2, uint16_t, fetch_and_or, "orr", 0)
585 EMIT_BITWISE_FETCH_AND_OP_N(2, uint16_t, fetch_and_xor, "eor", 0)
592 __sync_lock_test_and_set_4_c(uint32_t *mem, uint32_t val)
599 "\tldrex %0, %4\n" /* Load old value. */
600 "\tstrex %2, %3, %1\n" /* Attempt to store. */
601 "\tcmp %2, #0\n" /* Did it succeed? */
602 "\tbne 1b\n" /* Spin if failed. */
603 : "=&r" (old), "=m" (*mem), "=&r" (temp)
604 : "r" (val), "m" (*mem));
609 __sync_val_compare_and_swap_4_c(uint32_t *mem, uint32_t expected,
617 "\tldrex %0, %5\n" /* Load old value. */
618 "\tcmp %0, %3\n" /* Compare to expected value. */
619 "\tbne 2f\n" /* Values are unequal. */
620 "\tstrex %2, %4, %1\n" /* Attempt to store. */
621 "\tcmp %2, #0\n" /* Did it succeed? */
622 "\tbne 1b\n" /* Spin if failed. */
624 : "=&r" (old), "=m" (*mem), "=&r" (temp)
625 : "r" (expected), "r" (desired), "m" (*mem));
629 #define EMIT_FETCH_AND_OP_4(name, op) \
631 __sync_##name##_4##_c(uint32_t *mem, uint32_t val) \
633 uint32_t old, temp1, temp2; \
638 "\tldrex %0, %5\n" /* Load old value. */ \
639 "\t"op" %2, %0, %4\n" /* Calculate new value. */ \
640 "\tstrex %3, %2, %1\n" /* Attempt to store. */ \
641 "\tcmp %3, #0\n" /* Did it succeed? */ \
642 "\tbne 1b\n" /* Spin if failed. */ \
643 : "=&r" (old), "=m" (*mem), "=&r" (temp1), \
645 : "r" (val), "m" (*mem)); \
649 EMIT_FETCH_AND_OP_4(fetch_and_add, "add")
650 EMIT_FETCH_AND_OP_4(fetch_and_and, "and")
651 EMIT_FETCH_AND_OP_4(fetch_and_or, "orr")
652 EMIT_FETCH_AND_OP_4(fetch_and_sub, "sub")
653 EMIT_FETCH_AND_OP_4(fetch_and_xor, "eor")
656 __strong_reference(__sync_lock_test_and_set_1_c, __sync_lock_test_and_set_1);
657 __strong_reference(__sync_lock_test_and_set_2_c, __sync_lock_test_and_set_2);
658 __strong_reference(__sync_lock_test_and_set_4_c, __sync_lock_test_and_set_4);
659 __strong_reference(__sync_val_compare_and_swap_1_c, __sync_val_compare_and_swap_1);
660 __strong_reference(__sync_val_compare_and_swap_2_c, __sync_val_compare_and_swap_2);
661 __strong_reference(__sync_val_compare_and_swap_4_c, __sync_val_compare_and_swap_4);
662 __strong_reference(__sync_fetch_and_add_1_c, __sync_fetch_and_add_1);
663 __strong_reference(__sync_fetch_and_add_2_c, __sync_fetch_and_add_2);
664 __strong_reference(__sync_fetch_and_add_4_c, __sync_fetch_and_add_4);
665 __strong_reference(__sync_fetch_and_and_1_c, __sync_fetch_and_and_1);
666 __strong_reference(__sync_fetch_and_and_2_c, __sync_fetch_and_and_2);
667 __strong_reference(__sync_fetch_and_and_4_c, __sync_fetch_and_and_4);
668 __strong_reference(__sync_fetch_and_sub_1_c, __sync_fetch_and_sub_1);
669 __strong_reference(__sync_fetch_and_sub_2_c, __sync_fetch_and_sub_2);
670 __strong_reference(__sync_fetch_and_sub_4_c, __sync_fetch_and_sub_4);
671 __strong_reference(__sync_fetch_and_or_1_c, __sync_fetch_and_or_1);
672 __strong_reference(__sync_fetch_and_or_2_c, __sync_fetch_and_or_2);
673 __strong_reference(__sync_fetch_and_or_4_c, __sync_fetch_and_or_4);
674 __strong_reference(__sync_fetch_and_xor_1_c, __sync_fetch_and_xor_1);
675 __strong_reference(__sync_fetch_and_xor_2_c, __sync_fetch_and_xor_2);
676 __strong_reference(__sync_fetch_and_xor_4_c, __sync_fetch_and_xor_4);
679 #else /* __ARM_ARCH < 6 */
684 #error "On SMP systems we should have proper atomic operations."
688 * On uniprocessor systems, we can perform the atomic operations by
689 * disabling interrupts.
692 #define EMIT_VAL_COMPARE_AND_SWAP_N(N, uintN_t) \
694 __sync_val_compare_and_swap_##N(uintN_t *mem, uintN_t expected, \
699 WITHOUT_INTERRUPTS({ \
701 if (*mem == expected) \
707 #define EMIT_FETCH_AND_OP_N(N, uintN_t, name, op) \
709 __sync_##name##_##N(uintN_t *mem, uintN_t val) \
713 WITHOUT_INTERRUPTS({ \
720 #define EMIT_ALL_OPS_N(N, uintN_t) \
721 EMIT_VAL_COMPARE_AND_SWAP_N(N, uintN_t) \
722 EMIT_FETCH_AND_OP_N(N, uintN_t, lock_test_and_set, =) \
723 EMIT_FETCH_AND_OP_N(N, uintN_t, fetch_and_add, +=) \
724 EMIT_FETCH_AND_OP_N(N, uintN_t, fetch_and_and, &=) \
725 EMIT_FETCH_AND_OP_N(N, uintN_t, fetch_and_or, |=) \
726 EMIT_FETCH_AND_OP_N(N, uintN_t, fetch_and_sub, -=) \
727 EMIT_FETCH_AND_OP_N(N, uintN_t, fetch_and_xor, ^=)
729 EMIT_ALL_OPS_N(1, uint8_t)
730 EMIT_ALL_OPS_N(2, uint16_t)
731 EMIT_ALL_OPS_N(4, uint32_t)
732 EMIT_ALL_OPS_N(8, uint64_t)
733 #undef EMIT_ALL_OPS_N
738 * For userspace on uniprocessor systems, we can implement the atomic
739 * operations by using a Restartable Atomic Sequence. This makes the
740 * kernel restart the code from the beginning when interrupted.
743 #define EMIT_LOCK_TEST_AND_SET_N(N, uintN_t, ldr, str) \
745 __sync_lock_test_and_set_##N##_c(uintN_t *mem, uintN_t val) \
747 uint32_t old, temp, ras_start; \
749 ras_start = ARM_RAS_START; \
751 /* Set up Restartable Atomic Sequence. */ \
756 "\tstr %2, [%5, #4]\n" \
758 "\t"ldr" %0, %4\n" /* Load old value. */ \
759 "\t"str" %3, %1\n" /* Store new value. */ \
761 /* Tear down Restartable Atomic Sequence. */ \
763 "\tmov %2, #0x00000000\n" \
765 "\tmov %2, #0xffffffff\n" \
766 "\tstr %2, [%5, #4]\n" \
767 : "=&r" (old), "=m" (*mem), "=&r" (temp) \
768 : "r" (val), "m" (*mem), "r" (ras_start)); \
772 #define EMIT_VAL_COMPARE_AND_SWAP_N(N, uintN_t, ldr, streq) \
774 __sync_val_compare_and_swap_##N##_c(uintN_t *mem, uintN_t expected, \
777 uint32_t old, temp, ras_start; \
779 ras_start = ARM_RAS_START; \
781 /* Set up Restartable Atomic Sequence. */ \
786 "\tstr %2, [%6, #4]\n" \
788 "\t"ldr" %0, %5\n" /* Load old value. */ \
789 "\tcmp %0, %3\n" /* Compare to expected value. */\
790 "\t"streq" %4, %1\n" /* Store new value. */ \
792 /* Tear down Restartable Atomic Sequence. */ \
794 "\tmov %2, #0x00000000\n" \
796 "\tmov %2, #0xffffffff\n" \
797 "\tstr %2, [%6, #4]\n" \
798 : "=&r" (old), "=m" (*mem), "=&r" (temp) \
799 : "r" (expected), "r" (desired), "m" (*mem), \
804 #define EMIT_FETCH_AND_OP_N(N, uintN_t, ldr, str, name, op) \
806 __sync_##name##_##N##_c(uintN_t *mem, uintN_t val) \
808 uint32_t old, temp, ras_start; \
810 ras_start = ARM_RAS_START; \
812 /* Set up Restartable Atomic Sequence. */ \
817 "\tstr %2, [%5, #4]\n" \
819 "\t"ldr" %0, %4\n" /* Load old value. */ \
820 "\t"op" %2, %0, %3\n" /* Calculate new value. */ \
821 "\t"str" %2, %1\n" /* Store new value. */ \
823 /* Tear down Restartable Atomic Sequence. */ \
825 "\tmov %2, #0x00000000\n" \
827 "\tmov %2, #0xffffffff\n" \
828 "\tstr %2, [%5, #4]\n" \
829 : "=&r" (old), "=m" (*mem), "=&r" (temp) \
830 : "r" (val), "m" (*mem), "r" (ras_start)); \
834 #define EMIT_ALL_OPS_N(N, uintN_t, ldr, str, streq) \
835 EMIT_LOCK_TEST_AND_SET_N(N, uintN_t, ldr, str) \
836 EMIT_VAL_COMPARE_AND_SWAP_N(N, uintN_t, ldr, streq) \
837 EMIT_FETCH_AND_OP_N(N, uintN_t, ldr, str, fetch_and_add, "add") \
838 EMIT_FETCH_AND_OP_N(N, uintN_t, ldr, str, fetch_and_and, "and") \
839 EMIT_FETCH_AND_OP_N(N, uintN_t, ldr, str, fetch_and_or, "orr") \
840 EMIT_FETCH_AND_OP_N(N, uintN_t, ldr, str, fetch_and_sub, "sub") \
841 EMIT_FETCH_AND_OP_N(N, uintN_t, ldr, str, fetch_and_xor, "eor")
844 EMIT_ALL_OPS_N(1, uint8_t, "ldrb", "strb", "strbeq")
845 EMIT_ALL_OPS_N(2, uint16_t, "ldrh", "strh", "strheq")
847 EMIT_ALL_OPS_N(1, uint8_t, "ldrb", "strb", "streqb")
848 EMIT_ALL_OPS_N(2, uint16_t, "ldrh", "strh", "streqh")
850 EMIT_ALL_OPS_N(4, uint32_t, "ldr", "str", "streq")
853 __strong_reference(__sync_lock_test_and_set_1_c, __sync_lock_test_and_set_1);
854 __strong_reference(__sync_lock_test_and_set_2_c, __sync_lock_test_and_set_2);
855 __strong_reference(__sync_lock_test_and_set_4_c, __sync_lock_test_and_set_4);
856 __strong_reference(__sync_val_compare_and_swap_1_c, __sync_val_compare_and_swap_1);
857 __strong_reference(__sync_val_compare_and_swap_2_c, __sync_val_compare_and_swap_2);
858 __strong_reference(__sync_val_compare_and_swap_4_c, __sync_val_compare_and_swap_4);
859 __strong_reference(__sync_fetch_and_add_1_c, __sync_fetch_and_add_1);
860 __strong_reference(__sync_fetch_and_add_2_c, __sync_fetch_and_add_2);
861 __strong_reference(__sync_fetch_and_add_4_c, __sync_fetch_and_add_4);
862 __strong_reference(__sync_fetch_and_and_1_c, __sync_fetch_and_and_1);
863 __strong_reference(__sync_fetch_and_and_2_c, __sync_fetch_and_and_2);
864 __strong_reference(__sync_fetch_and_and_4_c, __sync_fetch_and_and_4);
865 __strong_reference(__sync_fetch_and_sub_1_c, __sync_fetch_and_sub_1);
866 __strong_reference(__sync_fetch_and_sub_2_c, __sync_fetch_and_sub_2);
867 __strong_reference(__sync_fetch_and_sub_4_c, __sync_fetch_and_sub_4);
868 __strong_reference(__sync_fetch_and_or_1_c, __sync_fetch_and_or_1);
869 __strong_reference(__sync_fetch_and_or_2_c, __sync_fetch_and_or_2);
870 __strong_reference(__sync_fetch_and_or_4_c, __sync_fetch_and_or_4);
871 __strong_reference(__sync_fetch_and_xor_1_c, __sync_fetch_and_xor_1);
872 __strong_reference(__sync_fetch_and_xor_2_c, __sync_fetch_and_xor_2);
873 __strong_reference(__sync_fetch_and_xor_4_c, __sync_fetch_and_xor_4);
874 #endif /* __ARM_ARCH */
880 #endif /* __SYNC_ATOMICS */