2 * Copyright (c) 2013 Ed Schouten <ed@FreeBSD.org>
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
30 #include <sys/param.h>
31 #include <sys/stdatomic.h>
32 #include <sys/types.h>
34 #include <machine/cpufunc.h>
35 #include <machine/sysarch.h>
38 #include "opt_global.h"
42 * Executing statements with interrupts disabled.
45 #if defined(_KERNEL) && !defined(SMP)
46 #define WITHOUT_INTERRUPTS(s) do { \
49 regs = intr_disable(); \
53 #endif /* _KERNEL && !SMP */
58 * It turns out __sync_synchronize() does not emit any code when used
59 * with GCC 4.2. Implement our own version that does work reliably.
61 * Although __sync_lock_test_and_set() should only perform an acquire
62 * barrier, make it do a full barrier like the other functions. This
63 * should make <stdatomic.h>'s atomic_exchange_explicit() work reliably.
66 #if defined(_KERNEL) && !defined(SMP)
71 __asm volatile ("" : : : "memory");
73 #elif defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__)
78 __asm volatile ("dmb" : : : "memory");
80 #elif defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || \
81 defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || \
82 defined(__ARM_ARCH_6ZK__)
87 __asm volatile ("mcr p15, 0, %0, c7, c10, 5" : : "r" (0) : "memory");
91 #if defined(__CLANG_ATOMICS) || defined(__GNUC_ATOMICS)
94 * New C11 __atomic_* API.
97 #if defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || \
98 defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || \
99 defined(__ARM_ARCH_6ZK__) || \
100 defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__)
102 /* These systems should be supported by the compiler. */
104 #else /* __ARM_ARCH_5__ */
106 /* Clang doesn't allow us to reimplement builtins without this. */
108 #pragma redefine_extname __sync_synchronize_ext __sync_synchronize
109 #define __sync_synchronize __sync_synchronize_ext
113 __sync_synchronize(void)
120 #error "On SMP systems we should have proper atomic operations."
124 * On uniprocessor systems, we can perform the atomic operations by
125 * disabling interrupts.
128 #define EMIT_LOAD_N(N, uintN_t) \
130 __atomic_load_##N(uintN_t *mem, int model __unused) \
134 WITHOUT_INTERRUPTS({ \
140 #define EMIT_STORE_N(N, uintN_t) \
142 __atomic_store_##N(uintN_t *mem, uintN_t val, int model __unused) \
145 WITHOUT_INTERRUPTS({ \
150 #define EMIT_COMPARE_EXCHANGE_N(N, uintN_t) \
152 __atomic_compare_exchange_##N(uintN_t *mem, uintN_t *expected, \
153 uintN_t desired, int success __unused, int failure __unused) \
157 WITHOUT_INTERRUPTS({ \
158 if (*mem == *expected) { \
169 #define EMIT_FETCH_OP_N(N, uintN_t, name, op) \
171 __atomic_##name##_##N(uintN_t *mem, uintN_t val, int model __unused) \
175 WITHOUT_INTERRUPTS({ \
182 #define EMIT_ALL_OPS_N(N, uintN_t) \
183 EMIT_LOAD_N(N, uintN_t) \
184 EMIT_STORE_N(N, uintN_t) \
185 EMIT_COMPARE_EXCHANGE_N(N, uintN_t) \
186 EMIT_FETCH_OP_N(N, uintN_t, exchange, =) \
187 EMIT_FETCH_OP_N(N, uintN_t, fetch_add, +=) \
188 EMIT_FETCH_OP_N(N, uintN_t, fetch_and, &=) \
189 EMIT_FETCH_OP_N(N, uintN_t, fetch_or, |=) \
190 EMIT_FETCH_OP_N(N, uintN_t, fetch_sub, -=) \
191 EMIT_FETCH_OP_N(N, uintN_t, fetch_xor, ^=)
193 EMIT_ALL_OPS_N(1, uint8_t)
194 EMIT_ALL_OPS_N(2, uint16_t)
195 EMIT_ALL_OPS_N(4, uint32_t)
196 EMIT_ALL_OPS_N(8, uint64_t)
197 #undef EMIT_ALL_OPS_N
202 * For userspace on uniprocessor systems, we can implement the atomic
203 * operations by using a Restartable Atomic Sequence. This makes the
204 * kernel restart the code from the beginning when interrupted.
207 #define EMIT_LOAD_N(N, uintN_t) \
209 __atomic_load_##N(uintN_t *mem, int model __unused) \
215 #define EMIT_STORE_N(N, uintN_t) \
217 __atomic_store_##N(uintN_t *mem, uintN_t val, int model __unused) \
223 #define EMIT_EXCHANGE_N(N, uintN_t, ldr, str) \
225 __atomic_exchange_##N(uintN_t *mem, uintN_t val, int model __unused) \
227 uint32_t old, temp, ras_start; \
229 ras_start = ARM_RAS_START; \
231 /* Set up Restartable Atomic Sequence. */ \
236 "\tstr %2, [%5, #4]\n" \
238 "\t"ldr" %0, %4\n" /* Load old value. */ \
239 "\t"str" %3, %1\n" /* Store new value. */ \
241 /* Tear down Restartable Atomic Sequence. */ \
243 "\tmov %2, #0x00000000\n" \
245 "\tmov %2, #0xffffffff\n" \
246 "\tstr %2, [%5, #4]\n" \
247 : "=&r" (old), "=m" (*mem), "=&r" (temp) \
248 : "r" (val), "m" (*mem), "r" (ras_start)); \
252 #define EMIT_COMPARE_EXCHANGE_N(N, uintN_t, ldr, streq) \
254 __atomic_compare_exchange_##N(uintN_t *mem, uintN_t *pexpected, \
255 uintN_t desired, int success __unused, int failure __unused) \
257 uint32_t expected, old, temp, ras_start; \
259 expected = *pexpected; \
260 ras_start = ARM_RAS_START; \
262 /* Set up Restartable Atomic Sequence. */ \
267 "\tstr %2, [%6, #4]\n" \
269 "\t"ldr" %0, %5\n" /* Load old value. */ \
270 "\tcmp %0, %3\n" /* Compare to expected value. */\
271 "\t"streq" %4, %1\n" /* Store new value. */ \
273 /* Tear down Restartable Atomic Sequence. */ \
275 "\tmov %2, #0x00000000\n" \
277 "\tmov %2, #0xffffffff\n" \
278 "\tstr %2, [%6, #4]\n" \
279 : "=&r" (old), "=m" (*mem), "=&r" (temp) \
280 : "r" (expected), "r" (desired), "m" (*mem), \
282 if (old == expected) { \
290 #define EMIT_FETCH_OP_N(N, uintN_t, ldr, str, name, op) \
292 __atomic_##name##_##N(uintN_t *mem, uintN_t val, int model __unused) \
294 uint32_t old, temp, ras_start; \
296 ras_start = ARM_RAS_START; \
298 /* Set up Restartable Atomic Sequence. */ \
303 "\tstr %2, [%5, #4]\n" \
305 "\t"ldr" %0, %4\n" /* Load old value. */ \
306 "\t"op" %2, %0, %3\n" /* Calculate new value. */ \
307 "\t"str" %2, %1\n" /* Store new value. */ \
309 /* Tear down Restartable Atomic Sequence. */ \
311 "\tmov %2, #0x00000000\n" \
313 "\tmov %2, #0xffffffff\n" \
314 "\tstr %2, [%5, #4]\n" \
315 : "=&r" (old), "=m" (*mem), "=&r" (temp) \
316 : "r" (val), "m" (*mem), "r" (ras_start)); \
320 #define EMIT_ALL_OPS_N(N, uintN_t, ldr, str, streq) \
321 EMIT_LOAD_N(N, uintN_t) \
322 EMIT_STORE_N(N, uintN_t) \
323 EMIT_EXCHANGE_N(N, uintN_t, ldr, str) \
324 EMIT_COMPARE_EXCHANGE_N(N, uintN_t, ldr, streq) \
325 EMIT_FETCH_OP_N(N, uintN_t, ldr, str, fetch_add, "add") \
326 EMIT_FETCH_OP_N(N, uintN_t, ldr, str, fetch_and, "and") \
327 EMIT_FETCH_OP_N(N, uintN_t, ldr, str, fetch_or, "orr") \
328 EMIT_FETCH_OP_N(N, uintN_t, ldr, str, fetch_sub, "sub") \
329 EMIT_FETCH_OP_N(N, uintN_t, ldr, str, fetch_xor, "eor")
331 EMIT_ALL_OPS_N(1, uint8_t, "ldrb", "strb", "strbeq")
332 EMIT_ALL_OPS_N(2, uint16_t, "ldrh", "strh", "strheq")
333 EMIT_ALL_OPS_N(4, uint32_t, "ldr", "str", "streq")
334 #undef EMIT_ALL_OPS_N
340 #endif /* __CLANG_ATOMICS || __GNUC_ATOMICS */
342 #if defined(__SYNC_ATOMICS) || defined(EMIT_SYNC_ATOMICS)
345 #pragma redefine_extname __sync_lock_test_and_set_1_c __sync_lock_test_and_set_1
346 #pragma redefine_extname __sync_lock_test_and_set_2_c __sync_lock_test_and_set_2
347 #pragma redefine_extname __sync_lock_test_and_set_4_c __sync_lock_test_and_set_4
348 #pragma redefine_extname __sync_val_compare_and_swap_1_c __sync_val_compare_and_swap_1
349 #pragma redefine_extname __sync_val_compare_and_swap_2_c __sync_val_compare_and_swap_2
350 #pragma redefine_extname __sync_val_compare_and_swap_4_c __sync_val_compare_and_swap_4
351 #pragma redefine_extname __sync_fetch_and_add_1_c __sync_fetch_and_add_1
352 #pragma redefine_extname __sync_fetch_and_add_2_c __sync_fetch_and_add_2
353 #pragma redefine_extname __sync_fetch_and_add_4_c __sync_fetch_and_add_4
354 #pragma redefine_extname __sync_fetch_and_and_1_c __sync_fetch_and_and_1
355 #pragma redefine_extname __sync_fetch_and_and_2_c __sync_fetch_and_and_2
356 #pragma redefine_extname __sync_fetch_and_and_4_c __sync_fetch_and_and_4
357 #pragma redefine_extname __sync_fetch_and_or_1_c __sync_fetch_and_or_1
358 #pragma redefine_extname __sync_fetch_and_or_2_c __sync_fetch_and_or_2
359 #pragma redefine_extname __sync_fetch_and_or_4_c __sync_fetch_and_or_4
360 #pragma redefine_extname __sync_fetch_and_xor_1_c __sync_fetch_and_xor_1
361 #pragma redefine_extname __sync_fetch_and_xor_2_c __sync_fetch_and_xor_2
362 #pragma redefine_extname __sync_fetch_and_xor_4_c __sync_fetch_and_xor_4
363 #pragma redefine_extname __sync_fetch_and_sub_1_c __sync_fetch_and_sub_1
364 #pragma redefine_extname __sync_fetch_and_sub_2_c __sync_fetch_and_sub_2
365 #pragma redefine_extname __sync_fetch_and_sub_4_c __sync_fetch_and_sub_4
372 #if defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || \
373 defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || \
374 defined(__ARM_ARCH_6ZK__) || \
375 defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__)
377 /* Implementations for old GCC versions, lacking support for atomics. */
385 * Given a memory address pointing to an 8-bit or 16-bit integer, return
386 * the address of the 32-bit word containing it.
389 static inline uint32_t *
390 round_to_word(void *ptr)
393 return ((uint32_t *)((intptr_t)ptr & ~3));
397 * Utility functions for loading and storing 8-bit and 16-bit integers
398 * in 32-bit words at an offset corresponding with the location of the
403 put_1(reg_t *r, const uint8_t *offset_ptr, uint8_t val)
407 offset = (intptr_t)offset_ptr & 3;
411 static inline uint8_t
412 get_1(const reg_t *r, const uint8_t *offset_ptr)
416 offset = (intptr_t)offset_ptr & 3;
417 return (r->v8[offset]);
421 put_2(reg_t *r, const uint16_t *offset_ptr, uint16_t val)
429 offset = (intptr_t)offset_ptr & 3;
431 r->v8[offset] = bytes.out[0];
432 r->v8[offset + 1] = bytes.out[1];
435 static inline uint16_t
436 get_2(const reg_t *r, const uint16_t *offset_ptr)
444 offset = (intptr_t)offset_ptr & 3;
445 bytes.in[0] = r->v8[offset];
446 bytes.in[1] = r->v8[offset + 1];
451 * 8-bit and 16-bit routines.
453 * These operations are not natively supported by the CPU, so we use
454 * some shifting and bitmasking on top of the 32-bit instructions.
457 #define EMIT_LOCK_TEST_AND_SET_N(N, uintN_t) \
459 __sync_lock_test_and_set_##N##_c(uintN_t *mem, uintN_t val) \
462 reg_t val32, negmask, old; \
463 uint32_t temp1, temp2; \
465 mem32 = round_to_word(mem); \
466 val32.v32 = 0x00000000; \
467 put_##N(&val32, mem, val); \
468 negmask.v32 = 0xffffffff; \
469 put_##N(&negmask, mem, 0); \
474 "\tldrex %0, %6\n" /* Load old value. */ \
475 "\tand %2, %5, %0\n" /* Remove the old value. */ \
476 "\torr %2, %2, %4\n" /* Put in the new value. */ \
477 "\tstrex %3, %2, %1\n" /* Attempt to store. */ \
478 "\tcmp %3, #0\n" /* Did it succeed? */ \
479 "\tbne 1b\n" /* Spin if failed. */ \
480 : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp1), \
482 : "r" (val32.v32), "r" (negmask.v32), "m" (*mem32)); \
483 return (get_##N(&old, mem)); \
486 EMIT_LOCK_TEST_AND_SET_N(1, uint8_t)
487 EMIT_LOCK_TEST_AND_SET_N(2, uint16_t)
489 #define EMIT_VAL_COMPARE_AND_SWAP_N(N, uintN_t) \
491 __sync_val_compare_and_swap_##N##_c(uintN_t *mem, uintN_t expected, \
495 reg_t expected32, desired32, posmask, old; \
496 uint32_t negmask, temp1, temp2; \
498 mem32 = round_to_word(mem); \
499 expected32.v32 = 0x00000000; \
500 put_##N(&expected32, mem, expected); \
501 desired32.v32 = 0x00000000; \
502 put_##N(&desired32, mem, desired); \
503 posmask.v32 = 0x00000000; \
504 put_##N(&posmask, mem, ~0); \
505 negmask = ~posmask.v32; \
510 "\tldrex %0, %8\n" /* Load old value. */ \
511 "\tand %2, %6, %0\n" /* Isolate the old value. */ \
512 "\tcmp %2, %4\n" /* Compare to expected value. */\
513 "\tbne 2f\n" /* Values are unequal. */ \
514 "\tand %2, %7, %0\n" /* Remove the old value. */ \
515 "\torr %2, %5\n" /* Put in the new value. */ \
516 "\tstrex %3, %2, %1\n" /* Attempt to store. */ \
517 "\tcmp %3, #0\n" /* Did it succeed? */ \
518 "\tbne 1b\n" /* Spin if failed. */ \
520 : "=&r" (old), "=m" (*mem32), "=&r" (temp1), \
522 : "r" (expected32.v32), "r" (desired32.v32), \
523 "r" (posmask.v32), "r" (negmask), "m" (*mem32)); \
524 return (get_##N(&old, mem)); \
527 EMIT_VAL_COMPARE_AND_SWAP_N(1, uint8_t)
528 EMIT_VAL_COMPARE_AND_SWAP_N(2, uint16_t)
530 #define EMIT_ARITHMETIC_FETCH_AND_OP_N(N, uintN_t, name, op) \
532 __sync_##name##_##N##_c(uintN_t *mem, uintN_t val) \
535 reg_t val32, posmask, old; \
536 uint32_t negmask, temp1, temp2; \
538 mem32 = round_to_word(mem); \
539 val32.v32 = 0x00000000; \
540 put_##N(&val32, mem, val); \
541 posmask.v32 = 0x00000000; \
542 put_##N(&posmask, mem, ~0); \
543 negmask = ~posmask.v32; \
548 "\tldrex %0, %7\n" /* Load old value. */ \
549 "\t"op" %2, %0, %4\n" /* Calculate new value. */ \
550 "\tand %2, %5\n" /* Isolate the new value. */ \
551 "\tand %3, %6, %0\n" /* Remove the old value. */ \
552 "\torr %2, %2, %3\n" /* Put in the new value. */ \
553 "\tstrex %3, %2, %1\n" /* Attempt to store. */ \
554 "\tcmp %3, #0\n" /* Did it succeed? */ \
555 "\tbne 1b\n" /* Spin if failed. */ \
556 : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp1), \
558 : "r" (val32.v32), "r" (posmask.v32), "r" (negmask), \
560 return (get_##N(&old, mem)); \
563 EMIT_ARITHMETIC_FETCH_AND_OP_N(1, uint8_t, fetch_and_add, "add")
564 EMIT_ARITHMETIC_FETCH_AND_OP_N(1, uint8_t, fetch_and_sub, "sub")
565 EMIT_ARITHMETIC_FETCH_AND_OP_N(2, uint16_t, fetch_and_add, "add")
566 EMIT_ARITHMETIC_FETCH_AND_OP_N(2, uint16_t, fetch_and_sub, "sub")
568 #define EMIT_BITWISE_FETCH_AND_OP_N(N, uintN_t, name, op, idempotence) \
570 __sync_##name##_##N##_c(uintN_t *mem, uintN_t val) \
574 uint32_t temp1, temp2; \
576 mem32 = round_to_word(mem); \
577 val32.v32 = idempotence ? 0xffffffff : 0x00000000; \
578 put_##N(&val32, mem, val); \
583 "\tldrex %0, %5\n" /* Load old value. */ \
584 "\t"op" %2, %4, %0\n" /* Calculate new value. */ \
585 "\tstrex %3, %2, %1\n" /* Attempt to store. */ \
586 "\tcmp %3, #0\n" /* Did it succeed? */ \
587 "\tbne 1b\n" /* Spin if failed. */ \
588 : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp1), \
590 : "r" (val32.v32), "m" (*mem32)); \
591 return (get_##N(&old, mem)); \
594 EMIT_BITWISE_FETCH_AND_OP_N(1, uint8_t, fetch_and_and, "and", 1)
595 EMIT_BITWISE_FETCH_AND_OP_N(1, uint8_t, fetch_and_or, "orr", 0)
596 EMIT_BITWISE_FETCH_AND_OP_N(1, uint8_t, fetch_and_xor, "eor", 0)
597 EMIT_BITWISE_FETCH_AND_OP_N(2, uint16_t, fetch_and_and, "and", 1)
598 EMIT_BITWISE_FETCH_AND_OP_N(2, uint16_t, fetch_and_or, "orr", 0)
599 EMIT_BITWISE_FETCH_AND_OP_N(2, uint16_t, fetch_and_xor, "eor", 0)
606 __sync_lock_test_and_set_4_c(uint32_t *mem, uint32_t val)
613 "\tldrex %0, %4\n" /* Load old value. */
614 "\tstrex %2, %3, %1\n" /* Attempt to store. */
615 "\tcmp %2, #0\n" /* Did it succeed? */
616 "\tbne 1b\n" /* Spin if failed. */
617 : "=&r" (old), "=m" (*mem), "=&r" (temp)
618 : "r" (val), "m" (*mem));
623 __sync_val_compare_and_swap_4_c(uint32_t *mem, uint32_t expected,
631 "\tldrex %0, %5\n" /* Load old value. */
632 "\tcmp %0, %3\n" /* Compare to expected value. */
633 "\tbne 2f\n" /* Values are unequal. */
634 "\tstrex %2, %4, %1\n" /* Attempt to store. */
635 "\tcmp %2, #0\n" /* Did it succeed? */
636 "\tbne 1b\n" /* Spin if failed. */
638 : "=&r" (old), "=m" (*mem), "=&r" (temp)
639 : "r" (expected), "r" (desired), "m" (*mem));
643 #define EMIT_FETCH_AND_OP_4(name, op) \
645 __sync_##name##_4##_c(uint32_t *mem, uint32_t val) \
647 uint32_t old, temp1, temp2; \
652 "\tldrex %0, %5\n" /* Load old value. */ \
653 "\t"op" %2, %0, %4\n" /* Calculate new value. */ \
654 "\tstrex %3, %2, %1\n" /* Attempt to store. */ \
655 "\tcmp %3, #0\n" /* Did it succeed? */ \
656 "\tbne 1b\n" /* Spin if failed. */ \
657 : "=&r" (old), "=m" (*mem), "=&r" (temp1), \
659 : "r" (val), "m" (*mem)); \
663 EMIT_FETCH_AND_OP_4(fetch_and_add, "add")
664 EMIT_FETCH_AND_OP_4(fetch_and_and, "and")
665 EMIT_FETCH_AND_OP_4(fetch_and_or, "orr")
666 EMIT_FETCH_AND_OP_4(fetch_and_sub, "sub")
667 EMIT_FETCH_AND_OP_4(fetch_and_xor, "eor")
670 __strong_reference(__sync_lock_test_and_set_1_c, __sync_lock_test_and_set_1);
671 __strong_reference(__sync_lock_test_and_set_2_c, __sync_lock_test_and_set_2);
672 __strong_reference(__sync_lock_test_and_set_4_c, __sync_lock_test_and_set_4);
673 __strong_reference(__sync_val_compare_and_swap_1_c, __sync_val_compare_and_swap_1);
674 __strong_reference(__sync_val_compare_and_swap_2_c, __sync_val_compare_and_swap_2);
675 __strong_reference(__sync_val_compare_and_swap_4_c, __sync_val_compare_and_swap_4);
676 __strong_reference(__sync_fetch_and_add_1_c, __sync_fetch_and_add_1);
677 __strong_reference(__sync_fetch_and_add_2_c, __sync_fetch_and_add_2);
678 __strong_reference(__sync_fetch_and_add_4_c, __sync_fetch_and_add_4);
679 __strong_reference(__sync_fetch_and_and_1_c, __sync_fetch_and_and_1);
680 __strong_reference(__sync_fetch_and_and_2_c, __sync_fetch_and_and_2);
681 __strong_reference(__sync_fetch_and_and_4_c, __sync_fetch_and_and_4);
682 __strong_reference(__sync_fetch_and_sub_1_c, __sync_fetch_and_sub_1);
683 __strong_reference(__sync_fetch_and_sub_2_c, __sync_fetch_and_sub_2);
684 __strong_reference(__sync_fetch_and_sub_4_c, __sync_fetch_and_sub_4);
685 __strong_reference(__sync_fetch_and_or_1_c, __sync_fetch_and_or_1);
686 __strong_reference(__sync_fetch_and_or_2_c, __sync_fetch_and_or_2);
687 __strong_reference(__sync_fetch_and_or_4_c, __sync_fetch_and_or_4);
688 __strong_reference(__sync_fetch_and_xor_1_c, __sync_fetch_and_xor_1);
689 __strong_reference(__sync_fetch_and_xor_2_c, __sync_fetch_and_xor_2);
690 __strong_reference(__sync_fetch_and_xor_4_c, __sync_fetch_and_xor_4);
693 #else /* __ARM_ARCH_5__ */
698 #error "On SMP systems we should have proper atomic operations."
702 * On uniprocessor systems, we can perform the atomic operations by
703 * disabling interrupts.
706 #define EMIT_VAL_COMPARE_AND_SWAP_N(N, uintN_t) \
708 __sync_val_compare_and_swap_##N(uintN_t *mem, uintN_t expected, \
713 WITHOUT_INTERRUPTS({ \
715 if (*mem == expected) \
721 #define EMIT_FETCH_AND_OP_N(N, uintN_t, name, op) \
723 __sync_##name##_##N(uintN_t *mem, uintN_t val) \
727 WITHOUT_INTERRUPTS({ \
734 #define EMIT_ALL_OPS_N(N, uintN_t) \
735 EMIT_VAL_COMPARE_AND_SWAP_N(N, uintN_t) \
736 EMIT_FETCH_AND_OP_N(N, uintN_t, lock_test_and_set, =) \
737 EMIT_FETCH_AND_OP_N(N, uintN_t, fetch_and_add, +=) \
738 EMIT_FETCH_AND_OP_N(N, uintN_t, fetch_and_and, &=) \
739 EMIT_FETCH_AND_OP_N(N, uintN_t, fetch_and_or, |=) \
740 EMIT_FETCH_AND_OP_N(N, uintN_t, fetch_and_sub, -=) \
741 EMIT_FETCH_AND_OP_N(N, uintN_t, fetch_and_xor, ^=)
743 EMIT_ALL_OPS_N(1, uint8_t)
744 EMIT_ALL_OPS_N(2, uint16_t)
745 EMIT_ALL_OPS_N(4, uint32_t)
746 EMIT_ALL_OPS_N(8, uint64_t)
747 #undef EMIT_ALL_OPS_N
752 * For userspace on uniprocessor systems, we can implement the atomic
753 * operations by using a Restartable Atomic Sequence. This makes the
754 * kernel restart the code from the beginning when interrupted.
757 #define EMIT_LOCK_TEST_AND_SET_N(N, uintN_t, ldr, str) \
759 __sync_lock_test_and_set_##N##_c(uintN_t *mem, uintN_t val) \
761 uint32_t old, temp, ras_start; \
763 ras_start = ARM_RAS_START; \
765 /* Set up Restartable Atomic Sequence. */ \
770 "\tstr %2, [%5, #4]\n" \
772 "\t"ldr" %0, %4\n" /* Load old value. */ \
773 "\t"str" %3, %1\n" /* Store new value. */ \
775 /* Tear down Restartable Atomic Sequence. */ \
777 "\tmov %2, #0x00000000\n" \
779 "\tmov %2, #0xffffffff\n" \
780 "\tstr %2, [%5, #4]\n" \
781 : "=&r" (old), "=m" (*mem), "=&r" (temp) \
782 : "r" (val), "m" (*mem), "r" (ras_start)); \
786 #define EMIT_VAL_COMPARE_AND_SWAP_N(N, uintN_t, ldr, streq) \
788 __sync_val_compare_and_swap_##N##_c(uintN_t *mem, uintN_t expected, \
791 uint32_t old, temp, ras_start; \
793 ras_start = ARM_RAS_START; \
795 /* Set up Restartable Atomic Sequence. */ \
800 "\tstr %2, [%6, #4]\n" \
802 "\t"ldr" %0, %5\n" /* Load old value. */ \
803 "\tcmp %0, %3\n" /* Compare to expected value. */\
804 "\t"streq" %4, %1\n" /* Store new value. */ \
806 /* Tear down Restartable Atomic Sequence. */ \
808 "\tmov %2, #0x00000000\n" \
810 "\tmov %2, #0xffffffff\n" \
811 "\tstr %2, [%6, #4]\n" \
812 : "=&r" (old), "=m" (*mem), "=&r" (temp) \
813 : "r" (expected), "r" (desired), "m" (*mem), \
818 #define EMIT_FETCH_AND_OP_N(N, uintN_t, ldr, str, name, op) \
820 __sync_##name##_##N##_c(uintN_t *mem, uintN_t val) \
822 uint32_t old, temp, ras_start; \
824 ras_start = ARM_RAS_START; \
826 /* Set up Restartable Atomic Sequence. */ \
831 "\tstr %2, [%5, #4]\n" \
833 "\t"ldr" %0, %4\n" /* Load old value. */ \
834 "\t"op" %2, %0, %3\n" /* Calculate new value. */ \
835 "\t"str" %2, %1\n" /* Store new value. */ \
837 /* Tear down Restartable Atomic Sequence. */ \
839 "\tmov %2, #0x00000000\n" \
841 "\tmov %2, #0xffffffff\n" \
842 "\tstr %2, [%5, #4]\n" \
843 : "=&r" (old), "=m" (*mem), "=&r" (temp) \
844 : "r" (val), "m" (*mem), "r" (ras_start)); \
848 #define EMIT_ALL_OPS_N(N, uintN_t, ldr, str, streq) \
849 EMIT_LOCK_TEST_AND_SET_N(N, uintN_t, ldr, str) \
850 EMIT_VAL_COMPARE_AND_SWAP_N(N, uintN_t, ldr, streq) \
851 EMIT_FETCH_AND_OP_N(N, uintN_t, ldr, str, fetch_and_add, "add") \
852 EMIT_FETCH_AND_OP_N(N, uintN_t, ldr, str, fetch_and_and, "and") \
853 EMIT_FETCH_AND_OP_N(N, uintN_t, ldr, str, fetch_and_or, "orr") \
854 EMIT_FETCH_AND_OP_N(N, uintN_t, ldr, str, fetch_and_sub, "sub") \
855 EMIT_FETCH_AND_OP_N(N, uintN_t, ldr, str, fetch_and_xor, "eor")
857 EMIT_ALL_OPS_N(1, uint8_t, "ldrb", "strb", "streqb")
858 EMIT_ALL_OPS_N(2, uint16_t, "ldrh", "strh", "streqh")
859 EMIT_ALL_OPS_N(4, uint32_t, "ldr", "str", "streq")
862 __strong_reference(__sync_lock_test_and_set_1_c, __sync_lock_test_and_set_1);
863 __strong_reference(__sync_lock_test_and_set_2_c, __sync_lock_test_and_set_2);
864 __strong_reference(__sync_lock_test_and_set_4_c, __sync_lock_test_and_set_4);
865 __strong_reference(__sync_val_compare_and_swap_1_c, __sync_val_compare_and_swap_1);
866 __strong_reference(__sync_val_compare_and_swap_2_c, __sync_val_compare_and_swap_2);
867 __strong_reference(__sync_val_compare_and_swap_4_c, __sync_val_compare_and_swap_4);
868 __strong_reference(__sync_fetch_and_add_1_c, __sync_fetch_and_add_1);
869 __strong_reference(__sync_fetch_and_add_2_c, __sync_fetch_and_add_2);
870 __strong_reference(__sync_fetch_and_add_4_c, __sync_fetch_and_add_4);
871 __strong_reference(__sync_fetch_and_and_1_c, __sync_fetch_and_and_1);
872 __strong_reference(__sync_fetch_and_and_2_c, __sync_fetch_and_and_2);
873 __strong_reference(__sync_fetch_and_and_4_c, __sync_fetch_and_and_4);
874 __strong_reference(__sync_fetch_and_sub_1_c, __sync_fetch_and_sub_1);
875 __strong_reference(__sync_fetch_and_sub_2_c, __sync_fetch_and_sub_2);
876 __strong_reference(__sync_fetch_and_sub_4_c, __sync_fetch_and_sub_4);
877 __strong_reference(__sync_fetch_and_or_1_c, __sync_fetch_and_or_1);
878 __strong_reference(__sync_fetch_and_or_2_c, __sync_fetch_and_or_2);
879 __strong_reference(__sync_fetch_and_or_4_c, __sync_fetch_and_or_4);
880 __strong_reference(__sync_fetch_and_xor_1_c, __sync_fetch_and_xor_1);
881 __strong_reference(__sync_fetch_and_xor_2_c, __sync_fetch_and_xor_2);
882 __strong_reference(__sync_fetch_and_xor_4_c, __sync_fetch_and_xor_4);
889 #endif /* __SYNC_ATOMICS */