1 /* $NetBSD: atomic.h,v 1.1 2002/10/19 12:22:34 bsh Exp $ */
4 * Copyright (C) 2003-2004 Olivier Houchard
5 * Copyright (C) 1994-1997 Mark Brinicombe
6 * Copyright (C) 1994 Brini
9 * This code is derived from software written for Brini by Mark Brinicombe
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 * 3. All advertising materials mentioning features or use of this software
20 * must display the following acknowledgement:
21 * This product includes software developed by Brini.
22 * 4. The name of Brini may not be used to endorse or promote products
23 * derived from this software without specific prior written permission.
25 * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR
26 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
27 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
28 * IN NO EVENT SHALL BRINI BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
29 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
30 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
31 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
33 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
34 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
39 #ifndef _MACHINE_ATOMIC_H_
40 #define _MACHINE_ATOMIC_H_
42 #include <sys/types.h>
45 #include <machine/sysarch.h>
47 #include <machine/cpuconf.h>
50 #if defined (__ARM_ARCH_7__) || defined (__ARM_ARCH_7A__)
51 #define isb() __asm __volatile("isb" : : : "memory")
52 #define dsb() __asm __volatile("dsb" : : : "memory")
53 #define dmb() __asm __volatile("dmb" : : : "memory")
54 #elif defined (__ARM_ARCH_6__) || defined (__ARM_ARCH_6J__) || \
55 defined (__ARM_ARCH_6K__) || defined (__ARM_ARCH_6T2__) || \
56 defined (__ARM_ARCH_6Z__) || defined (__ARM_ARCH_6ZK__)
57 #define isb() __asm __volatile("mcr p15, 0, %0, c7, c5, 4" : : "r" (0) : "memory")
58 #define dsb() __asm __volatile("mcr p15, 0, %0, c7, c10, 4" : : "r" (0) : "memory")
59 #define dmb() __asm __volatile("mcr p15, 0, %0, c7, c10, 5" : : "r" (0) : "memory")
71 #define I32_bit (1 << 7) /* IRQ disable */
74 #define F32_bit (1 << 6) /* FIQ disable */
78 * It would be nice to use _HAVE_ARMv6_INSTRUCTIONS from machine/asm.h
79 * here, but that header can't be included here because this is C
80 * code. I would like to move the _HAVE_ARMv6_INSTRUCTIONS definition
81 * out of asm.h so it can be used in both asm and C code. - kientzle@
83 #if defined (__ARM_ARCH_7__) || \
84 defined (__ARM_ARCH_7A__) || \
85 defined (__ARM_ARCH_6__) || \
86 defined (__ARM_ARCH_6J__) || \
87 defined (__ARM_ARCH_6K__) || \
88 defined (__ARM_ARCH_6T2__) || \
89 defined (__ARM_ARCH_6Z__) || \
90 defined (__ARM_ARCH_6ZK__)
95 #if defined (__ARM_ARCH_7__) || defined (__ARM_ARCH_7A__)
96 __asm __volatile("dmb" : : : "memory");
98 __asm __volatile("mcr p15, 0, r0, c7, c10, 5" : : : "memory");
102 #define ATOMIC_ACQ_REL_LONG(NAME) \
103 static __inline void \
104 atomic_##NAME##_acq_long(__volatile u_long *p, u_long v) \
106 atomic_##NAME##_long(p, v); \
110 static __inline void \
111 atomic_##NAME##_rel_long(__volatile u_long *p, u_long v) \
114 atomic_##NAME##_long(p, v); \
117 #define ATOMIC_ACQ_REL(NAME, WIDTH) \
118 static __inline void \
119 atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\
121 atomic_##NAME##_##WIDTH(p, v); \
125 static __inline void \
126 atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\
129 atomic_##NAME##_##WIDTH(p, v); \
133 atomic_set_32(volatile uint32_t *address, uint32_t setmask)
135 uint32_t tmp = 0, tmp2 = 0;
137 __asm __volatile("1: ldrex %0, [%2]\n"
139 "strex %1, %0, [%2]\n"
143 : "=&r" (tmp), "+r" (tmp2)
144 , "+r" (address), "+r" (setmask) : : "cc", "memory");
149 atomic_set_long(volatile u_long *address, u_long setmask)
151 u_long tmp = 0, tmp2 = 0;
153 __asm __volatile("1: ldrex %0, [%2]\n"
155 "strex %1, %0, [%2]\n"
159 : "=&r" (tmp), "+r" (tmp2)
160 , "+r" (address), "+r" (setmask) : : "cc", "memory");
165 atomic_clear_32(volatile uint32_t *address, uint32_t setmask)
167 uint32_t tmp = 0, tmp2 = 0;
169 __asm __volatile("1: ldrex %0, [%2]\n"
171 "strex %1, %0, [%2]\n"
175 : "=&r" (tmp), "+r" (tmp2)
176 ,"+r" (address), "+r" (setmask) : : "cc", "memory");
180 atomic_clear_long(volatile u_long *address, u_long setmask)
182 u_long tmp = 0, tmp2 = 0;
184 __asm __volatile("1: ldrex %0, [%2]\n"
186 "strex %1, %0, [%2]\n"
190 : "=&r" (tmp), "+r" (tmp2)
191 ,"+r" (address), "+r" (setmask) : : "cc", "memory");
194 static __inline u_int32_t
195 atomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval)
199 __asm __volatile("1: ldrex %0, [%1]\n"
204 "strex %0, %3, [%1]\n"
211 ,"+r" (p), "+r" (cmpval), "+r" (newval) : : "cc",
216 static __inline u_long
217 atomic_cmpset_long(volatile u_long *p, volatile u_long cmpval, volatile u_long newval)
221 __asm __volatile("1: ldrex %0, [%1]\n"
226 "strex %0, %3, [%1]\n"
233 ,"+r" (p), "+r" (cmpval), "+r" (newval) : : "cc",
238 static __inline u_int32_t
239 atomic_cmpset_acq_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval)
241 u_int32_t ret = atomic_cmpset_32(p, cmpval, newval);
247 static __inline u_long
248 atomic_cmpset_acq_long(volatile u_long *p, volatile u_long cmpval, volatile u_long newval)
250 u_long ret = atomic_cmpset_long(p, cmpval, newval);
256 static __inline u_int32_t
257 atomic_cmpset_rel_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval)
261 return (atomic_cmpset_32(p, cmpval, newval));
264 static __inline u_long
265 atomic_cmpset_rel_long(volatile u_long *p, volatile u_long cmpval, volatile u_long newval)
269 return (atomic_cmpset_long(p, cmpval, newval));
274 atomic_add_32(volatile u_int32_t *p, u_int32_t val)
276 uint32_t tmp = 0, tmp2 = 0;
278 __asm __volatile("1: ldrex %0, [%2]\n"
280 "strex %1, %0, [%2]\n"
284 : "=&r" (tmp), "+r" (tmp2)
285 ,"+r" (p), "+r" (val) : : "cc", "memory");
289 atomic_add_long(volatile u_long *p, u_long val)
291 u_long tmp = 0, tmp2 = 0;
293 __asm __volatile("1: ldrex %0, [%2]\n"
295 "strex %1, %0, [%2]\n"
299 : "=&r" (tmp), "+r" (tmp2)
300 ,"+r" (p), "+r" (val) : : "cc", "memory");
304 atomic_subtract_32(volatile u_int32_t *p, u_int32_t val)
306 uint32_t tmp = 0, tmp2 = 0;
308 __asm __volatile("1: ldrex %0, [%2]\n"
310 "strex %1, %0, [%2]\n"
314 : "=&r" (tmp), "+r" (tmp2)
315 ,"+r" (p), "+r" (val) : : "cc", "memory");
319 atomic_subtract_long(volatile u_long *p, u_long val)
321 u_long tmp = 0, tmp2 = 0;
323 __asm __volatile("1: ldrex %0, [%2]\n"
325 "strex %1, %0, [%2]\n"
329 : "=&r" (tmp), "+r" (tmp2)
330 ,"+r" (p), "+r" (val) : : "cc", "memory");
333 ATOMIC_ACQ_REL(clear, 32)
334 ATOMIC_ACQ_REL(add, 32)
335 ATOMIC_ACQ_REL(subtract, 32)
336 ATOMIC_ACQ_REL(set, 32)
337 ATOMIC_ACQ_REL_LONG(clear)
338 ATOMIC_ACQ_REL_LONG(add)
339 ATOMIC_ACQ_REL_LONG(subtract)
340 ATOMIC_ACQ_REL_LONG(set)
342 #undef ATOMIC_ACQ_REL
343 #undef ATOMIC_ACQ_REL_LONG
345 static __inline uint32_t
346 atomic_fetchadd_32(volatile uint32_t *p, uint32_t val)
348 uint32_t tmp = 0, tmp2 = 0, ret = 0;
350 __asm __volatile("1: ldrex %0, [%3]\n"
352 "strex %2, %1, [%3]\n"
356 : "+r" (ret), "=&r" (tmp), "+r" (tmp2)
357 ,"+r" (p), "+r" (val) : : "cc", "memory");
361 static __inline uint32_t
362 atomic_readandclear_32(volatile u_int32_t *p)
364 uint32_t ret, tmp = 0, tmp2 = 0;
366 __asm __volatile("1: ldrex %0, [%3]\n"
368 "strex %2, %1, [%3]\n"
372 : "=r" (ret), "=&r" (tmp), "+r" (tmp2)
373 ,"+r" (p) : : "cc", "memory");
377 static __inline uint32_t
378 atomic_load_acq_32(volatile uint32_t *p)
388 atomic_store_rel_32(volatile uint32_t *p, uint32_t v)
395 static __inline u_long
396 atomic_fetchadd_long(volatile u_long *p, u_long val)
398 u_long tmp = 0, tmp2 = 0, ret = 0;
400 __asm __volatile("1: ldrex %0, [%3]\n"
402 "strex %2, %1, [%3]\n"
406 : "+r" (ret), "=&r" (tmp), "+r" (tmp2)
407 ,"+r" (p), "+r" (val) : : "cc", "memory");
411 static __inline u_long
412 atomic_readandclear_long(volatile u_long *p)
414 u_long ret, tmp = 0, tmp2 = 0;
416 __asm __volatile("1: ldrex %0, [%3]\n"
418 "strex %2, %1, [%3]\n"
422 : "=r" (ret), "=&r" (tmp), "+r" (tmp2)
423 ,"+r" (p) : : "cc", "memory");
427 static __inline u_long
428 atomic_load_acq_long(volatile u_long *p)
438 atomic_store_rel_long(volatile u_long *p, u_long v)
446 #define __with_interrupts_disabled(expr) \
448 u_int cpsr_save, tmp; \
453 "msr cpsr_all, %1;" \
454 : "=r" (cpsr_save), "=r" (tmp) \
455 : "I" (I32_bit | F32_bit) \
465 static __inline uint32_t
466 __swp(uint32_t val, volatile uint32_t *ptr)
468 __asm __volatile("swp %0, %2, [%3]"
469 : "=&r" (val), "=m" (*ptr)
470 : "r" (val), "r" (ptr), "m" (*ptr)
478 atomic_set_32(volatile uint32_t *address, uint32_t setmask)
480 __with_interrupts_disabled(*address |= setmask);
484 atomic_clear_32(volatile uint32_t *address, uint32_t clearmask)
486 __with_interrupts_disabled(*address &= ~clearmask);
489 static __inline u_int32_t
490 atomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval)
494 __with_interrupts_disabled(
507 atomic_add_32(volatile u_int32_t *p, u_int32_t val)
509 __with_interrupts_disabled(*p += val);
513 atomic_subtract_32(volatile u_int32_t *p, u_int32_t val)
515 __with_interrupts_disabled(*p -= val);
518 static __inline uint32_t
519 atomic_fetchadd_32(volatile uint32_t *p, uint32_t v)
523 __with_interrupts_disabled(
533 static __inline u_int32_t
534 atomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval)
536 register int done, ras_start = ARM_RAS_START;
538 __asm __volatile("1:\n"
549 "mov %1, #0xffffffff\n"
553 : "+r" (ras_start), "=r" (done)
554 ,"+r" (p), "+r" (cmpval), "+r" (newval) : : "cc", "memory");
559 atomic_add_32(volatile u_int32_t *p, u_int32_t val)
561 int start, ras_start = ARM_RAS_START;
563 __asm __volatile("1:\n"
574 "mov %1, #0xffffffff\n"
576 : "+r" (ras_start), "=r" (start), "+r" (p), "+r" (val)
581 atomic_subtract_32(volatile u_int32_t *p, u_int32_t val)
583 int start, ras_start = ARM_RAS_START;
585 __asm __volatile("1:\n"
596 "mov %1, #0xffffffff\n"
599 : "+r" (ras_start), "=r" (start), "+r" (p), "+r" (val)
604 atomic_set_32(volatile uint32_t *address, uint32_t setmask)
606 int start, ras_start = ARM_RAS_START;
608 __asm __volatile("1:\n"
619 "mov %1, #0xffffffff\n"
622 : "+r" (ras_start), "=r" (start), "+r" (address), "+r" (setmask)
627 atomic_clear_32(volatile uint32_t *address, uint32_t clearmask)
629 int start, ras_start = ARM_RAS_START;
631 __asm __volatile("1:\n"
642 "mov %1, #0xffffffff\n"
644 : "+r" (ras_start), "=r" (start), "+r" (address), "+r" (clearmask)
649 static __inline uint32_t
650 atomic_fetchadd_32(volatile uint32_t *p, uint32_t v)
652 uint32_t start, tmp, ras_start = ARM_RAS_START;
654 __asm __volatile("1:\n"
666 "mov %2, #0xffffffff\n"
668 : "+r" (ras_start), "=r" (start), "=r" (tmp), "+r" (p), "+r" (v)
676 static __inline uint32_t
677 atomic_readandclear_32(volatile u_int32_t *p)
680 return (__swp(0, p));
683 #define atomic_cmpset_rel_32 atomic_cmpset_32
684 #define atomic_cmpset_acq_32 atomic_cmpset_32
685 #define atomic_set_rel_32 atomic_set_32
686 #define atomic_set_acq_32 atomic_set_32
687 #define atomic_clear_rel_32 atomic_clear_32
688 #define atomic_clear_acq_32 atomic_clear_32
689 #define atomic_add_rel_32 atomic_add_32
690 #define atomic_add_acq_32 atomic_add_32
691 #define atomic_subtract_rel_32 atomic_subtract_32
692 #define atomic_subtract_acq_32 atomic_subtract_32
693 #define atomic_store_rel_32 atomic_store_32
694 #define atomic_store_rel_long atomic_store_long
695 #define atomic_load_acq_32 atomic_load_32
696 #define atomic_load_acq_long atomic_load_long
697 #define atomic_add_acq_long atomic_add_long
698 #define atomic_add_rel_long atomic_add_long
699 #define atomic_subtract_acq_long atomic_subtract_long
700 #define atomic_subtract_rel_long atomic_subtract_long
701 #define atomic_clear_acq_long atomic_clear_long
702 #define atomic_clear_rel_long atomic_clear_long
703 #define atomic_set_acq_long atomic_set_long
704 #define atomic_set_rel_long atomic_set_long
705 #define atomic_cmpset_acq_long atomic_cmpset_long
706 #define atomic_cmpset_rel_long atomic_cmpset_long
707 #define atomic_load_acq_long atomic_load_long
708 #undef __with_interrupts_disabled
711 atomic_add_long(volatile u_long *p, u_long v)
714 atomic_add_32((volatile uint32_t *)p, v);
718 atomic_clear_long(volatile u_long *p, u_long v)
721 atomic_clear_32((volatile uint32_t *)p, v);
725 atomic_cmpset_long(volatile u_long *dst, u_long old, u_long newe)
728 return (atomic_cmpset_32((volatile uint32_t *)dst, old, newe));
731 static __inline u_long
732 atomic_fetchadd_long(volatile u_long *p, u_long v)
735 return (atomic_fetchadd_32((volatile uint32_t *)p, v));
739 atomic_readandclear_long(volatile u_long *p)
742 atomic_readandclear_32((volatile uint32_t *)p);
746 atomic_set_long(volatile u_long *p, u_long v)
749 atomic_set_32((volatile uint32_t *)p, v);
753 atomic_subtract_long(volatile u_long *p, u_long v)
756 atomic_subtract_32((volatile uint32_t *)p, v);
761 #endif /* Arch >= v6 */
764 atomic_load_32(volatile uint32_t *v)
771 atomic_store_32(volatile uint32_t *dst, uint32_t src)
777 atomic_load_long(volatile u_long *v)
784 atomic_store_long(volatile u_long *dst, u_long src)
789 #define atomic_clear_ptr atomic_clear_32
790 #define atomic_set_ptr atomic_set_32
791 #define atomic_cmpset_ptr atomic_cmpset_32
792 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_32
793 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_32
794 #define atomic_store_ptr atomic_store_32
795 #define atomic_store_rel_ptr atomic_store_rel_32
797 #define atomic_add_int atomic_add_32
798 #define atomic_add_acq_int atomic_add_acq_32
799 #define atomic_add_rel_int atomic_add_rel_32
800 #define atomic_subtract_int atomic_subtract_32
801 #define atomic_subtract_acq_int atomic_subtract_acq_32
802 #define atomic_subtract_rel_int atomic_subtract_rel_32
803 #define atomic_clear_int atomic_clear_32
804 #define atomic_clear_acq_int atomic_clear_acq_32
805 #define atomic_clear_rel_int atomic_clear_rel_32
806 #define atomic_set_int atomic_set_32
807 #define atomic_set_acq_int atomic_set_acq_32
808 #define atomic_set_rel_int atomic_set_rel_32
809 #define atomic_cmpset_int atomic_cmpset_32
810 #define atomic_cmpset_acq_int atomic_cmpset_acq_32
811 #define atomic_cmpset_rel_int atomic_cmpset_rel_32
812 #define atomic_fetchadd_int atomic_fetchadd_32
813 #define atomic_readandclear_int atomic_readandclear_32
814 #define atomic_load_acq_int atomic_load_acq_32
815 #define atomic_store_rel_int atomic_store_rel_32
817 #endif /* _MACHINE_ATOMIC_H_ */