2 * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
5 * Portions of this software were developed by SRI International and the
6 * University of Cambridge Computer Laboratory under DARPA/AFRL contract
7 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
9 * Portions of this software were developed by the University of Cambridge
10 * Computer Laboratory as part of the CTSRD Project, with support from the
11 * UK Higher Education Innovation Fund (HEIF).
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
37 #ifndef _MACHINE_ATOMIC_H_
38 #define _MACHINE_ATOMIC_H_
40 #include <sys/atomic_common.h>
42 #define fence() __asm __volatile("fence" ::: "memory");
47 #define ATOMIC_ACQ_REL(NAME, WIDTH) \
48 static __inline void \
49 atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\
51 atomic_##NAME##_##WIDTH(p, v); \
55 static __inline void \
56 atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\
59 atomic_##NAME##_##WIDTH(p, v); \
63 atomic_add_32(volatile uint32_t *p, uint32_t val)
66 __asm __volatile("amoadd.w zero, %1, %0"
73 atomic_subtract_32(volatile uint32_t *p, uint32_t val)
76 __asm __volatile("amoadd.w zero, %1, %0"
83 atomic_set_32(volatile uint32_t *p, uint32_t val)
86 __asm __volatile("amoor.w zero, %1, %0"
93 atomic_clear_32(volatile uint32_t *p, uint32_t val)
96 __asm __volatile("amoand.w zero, %1, %0"
103 atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)
112 "li %1, 1\n" /* Preset to fail */
118 : "=&r" (tmp), "=&r" (res), "+A" (*p)
119 : "rJ" (cmpval), "rJ" (newval)
126 atomic_fcmpset_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval)
135 "li %1, 1\n" /* Preset to fail */
136 "lr.w %0, %2\n" /* Load old value */
137 "bne %0, %z4, 1f\n" /* Compare */
138 "sc.w %1, %z5, %2\n" /* Try to store new value */
141 "sw %0, %3\n" /* Save old value */
143 : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval)
144 : "rJ" (*cmpval), "rJ" (newval)
150 static __inline uint32_t
151 atomic_fetchadd_32(volatile uint32_t *p, uint32_t val)
155 __asm __volatile("amoadd.w %0, %2, %1"
156 : "=&r" (ret), "+A" (*p)
163 static __inline uint32_t
164 atomic_readandclear_32(volatile uint32_t *p)
171 __asm __volatile("amoswap.w %0, %2, %1"
172 : "=&r"(ret), "+A" (*p)
179 #define atomic_add_int atomic_add_32
180 #define atomic_clear_int atomic_clear_32
181 #define atomic_cmpset_int atomic_cmpset_32
182 #define atomic_fcmpset_int atomic_fcmpset_32
183 #define atomic_fetchadd_int atomic_fetchadd_32
184 #define atomic_readandclear_int atomic_readandclear_32
185 #define atomic_set_int atomic_set_32
186 #define atomic_subtract_int atomic_subtract_32
188 ATOMIC_ACQ_REL(set, 32)
189 ATOMIC_ACQ_REL(clear, 32)
190 ATOMIC_ACQ_REL(add, 32)
191 ATOMIC_ACQ_REL(subtract, 32)
194 atomic_cmpset_acq_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)
198 res = atomic_cmpset_32(p, cmpval, newval);
206 atomic_cmpset_rel_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)
211 return (atomic_cmpset_32(p, cmpval, newval));
215 atomic_fcmpset_acq_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval)
219 res = atomic_fcmpset_32(p, cmpval, newval);
227 atomic_fcmpset_rel_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval)
232 return (atomic_fcmpset_32(p, cmpval, newval));
235 static __inline uint32_t
236 atomic_load_acq_32(volatile uint32_t *p)
248 atomic_store_rel_32(volatile uint32_t *p, uint32_t val)
256 #define atomic_add_acq_int atomic_add_acq_32
257 #define atomic_clear_acq_int atomic_clear_acq_32
258 #define atomic_cmpset_acq_int atomic_cmpset_acq_32
259 #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32
260 #define atomic_load_acq_int atomic_load_acq_32
261 #define atomic_set_acq_int atomic_set_acq_32
262 #define atomic_subtract_acq_int atomic_subtract_acq_32
264 #define atomic_add_rel_int atomic_add_rel_32
265 #define atomic_clear_rel_int atomic_add_rel_32
266 #define atomic_cmpset_rel_int atomic_cmpset_rel_32
267 #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32
268 #define atomic_set_rel_int atomic_set_rel_32
269 #define atomic_subtract_rel_int atomic_subtract_rel_32
270 #define atomic_store_rel_int atomic_store_rel_32
273 atomic_add_64(volatile uint64_t *p, uint64_t val)
276 __asm __volatile("amoadd.d zero, %1, %0"
283 atomic_subtract_64(volatile uint64_t *p, uint64_t val)
286 __asm __volatile("amoadd.d zero, %1, %0"
293 atomic_set_64(volatile uint64_t *p, uint64_t val)
296 __asm __volatile("amoor.d zero, %1, %0"
303 atomic_clear_64(volatile uint64_t *p, uint64_t val)
306 __asm __volatile("amoand.d zero, %1, %0"
313 atomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval)
322 "li %1, 1\n" /* Preset to fail */
328 : "=&r" (tmp), "=&r" (res), "+A" (*p)
329 : "rJ" (cmpval), "rJ" (newval)
336 atomic_fcmpset_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval)
345 "li %1, 1\n" /* Preset to fail */
346 "lr.d %0, %2\n" /* Load old value */
347 "bne %0, %z4, 1f\n" /* Compare */
348 "sc.d %1, %z5, %2\n" /* Try to store new value */
351 "sd %0, %3\n" /* Save old value */
353 : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval)
354 : "rJ" (*cmpval), "rJ" (newval)
360 static __inline uint64_t
361 atomic_fetchadd_64(volatile uint64_t *p, uint64_t val)
365 __asm __volatile("amoadd.d %0, %2, %1"
366 : "=&r" (ret), "+A" (*p)
373 static __inline uint64_t
374 atomic_readandclear_64(volatile uint64_t *p)
381 __asm __volatile("amoswap.d %0, %2, %1"
382 : "=&r"(ret), "+A" (*p)
389 static __inline uint32_t
390 atomic_swap_32(volatile uint32_t *p, uint32_t val)
394 __asm __volatile("amoswap.w %0, %2, %1"
395 : "=&r"(old), "+A" (*p)
402 static __inline uint64_t
403 atomic_swap_64(volatile uint64_t *p, uint64_t val)
407 __asm __volatile("amoswap.d %0, %2, %1"
408 : "=&r"(old), "+A" (*p)
415 #define atomic_add_long atomic_add_64
416 #define atomic_clear_long atomic_clear_64
417 #define atomic_cmpset_long atomic_cmpset_64
418 #define atomic_fcmpset_long atomic_fcmpset_64
419 #define atomic_fetchadd_long atomic_fetchadd_64
420 #define atomic_readandclear_long atomic_readandclear_64
421 #define atomic_set_long atomic_set_64
422 #define atomic_subtract_long atomic_subtract_64
424 #define atomic_add_ptr atomic_add_64
425 #define atomic_clear_ptr atomic_clear_64
426 #define atomic_cmpset_ptr atomic_cmpset_64
427 #define atomic_fcmpset_ptr atomic_fcmpset_64
428 #define atomic_fetchadd_ptr atomic_fetchadd_64
429 #define atomic_readandclear_ptr atomic_readandclear_64
430 #define atomic_set_ptr atomic_set_64
431 #define atomic_subtract_ptr atomic_subtract_64
433 ATOMIC_ACQ_REL(set, 64)
434 ATOMIC_ACQ_REL(clear, 64)
435 ATOMIC_ACQ_REL(add, 64)
436 ATOMIC_ACQ_REL(subtract, 64)
439 atomic_cmpset_acq_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval)
443 res = atomic_cmpset_64(p, cmpval, newval);
451 atomic_cmpset_rel_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval)
456 return (atomic_cmpset_64(p, cmpval, newval));
460 atomic_fcmpset_acq_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval)
464 res = atomic_fcmpset_64(p, cmpval, newval);
472 atomic_fcmpset_rel_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval)
477 return (atomic_fcmpset_64(p, cmpval, newval));
480 static __inline uint64_t
481 atomic_load_acq_64(volatile uint64_t *p)
493 atomic_store_rel_64(volatile uint64_t *p, uint64_t val)
501 #define atomic_add_acq_long atomic_add_acq_64
502 #define atomic_clear_acq_long atomic_add_acq_64
503 #define atomic_cmpset_acq_long atomic_cmpset_acq_64
504 #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64
505 #define atomic_load_acq_long atomic_load_acq_64
506 #define atomic_set_acq_long atomic_set_acq_64
507 #define atomic_subtract_acq_long atomic_subtract_acq_64
509 #define atomic_add_acq_ptr atomic_add_acq_64
510 #define atomic_clear_acq_ptr atomic_add_acq_64
511 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64
512 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64
513 #define atomic_load_acq_ptr atomic_load_acq_64
514 #define atomic_set_acq_ptr atomic_set_acq_64
515 #define atomic_subtract_acq_ptr atomic_subtract_acq_64
517 #undef ATOMIC_ACQ_REL
520 atomic_thread_fence_acq(void)
527 atomic_thread_fence_rel(void)
534 atomic_thread_fence_acq_rel(void)
541 atomic_thread_fence_seq_cst(void)
547 #define atomic_add_rel_long atomic_add_rel_64
548 #define atomic_clear_rel_long atomic_clear_rel_64
550 #define atomic_add_rel_long atomic_add_rel_64
551 #define atomic_clear_rel_long atomic_clear_rel_64
552 #define atomic_cmpset_rel_long atomic_cmpset_rel_64
553 #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64
554 #define atomic_set_rel_long atomic_set_rel_64
555 #define atomic_subtract_rel_long atomic_subtract_rel_64
556 #define atomic_store_rel_long atomic_store_rel_64
558 #define atomic_add_rel_ptr atomic_add_rel_64
559 #define atomic_clear_rel_ptr atomic_clear_rel_64
560 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64
561 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64
562 #define atomic_set_rel_ptr atomic_set_rel_64
563 #define atomic_subtract_rel_ptr atomic_subtract_rel_64
564 #define atomic_store_rel_ptr atomic_store_rel_64
566 #endif /* _MACHINE_ATOMIC_H_ */