2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 1998 Doug Rabson.
5 * Copyright (c) 2001 Jake Burkholder.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * from: FreeBSD: src/sys/i386/include/atomic.h,v 1.20 2001/02/11
33 #ifndef _MACHINE_ATOMIC_H_
34 #define _MACHINE_ATOMIC_H_
36 #include <machine/cpufunc.h>
38 #define mb() __asm__ __volatile__ ("membar #MemIssue": : :"memory")
42 #include <sys/atomic_common.h>
44 /* Userland needs different ASI's. */
46 #define __ASI_ATOMIC ASI_N
48 #define __ASI_ATOMIC ASI_P
52 * Various simple arithmetic on memory which is atomic in the presence
53 * of interrupts and multiple processors. See atomic(9) for details.
54 * Note that efficient hardware support exists only for the 32 and 64
55 * bit variants; the 8 and 16 bit versions are not provided and should
56 * not be used in MI code.
58 * This implementation takes advantage of the fact that the sparc64
59 * cas instruction is both a load and a store. The loop is often coded
65 * } while (cas(p, expect, new) != expect);
67 * which performs an unnnecessary load on each iteration that the cas
68 * operation fails. Modified as follows:
73 * result = cas(p, expect, new);
74 * if (result == expect)
79 * the return value of cas is used to avoid the extra reload.
81 * We only include a memory barrier in the rel variants as in total store
82 * order which we use for running the kernel and all of the userland atomic
83 * loads and stores behave as if the were followed by a membar with a mask
84 * of #LoadLoad | #LoadStore | #StoreStore. In order to be also sufficient
85 * for use of relaxed memory ordering, the atomic_cas() in the acq variants
86 * additionally would have to be followed by a membar #LoadLoad | #LoadStore.
87 * Due to the suggested assembly syntax of the membar operands containing a
88 * # character, they cannot be used in macros. The cmask and mmask bits thus
89 * are hard coded in machine/cpufunc.h and used here through macros.
90 * Hopefully the bit numbers won't change in the future.
93 #define itype(sz) uint ## sz ## _t
95 #define atomic_cas_32(p, e, s) casa((p), (e), (s), __ASI_ATOMIC)
96 #define atomic_cas_64(p, e, s) casxa((p), (e), (s), __ASI_ATOMIC)
98 #define atomic_cas(p, e, s, sz) \
99 atomic_cas_ ## sz((p), (e), (s))
101 #define atomic_cas_acq(p, e, s, sz) ({ \
103 v = atomic_cas((p), (e), (s), sz); \
104 __compiler_membar(); \
108 #define atomic_cas_rel(p, e, s, sz) ({ \
110 membar(LoadStore | StoreStore); \
111 v = atomic_cas((p), (e), (s), sz); \
115 #define atomic_op(p, op, v, sz) ({ \
117 for (e = *(volatile itype(sz) *)(p);; e = r) { \
119 r = atomic_cas_ ## sz((p), e, s); \
126 #define atomic_op_acq(p, op, v, sz) ({ \
128 t = atomic_op((p), op, (v), sz); \
129 __compiler_membar(); \
133 #define atomic_op_rel(p, op, v, sz) ({ \
135 membar(LoadStore | StoreStore); \
136 t = atomic_op((p), op, (v), sz); \
140 #define atomic_ld_acq(p, sz) ({ \
142 v = atomic_cas((p), 0, 0, sz); \
143 __compiler_membar(); \
147 #define atomic_ld_clear(p, sz) ({ \
149 for (e = *(volatile itype(sz) *)(p);; e = r) { \
150 r = atomic_cas((p), e, 0, sz); \
157 #define atomic_st(p, v, sz) do { \
159 for (e = *(volatile itype(sz) *)(p);; e = r) { \
160 r = atomic_cas((p), e, (v), sz); \
166 #define atomic_st_acq(p, v, sz) do { \
167 atomic_st((p), (v), sz); \
168 __compiler_membar(); \
171 #define atomic_st_rel(p, v, sz) do { \
172 membar(LoadStore | StoreStore); \
173 atomic_st((p), (v), sz); \
176 #define ATOMIC_GEN(name, ptype, vtype, atype, sz) \
178 static __inline vtype \
179 atomic_add_ ## name(volatile ptype p, atype v) \
181 return ((vtype)atomic_op((p), +, (v), sz)); \
183 static __inline vtype \
184 atomic_add_acq_ ## name(volatile ptype p, atype v) \
186 return ((vtype)atomic_op_acq((p), +, (v), sz)); \
188 static __inline vtype \
189 atomic_add_rel_ ## name(volatile ptype p, atype v) \
191 return ((vtype)atomic_op_rel((p), +, (v), sz)); \
194 static __inline vtype \
195 atomic_clear_ ## name(volatile ptype p, atype v) \
197 return ((vtype)atomic_op((p), &, ~(v), sz)); \
199 static __inline vtype \
200 atomic_clear_acq_ ## name(volatile ptype p, atype v) \
202 return ((vtype)atomic_op_acq((p), &, ~(v), sz)); \
204 static __inline vtype \
205 atomic_clear_rel_ ## name(volatile ptype p, atype v) \
207 return ((vtype)atomic_op_rel((p), &, ~(v), sz)); \
210 static __inline int \
211 atomic_cmpset_ ## name(volatile ptype p, vtype e, vtype s) \
213 return (((vtype)atomic_cas((p), (e), (s), sz)) == (e)); \
215 static __inline int \
216 atomic_cmpset_acq_ ## name(volatile ptype p, vtype e, vtype s) \
218 return (((vtype)atomic_cas_acq((p), (e), (s), sz)) == (e)); \
220 static __inline int \
221 atomic_cmpset_rel_ ## name(volatile ptype p, vtype e, vtype s) \
223 return (((vtype)atomic_cas_rel((p), (e), (s), sz)) == (e)); \
226 static __inline int \
227 atomic_fcmpset_ ## name(volatile ptype p, vtype *ep, vtype s) \
231 t = (vtype)atomic_cas((p), (*ep), (s), sz); \
237 static __inline int \
238 atomic_fcmpset_acq_ ## name(volatile ptype p, vtype *ep, vtype s) \
242 t = (vtype)atomic_cas_acq((p), (*ep), (s), sz); \
248 static __inline int \
249 atomic_fcmpset_rel_ ## name(volatile ptype p, vtype *ep, vtype s) \
253 t = (vtype)atomic_cas_rel((p), (*ep), (s), sz); \
260 static __inline vtype \
261 atomic_load_acq_ ## name(volatile ptype p) \
263 return ((vtype)atomic_cas_acq((p), 0, 0, sz)); \
266 static __inline vtype \
267 atomic_readandclear_ ## name(volatile ptype p) \
269 return ((vtype)atomic_ld_clear((p), sz)); \
272 static __inline vtype \
273 atomic_set_ ## name(volatile ptype p, atype v) \
275 return ((vtype)atomic_op((p), |, (v), sz)); \
277 static __inline vtype \
278 atomic_set_acq_ ## name(volatile ptype p, atype v) \
280 return ((vtype)atomic_op_acq((p), |, (v), sz)); \
282 static __inline vtype \
283 atomic_set_rel_ ## name(volatile ptype p, atype v) \
285 return ((vtype)atomic_op_rel((p), |, (v), sz)); \
288 static __inline vtype \
289 atomic_subtract_ ## name(volatile ptype p, atype v) \
291 return ((vtype)atomic_op((p), -, (v), sz)); \
293 static __inline vtype \
294 atomic_subtract_acq_ ## name(volatile ptype p, atype v) \
296 return ((vtype)atomic_op_acq((p), -, (v), sz)); \
298 static __inline vtype \
299 atomic_subtract_rel_ ## name(volatile ptype p, atype v) \
301 return ((vtype)atomic_op_rel((p), -, (v), sz)); \
304 static __inline void \
305 atomic_store_acq_ ## name(volatile ptype p, vtype v) \
307 atomic_st_acq((p), (v), sz); \
309 static __inline void \
310 atomic_store_rel_ ## name(volatile ptype p, vtype v) \
312 atomic_st_rel((p), (v), sz); \
316 atomic_thread_fence_acq(void)
323 atomic_thread_fence_rel(void)
330 atomic_thread_fence_acq_rel(void)
337 atomic_thread_fence_seq_cst(void)
340 membar(LoadLoad | LoadStore | StoreStore | StoreLoad);
344 ATOMIC_GEN(int, u_int *, u_int, u_int, 32);
345 ATOMIC_GEN(32, uint32_t *, uint32_t, uint32_t, 32);
347 ATOMIC_GEN(long, u_long *, u_long, u_long, 64);
348 ATOMIC_GEN(64, uint64_t *, uint64_t, uint64_t, 64);
350 ATOMIC_GEN(ptr, uintptr_t *, uintptr_t, uintptr_t, 64);
352 #define atomic_fetchadd_int atomic_add_int
353 #define atomic_fetchadd_32 atomic_add_32
354 #define atomic_fetchadd_long atomic_add_long
355 #define atomic_fetchadd_64 atomic_add_64
359 #undef atomic_cas_acq
360 #undef atomic_cas_rel
365 #undef atomic_ld_clear
370 #endif /* !_MACHINE_ATOMIC_H_ */