2 * Copyright (c) 1998 Doug Rabson.
3 * Copyright (c) 2001 Jake Burkholder.
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * from: FreeBSD: src/sys/i386/include/atomic.h,v 1.20 2001/02/11
31 #ifndef _MACHINE_ATOMIC_H_
32 #define _MACHINE_ATOMIC_H_
34 #include <machine/cpufunc.h>
36 #define mb() __asm__ __volatile__ ("membar #MemIssue": : :"memory")
40 /* Userland needs different ASI's. */
42 #define __ASI_ATOMIC ASI_N
44 #define __ASI_ATOMIC ASI_P
48 * Various simple arithmetic on memory which is atomic in the presence
49 * of interrupts and multiple processors. See atomic(9) for details.
50 * Note that efficient hardware support exists only for the 32 and 64
51 * bit variants; the 8 and 16 bit versions are not provided and should
52 * not be used in MI code.
54 * This implementation takes advantage of the fact that the sparc64
55 * cas instruction is both a load and a store. The loop is often coded
61 * } while (cas(p, expect, new) != expect);
63 * which performs an unnnecessary load on each iteration that the cas
64 * operation fails. Modified as follows:
69 * result = cas(p, expect, new);
70 * if (result == expect)
75 * the return value of cas is used to avoid the extra reload.
77 * We only include a memory barrier in the rel variants as in total store
78 * order which we use for running the kernel and all of the userland atomic
79 * loads and stores behave as if the were followed by a membar with a mask
80 * of #LoadLoad | #LoadStore | #StoreStore. In order to be also sufficient
81 * for use of relaxed memory ordering, the atomic_cas() in the acq variants
82 * additionally would have to be followed by a membar #LoadLoad | #LoadStore.
83 * Due to the suggested assembly syntax of the membar operands containing a
84 * # character, they cannot be used in macros. The cmask and mmask bits thus
85 * are hard coded in machine/cpufunc.h and used here through macros.
86 * Hopefully the bit numbers won't change in the future.
89 #define itype(sz) uint ## sz ## _t
91 #define atomic_cas_32(p, e, s) casa((p), (e), (s), __ASI_ATOMIC)
92 #define atomic_cas_64(p, e, s) casxa((p), (e), (s), __ASI_ATOMIC)
94 #define atomic_cas(p, e, s, sz) \
95 atomic_cas_ ## sz((p), (e), (s))
97 #define atomic_cas_acq(p, e, s, sz) ({ \
99 v = atomic_cas((p), (e), (s), sz); \
100 __asm __volatile("" : : : "memory"); \
104 #define atomic_cas_rel(p, e, s, sz) ({ \
106 membar(LoadStore | StoreStore); \
107 v = atomic_cas((p), (e), (s), sz); \
111 #define atomic_op(p, op, v, sz) ({ \
113 for (e = *(volatile itype(sz) *)(p);; e = r) { \
115 r = atomic_cas_ ## sz((p), e, s); \
122 #define atomic_op_acq(p, op, v, sz) ({ \
124 t = atomic_op((p), op, (v), sz); \
125 __asm __volatile("" : : : "memory"); \
129 #define atomic_op_rel(p, op, v, sz) ({ \
131 membar(LoadStore | StoreStore); \
132 t = atomic_op((p), op, (v), sz); \
136 #define atomic_load(p, sz) \
137 atomic_cas((p), 0, 0, sz)
139 #define atomic_load_acq(p, sz) ({ \
141 v = atomic_load((p), sz); \
142 __asm __volatile("" : : : "memory"); \
146 #define atomic_load_clear(p, sz) ({ \
148 for (e = *(volatile itype(sz) *)(p);; e = r) { \
149 r = atomic_cas((p), e, 0, sz); \
156 #define atomic_store(p, v, sz) do { \
158 for (e = *(volatile itype(sz) *)(p);; e = r) { \
159 r = atomic_cas((p), e, (v), sz); \
165 #define atomic_store_rel(p, v, sz) do { \
166 membar(LoadStore | StoreStore); \
167 atomic_store((p), (v), sz); \
170 #define ATOMIC_GEN(name, ptype, vtype, atype, sz) \
172 static __inline vtype \
173 atomic_add_ ## name(volatile ptype p, atype v) \
175 return ((vtype)atomic_op((p), +, (v), sz)); \
177 static __inline vtype \
178 atomic_add_acq_ ## name(volatile ptype p, atype v) \
180 return ((vtype)atomic_op_acq((p), +, (v), sz)); \
182 static __inline vtype \
183 atomic_add_rel_ ## name(volatile ptype p, atype v) \
185 return ((vtype)atomic_op_rel((p), +, (v), sz)); \
188 static __inline vtype \
189 atomic_clear_ ## name(volatile ptype p, atype v) \
191 return ((vtype)atomic_op((p), &, ~(v), sz)); \
193 static __inline vtype \
194 atomic_clear_acq_ ## name(volatile ptype p, atype v) \
196 return ((vtype)atomic_op_acq((p), &, ~(v), sz)); \
198 static __inline vtype \
199 atomic_clear_rel_ ## name(volatile ptype p, atype v) \
201 return ((vtype)atomic_op_rel((p), &, ~(v), sz)); \
204 static __inline int \
205 atomic_cmpset_ ## name(volatile ptype p, vtype e, vtype s) \
207 return (((vtype)atomic_cas((p), (e), (s), sz)) == (e)); \
209 static __inline int \
210 atomic_cmpset_acq_ ## name(volatile ptype p, vtype e, vtype s) \
212 return (((vtype)atomic_cas_acq((p), (e), (s), sz)) == (e)); \
214 static __inline int \
215 atomic_cmpset_rel_ ## name(volatile ptype p, vtype e, vtype s) \
217 return (((vtype)atomic_cas_rel((p), (e), (s), sz)) == (e)); \
220 static __inline vtype \
221 atomic_load_ ## name(volatile ptype p) \
223 return ((vtype)atomic_cas((p), 0, 0, sz)); \
225 static __inline vtype \
226 atomic_load_acq_ ## name(volatile ptype p) \
228 return ((vtype)atomic_cas_acq((p), 0, 0, sz)); \
231 static __inline vtype \
232 atomic_readandclear_ ## name(volatile ptype p) \
234 return ((vtype)atomic_load_clear((p), sz)); \
237 static __inline vtype \
238 atomic_set_ ## name(volatile ptype p, atype v) \
240 return ((vtype)atomic_op((p), |, (v), sz)); \
242 static __inline vtype \
243 atomic_set_acq_ ## name(volatile ptype p, atype v) \
245 return ((vtype)atomic_op_acq((p), |, (v), sz)); \
247 static __inline vtype \
248 atomic_set_rel_ ## name(volatile ptype p, atype v) \
250 return ((vtype)atomic_op_rel((p), |, (v), sz)); \
253 static __inline vtype \
254 atomic_subtract_ ## name(volatile ptype p, atype v) \
256 return ((vtype)atomic_op((p), -, (v), sz)); \
258 static __inline vtype \
259 atomic_subtract_acq_ ## name(volatile ptype p, atype v) \
261 return ((vtype)atomic_op_acq((p), -, (v), sz)); \
263 static __inline vtype \
264 atomic_subtract_rel_ ## name(volatile ptype p, atype v) \
266 return ((vtype)atomic_op_rel((p), -, (v), sz)); \
269 static __inline void \
270 atomic_store_ ## name(volatile ptype p, vtype v) \
272 atomic_store((p), (v), sz); \
274 static __inline void \
275 atomic_store_rel_ ## name(volatile ptype p, vtype v) \
277 atomic_store_rel((p), (v), sz); \
280 ATOMIC_GEN(int, u_int *, u_int, u_int, 32);
281 ATOMIC_GEN(32, uint32_t *, uint32_t, uint32_t, 32);
283 ATOMIC_GEN(long, u_long *, u_long, u_long, 64);
284 ATOMIC_GEN(64, uint64_t *, uint64_t, uint64_t, 64);
286 ATOMIC_GEN(ptr, uintptr_t *, uintptr_t, uintptr_t, 64);
288 #define atomic_fetchadd_int atomic_add_int
289 #define atomic_fetchadd_32 atomic_add_32
290 #define atomic_fetchadd_long atomic_add_long
294 #undef atomic_cas_acq
295 #undef atomic_cas_rel
299 #undef atomic_load_acq
300 #undef atomic_store_rel
301 #undef atomic_load_clear
303 #endif /* !_MACHINE_ATOMIC_H_ */