2 * Copyright (c) 1998 Doug Rabson.
3 * Copyright (c) 2001 Jake Burkholder.
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * from: FreeBSD: src/sys/i386/include/atomic.h,v 1.20 2001/02/11
31 #ifndef _MACHINE_ATOMIC_H_
32 #define _MACHINE_ATOMIC_H_
34 #include <machine/cpufunc.h>
36 /* Userland needs different ASI's. */
38 #define __ASI_ATOMIC ASI_N
40 #define __ASI_ATOMIC ASI_P
44 * Various simple arithmetic on memory which is atomic in the presence
45 * of interrupts and multiple processors. See atomic(9) for details.
46 * Note that efficient hardware support exists only for the 32 and 64
47 * bit variants; the 8 and 16 bit versions are not provided and should
48 * not be used in MI code.
50 * This implementation takes advantage of the fact that the sparc64
51 * cas instruction is both a load and a store. The loop is often coded
57 * } while (cas(p, expect, new) != expect);
59 * which performs an unnnecessary load on each iteration that the cas
60 * operation fails. Modified as follows:
65 * result = cas(p, expect, new);
66 * if (result == expect)
71 * the return value of cas is used to avoid the extra reload.
73 * The memory barriers provided by the acq and rel variants are intended
74 * to be sufficient for use of relaxed memory ordering. Due to the
75 * suggested assembly syntax of the membar operands containing a #
76 * character, they cannot be used in macros. The cmask and mmask bits
77 * are hard coded in machine/cpufunc.h and used here through macros.
78 * Hopefully sun will choose not to change the bit numbers.
81 #define itype(sz) uint ## sz ## _t
83 #define atomic_cas_32(p, e, s) casa(p, e, s, __ASI_ATOMIC)
84 #define atomic_cas_64(p, e, s) casxa(p, e, s, __ASI_ATOMIC)
86 #define atomic_cas(p, e, s, sz) \
87 atomic_cas_ ## sz(p, e, s)
89 #define atomic_cas_acq(p, e, s, sz) ({ \
91 v = atomic_cas(p, e, s, sz); \
92 membar(LoadLoad | LoadStore); \
96 #define atomic_cas_rel(p, e, s, sz) ({ \
98 membar(LoadStore | StoreStore); \
99 v = atomic_cas(p, e, s, sz); \
103 #define atomic_op(p, op, v, sz) ({ \
105 for (e = *(volatile itype(sz) *)p;; e = r) { \
107 r = atomic_cas_ ## sz(p, e, s); \
114 #define atomic_op_acq(p, op, v, sz) ({ \
116 t = atomic_op(p, op, v, sz); \
117 membar(LoadLoad | LoadStore); \
121 #define atomic_op_rel(p, op, v, sz) ({ \
123 membar(LoadStore | StoreStore); \
124 t = atomic_op(p, op, v, sz); \
128 #define atomic_load(p, sz) \
129 atomic_cas(p, 0, 0, sz)
131 #define atomic_load_acq(p, sz) ({ \
133 v = atomic_load(p, sz); \
134 membar(LoadLoad | LoadStore); \
138 #define atomic_load_clear(p, sz) ({ \
140 for (e = *(volatile itype(sz) *)p;; e = r) { \
141 r = atomic_cas(p, e, 0, sz); \
148 #define atomic_store(p, v, sz) do { \
150 for (e = *(volatile itype(sz) *)p;; e = r) { \
151 r = atomic_cas(p, e, v, sz); \
157 #define atomic_store_rel(p, v, sz) do { \
158 membar(LoadStore | StoreStore); \
159 atomic_store(p, v, sz); \
162 #define ATOMIC_GEN(name, ptype, vtype, atype, sz) \
164 static __inline vtype \
165 atomic_add_ ## name(volatile ptype p, atype v) \
167 return ((vtype)atomic_op(p, +, v, sz)); \
169 static __inline vtype \
170 atomic_add_acq_ ## name(volatile ptype p, atype v) \
172 return ((vtype)atomic_op_acq(p, +, v, sz)); \
174 static __inline vtype \
175 atomic_add_rel_ ## name(volatile ptype p, atype v) \
177 return ((vtype)atomic_op_rel(p, +, v, sz)); \
180 static __inline vtype \
181 atomic_clear_ ## name(volatile ptype p, atype v) \
183 return ((vtype)atomic_op(p, &, ~v, sz)); \
185 static __inline vtype \
186 atomic_clear_acq_ ## name(volatile ptype p, atype v) \
188 return ((vtype)atomic_op_acq(p, &, ~v, sz)); \
190 static __inline vtype \
191 atomic_clear_rel_ ## name(volatile ptype p, atype v) \
193 return ((vtype)atomic_op_rel(p, &, ~v, sz)); \
196 static __inline int \
197 atomic_cmpset_ ## name(volatile ptype p, vtype e, vtype s) \
199 return (((vtype)atomic_cas(p, e, s, sz)) == e); \
201 static __inline int \
202 atomic_cmpset_acq_ ## name(volatile ptype p, vtype e, vtype s) \
204 return (((vtype)atomic_cas_acq(p, e, s, sz)) == e); \
206 static __inline int \
207 atomic_cmpset_rel_ ## name(volatile ptype p, vtype e, vtype s) \
209 return (((vtype)atomic_cas_rel(p, e, s, sz)) == e); \
212 static __inline vtype \
213 atomic_load_ ## name(volatile ptype p) \
215 return ((vtype)atomic_cas(p, 0, 0, sz)); \
217 static __inline vtype \
218 atomic_load_acq_ ## name(volatile ptype p) \
220 return ((vtype)atomic_cas_acq(p, 0, 0, sz)); \
223 static __inline vtype \
224 atomic_readandclear_ ## name(volatile ptype p) \
226 return ((vtype)atomic_load_clear(p, sz)); \
229 static __inline vtype \
230 atomic_set_ ## name(volatile ptype p, atype v) \
232 return ((vtype)atomic_op(p, |, v, sz)); \
234 static __inline vtype \
235 atomic_set_acq_ ## name(volatile ptype p, atype v) \
237 return ((vtype)atomic_op_acq(p, |, v, sz)); \
239 static __inline vtype \
240 atomic_set_rel_ ## name(volatile ptype p, atype v) \
242 return ((vtype)atomic_op_rel(p, |, v, sz)); \
245 static __inline vtype \
246 atomic_subtract_ ## name(volatile ptype p, atype v) \
248 return ((vtype)atomic_op(p, -, v, sz)); \
250 static __inline vtype \
251 atomic_subtract_acq_ ## name(volatile ptype p, atype v) \
253 return ((vtype)atomic_op_acq(p, -, v, sz)); \
255 static __inline vtype \
256 atomic_subtract_rel_ ## name(volatile ptype p, atype v) \
258 return ((vtype)atomic_op_rel(p, -, v, sz)); \
261 static __inline void \
262 atomic_store_ ## name(volatile ptype p, vtype v) \
264 atomic_store(p, v, sz); \
266 static __inline void \
267 atomic_store_rel_ ## name(volatile ptype p, vtype v) \
269 atomic_store_rel(p, v, sz); \
272 ATOMIC_GEN(int, u_int *, u_int, u_int, 32);
273 ATOMIC_GEN(32, uint32_t *, uint32_t, uint32_t, 32);
275 ATOMIC_GEN(long, u_long *, u_long, u_long, 64);
276 ATOMIC_GEN(64, uint64_t *, uint64_t, uint64_t, 64);
278 ATOMIC_GEN(ptr, uintptr_t *, uintptr_t, uintptr_t, 64);
280 #define atomic_fetchadd_int atomic_add_int
281 #define atomic_fetchadd_32 atomic_add_32
282 #define atomic_fetchadd_long atomic_add_long
286 #undef atomic_cas_acq
287 #undef atomic_cas_rel
291 #undef atomic_load_acq
292 #undef atomic_store_rel
293 #undef atomic_load_clear
295 #endif /* !_MACHINE_ATOMIC_H_ */