2 * Copyright (c) 1998 Doug Rabson.
3 * Copyright (c) 2001 Jake Burkholder.
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * from: FreeBSD: src/sys/i386/include/atomic.h,v 1.20 2001/02/11
31 #ifndef _MACHINE_ATOMIC_H_
32 #define _MACHINE_ATOMIC_H_
34 #include <machine/cpufunc.h>
36 #define mb() __asm__ __volatile__ ("membar #MemIssue": : :"memory")
40 /* Userland needs different ASI's. */
42 #define __ASI_ATOMIC ASI_N
44 #define __ASI_ATOMIC ASI_P
48 * Various simple arithmetic on memory which is atomic in the presence
49 * of interrupts and multiple processors. See atomic(9) for details.
50 * Note that efficient hardware support exists only for the 32 and 64
51 * bit variants; the 8 and 16 bit versions are not provided and should
52 * not be used in MI code.
54 * This implementation takes advantage of the fact that the sparc64
55 * cas instruction is both a load and a store. The loop is often coded
61 * } while (cas(p, expect, new) != expect);
63 * which performs an unnnecessary load on each iteration that the cas
64 * operation fails. Modified as follows:
69 * result = cas(p, expect, new);
70 * if (result == expect)
75 * the return value of cas is used to avoid the extra reload.
77 * The memory barriers provided by the acq and rel variants are intended
78 * to be sufficient for use of relaxed memory ordering. Due to the
79 * suggested assembly syntax of the membar operands containing a #
80 * character, they cannot be used in macros. The cmask and mmask bits
81 * are hard coded in machine/cpufunc.h and used here through macros.
82 * Hopefully sun will choose not to change the bit numbers.
85 #define itype(sz) uint ## sz ## _t
87 #define atomic_cas_32(p, e, s) casa(p, e, s, __ASI_ATOMIC)
88 #define atomic_cas_64(p, e, s) casxa(p, e, s, __ASI_ATOMIC)
90 #define atomic_cas(p, e, s, sz) \
91 atomic_cas_ ## sz(p, e, s)
93 #define atomic_cas_acq(p, e, s, sz) ({ \
95 v = atomic_cas(p, e, s, sz); \
96 membar(LoadLoad | LoadStore); \
100 #define atomic_cas_rel(p, e, s, sz) ({ \
102 membar(LoadStore | StoreStore); \
103 v = atomic_cas(p, e, s, sz); \
107 #define atomic_op(p, op, v, sz) ({ \
109 for (e = *(volatile itype(sz) *)p;; e = r) { \
111 r = atomic_cas_ ## sz(p, e, s); \
118 #define atomic_op_acq(p, op, v, sz) ({ \
120 t = atomic_op(p, op, v, sz); \
121 membar(LoadLoad | LoadStore); \
125 #define atomic_op_rel(p, op, v, sz) ({ \
127 membar(LoadStore | StoreStore); \
128 t = atomic_op(p, op, v, sz); \
132 #define atomic_load(p, sz) \
133 atomic_cas(p, 0, 0, sz)
135 #define atomic_load_acq(p, sz) ({ \
137 v = atomic_load(p, sz); \
138 membar(LoadLoad | LoadStore); \
142 #define atomic_load_clear(p, sz) ({ \
144 for (e = *(volatile itype(sz) *)p;; e = r) { \
145 r = atomic_cas(p, e, 0, sz); \
152 #define atomic_store(p, v, sz) do { \
154 for (e = *(volatile itype(sz) *)p;; e = r) { \
155 r = atomic_cas(p, e, v, sz); \
161 #define atomic_store_rel(p, v, sz) do { \
162 membar(LoadStore | StoreStore); \
163 atomic_store(p, v, sz); \
166 #define ATOMIC_GEN(name, ptype, vtype, atype, sz) \
168 static __inline vtype \
169 atomic_add_ ## name(volatile ptype p, atype v) \
171 return ((vtype)atomic_op(p, +, v, sz)); \
173 static __inline vtype \
174 atomic_add_acq_ ## name(volatile ptype p, atype v) \
176 return ((vtype)atomic_op_acq(p, +, v, sz)); \
178 static __inline vtype \
179 atomic_add_rel_ ## name(volatile ptype p, atype v) \
181 return ((vtype)atomic_op_rel(p, +, v, sz)); \
184 static __inline vtype \
185 atomic_clear_ ## name(volatile ptype p, atype v) \
187 return ((vtype)atomic_op(p, &, ~v, sz)); \
189 static __inline vtype \
190 atomic_clear_acq_ ## name(volatile ptype p, atype v) \
192 return ((vtype)atomic_op_acq(p, &, ~v, sz)); \
194 static __inline vtype \
195 atomic_clear_rel_ ## name(volatile ptype p, atype v) \
197 return ((vtype)atomic_op_rel(p, &, ~v, sz)); \
200 static __inline int \
201 atomic_cmpset_ ## name(volatile ptype p, vtype e, vtype s) \
203 return (((vtype)atomic_cas(p, e, s, sz)) == e); \
205 static __inline int \
206 atomic_cmpset_acq_ ## name(volatile ptype p, vtype e, vtype s) \
208 return (((vtype)atomic_cas_acq(p, e, s, sz)) == e); \
210 static __inline int \
211 atomic_cmpset_rel_ ## name(volatile ptype p, vtype e, vtype s) \
213 return (((vtype)atomic_cas_rel(p, e, s, sz)) == e); \
216 static __inline vtype \
217 atomic_load_ ## name(volatile ptype p) \
219 return ((vtype)atomic_cas(p, 0, 0, sz)); \
221 static __inline vtype \
222 atomic_load_acq_ ## name(volatile ptype p) \
224 return ((vtype)atomic_cas_acq(p, 0, 0, sz)); \
227 static __inline vtype \
228 atomic_readandclear_ ## name(volatile ptype p) \
230 return ((vtype)atomic_load_clear(p, sz)); \
233 static __inline vtype \
234 atomic_set_ ## name(volatile ptype p, atype v) \
236 return ((vtype)atomic_op(p, |, v, sz)); \
238 static __inline vtype \
239 atomic_set_acq_ ## name(volatile ptype p, atype v) \
241 return ((vtype)atomic_op_acq(p, |, v, sz)); \
243 static __inline vtype \
244 atomic_set_rel_ ## name(volatile ptype p, atype v) \
246 return ((vtype)atomic_op_rel(p, |, v, sz)); \
249 static __inline vtype \
250 atomic_subtract_ ## name(volatile ptype p, atype v) \
252 return ((vtype)atomic_op(p, -, v, sz)); \
254 static __inline vtype \
255 atomic_subtract_acq_ ## name(volatile ptype p, atype v) \
257 return ((vtype)atomic_op_acq(p, -, v, sz)); \
259 static __inline vtype \
260 atomic_subtract_rel_ ## name(volatile ptype p, atype v) \
262 return ((vtype)atomic_op_rel(p, -, v, sz)); \
265 static __inline void \
266 atomic_store_ ## name(volatile ptype p, vtype v) \
268 atomic_store(p, v, sz); \
270 static __inline void \
271 atomic_store_rel_ ## name(volatile ptype p, vtype v) \
273 atomic_store_rel(p, v, sz); \
276 ATOMIC_GEN(int, u_int *, u_int, u_int, 32);
277 ATOMIC_GEN(32, uint32_t *, uint32_t, uint32_t, 32);
279 ATOMIC_GEN(long, u_long *, u_long, u_long, 64);
280 ATOMIC_GEN(64, uint64_t *, uint64_t, uint64_t, 64);
282 ATOMIC_GEN(ptr, uintptr_t *, uintptr_t, uintptr_t, 64);
284 #define atomic_fetchadd_int atomic_add_int
285 #define atomic_fetchadd_32 atomic_add_32
286 #define atomic_fetchadd_long atomic_add_long
290 #undef atomic_cas_acq
291 #undef atomic_cas_rel
295 #undef atomic_load_acq
296 #undef atomic_store_rel
297 #undef atomic_load_clear
299 #endif /* !_MACHINE_ATOMIC_H_ */