2 * Copyright (c) 2008 Marcel Moolenaar
3 * Copyright (c) 2001 Benno Rice
4 * Copyright (c) 2001 David E. O'Brien
5 * Copyright (c) 1998 Doug Rabson
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 #ifndef _MACHINE_ATOMIC_H_
33 #define _MACHINE_ATOMIC_H_
36 #error this file needs sys/cdefs.h as a prerequisite
39 #define __ATOMIC_BARRIER \
40 __asm __volatile("sync" : : : "memory")
42 #define mb() __ATOMIC_BARRIER
51 #define __atomic_add_int(p, v, t) \
53 "1: lwarx %0, 0, %2\n" \
55 " stwcx. %0, 0, %2\n" \
57 : "=&r" (t), "=m" (*p) \
58 : "r" (p), "r" (v), "m" (*p) \
60 /* __atomic_add_int */
63 #define __atomic_add_long(p, v, t) \
65 "1: ldarx %0, 0, %2\n" \
67 " stdcx. %0, 0, %2\n" \
69 : "=&r" (t), "=m" (*p) \
70 : "r" (p), "r" (v), "m" (*p) \
72 /* __atomic_add_long */
74 #define __atomic_add_long(p, v, t) \
76 "1: lwarx %0, 0, %2\n" \
78 " stwcx. %0, 0, %2\n" \
80 : "=&r" (t), "=m" (*p) \
81 : "r" (p), "r" (v), "m" (*p) \
83 /* __atomic_add_long */
86 #define _ATOMIC_ADD(type) \
87 static __inline void \
88 atomic_add_##type(volatile u_##type *p, u_##type v) { \
90 __atomic_add_##type(p, v, t); \
93 static __inline void \
94 atomic_add_acq_##type(volatile u_##type *p, u_##type v) { \
96 __atomic_add_##type(p, v, t); \
100 static __inline void \
101 atomic_add_rel_##type(volatile u_##type *p, u_##type v) { \
104 __atomic_add_##type(p, v, t); \
111 #define atomic_add_32 atomic_add_int
112 #define atomic_add_acq_32 atomic_add_acq_int
113 #define atomic_add_rel_32 atomic_add_rel_int
116 #define atomic_add_64 atomic_add_long
117 #define atomic_add_acq_64 atomic_add_acq_long
118 #define atomic_add_rel_64 atomic_add_rel_long
120 #define atomic_add_ptr atomic_add_long
121 #define atomic_add_acq_ptr atomic_add_acq_long
122 #define atomic_add_rel_ptr atomic_add_rel_long
124 #define atomic_add_ptr atomic_add_int
125 #define atomic_add_acq_ptr atomic_add_acq_int
126 #define atomic_add_rel_ptr atomic_add_rel_int
129 #undef __atomic_add_long
130 #undef __atomic_add_int
137 #define __atomic_clear_int(p, v, t) \
139 "1: lwarx %0, 0, %2\n" \
140 " andc %0, %0, %3\n" \
141 " stwcx. %0, 0, %2\n" \
143 : "=&r" (t), "=m" (*p) \
144 : "r" (p), "r" (v), "m" (*p) \
146 /* __atomic_clear_int */
149 #define __atomic_clear_long(p, v, t) \
151 "1: ldarx %0, 0, %2\n" \
152 " andc %0, %0, %3\n" \
153 " stdcx. %0, 0, %2\n" \
155 : "=&r" (t), "=m" (*p) \
156 : "r" (p), "r" (v), "m" (*p) \
158 /* __atomic_clear_long */
160 #define __atomic_clear_long(p, v, t) \
162 "1: lwarx %0, 0, %2\n" \
163 " andc %0, %0, %3\n" \
164 " stwcx. %0, 0, %2\n" \
166 : "=&r" (t), "=m" (*p) \
167 : "r" (p), "r" (v), "m" (*p) \
169 /* __atomic_clear_long */
172 #define _ATOMIC_CLEAR(type) \
173 static __inline void \
174 atomic_clear_##type(volatile u_##type *p, u_##type v) { \
176 __atomic_clear_##type(p, v, t); \
179 static __inline void \
180 atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \
182 __atomic_clear_##type(p, v, t); \
186 static __inline void \
187 atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \
190 __atomic_clear_##type(p, v, t); \
198 #define atomic_clear_32 atomic_clear_int
199 #define atomic_clear_acq_32 atomic_clear_acq_int
200 #define atomic_clear_rel_32 atomic_clear_rel_int
203 #define atomic_clear_64 atomic_clear_long
204 #define atomic_clear_acq_64 atomic_clear_acq_long
205 #define atomic_clear_rel_64 atomic_clear_rel_long
207 #define atomic_clear_ptr atomic_clear_long
208 #define atomic_clear_acq_ptr atomic_clear_acq_long
209 #define atomic_clear_rel_ptr atomic_clear_rel_long
211 #define atomic_clear_ptr atomic_clear_int
212 #define atomic_clear_acq_ptr atomic_clear_acq_int
213 #define atomic_clear_rel_ptr atomic_clear_rel_int
216 #undef __atomic_clear_long
217 #undef __atomic_clear_int
220 * atomic_cmpset(p, o, n)
222 /* TODO -- see below */
227 /* TODO -- see below */
230 * atomic_readandclear(p)
232 /* TODO -- see below */
239 #define __atomic_set_int(p, v, t) \
241 "1: lwarx %0, 0, %2\n" \
243 " stwcx. %0, 0, %2\n" \
245 : "=&r" (t), "=m" (*p) \
246 : "r" (p), "r" (v), "m" (*p) \
248 /* __atomic_set_int */
251 #define __atomic_set_long(p, v, t) \
253 "1: ldarx %0, 0, %2\n" \
255 " stdcx. %0, 0, %2\n" \
257 : "=&r" (t), "=m" (*p) \
258 : "r" (p), "r" (v), "m" (*p) \
260 /* __atomic_set_long */
262 #define __atomic_set_long(p, v, t) \
264 "1: lwarx %0, 0, %2\n" \
266 " stwcx. %0, 0, %2\n" \
268 : "=&r" (t), "=m" (*p) \
269 : "r" (p), "r" (v), "m" (*p) \
271 /* __atomic_set_long */
274 #define _ATOMIC_SET(type) \
275 static __inline void \
276 atomic_set_##type(volatile u_##type *p, u_##type v) { \
278 __atomic_set_##type(p, v, t); \
281 static __inline void \
282 atomic_set_acq_##type(volatile u_##type *p, u_##type v) { \
284 __atomic_set_##type(p, v, t); \
288 static __inline void \
289 atomic_set_rel_##type(volatile u_##type *p, u_##type v) { \
292 __atomic_set_##type(p, v, t); \
299 #define atomic_set_32 atomic_set_int
300 #define atomic_set_acq_32 atomic_set_acq_int
301 #define atomic_set_rel_32 atomic_set_rel_int
304 #define atomic_set_64 atomic_set_long
305 #define atomic_set_acq_64 atomic_set_acq_long
306 #define atomic_set_rel_64 atomic_set_rel_long
308 #define atomic_set_ptr atomic_set_long
309 #define atomic_set_acq_ptr atomic_set_acq_long
310 #define atomic_set_rel_ptr atomic_set_rel_long
312 #define atomic_set_ptr atomic_set_int
313 #define atomic_set_acq_ptr atomic_set_acq_int
314 #define atomic_set_rel_ptr atomic_set_rel_int
317 #undef __atomic_set_long
318 #undef __atomic_set_int
321 * atomic_subtract(p, v)
325 #define __atomic_subtract_int(p, v, t) \
327 "1: lwarx %0, 0, %2\n" \
328 " subf %0, %3, %0\n" \
329 " stwcx. %0, 0, %2\n" \
331 : "=&r" (t), "=m" (*p) \
332 : "r" (p), "r" (v), "m" (*p) \
334 /* __atomic_subtract_int */
337 #define __atomic_subtract_long(p, v, t) \
339 "1: ldarx %0, 0, %2\n" \
340 " subf %0, %3, %0\n" \
341 " stdcx. %0, 0, %2\n" \
343 : "=&r" (t), "=m" (*p) \
344 : "r" (p), "r" (v), "m" (*p) \
346 /* __atomic_subtract_long */
348 #define __atomic_subtract_long(p, v, t) \
350 "1: lwarx %0, 0, %2\n" \
351 " subf %0, %3, %0\n" \
352 " stwcx. %0, 0, %2\n" \
354 : "=&r" (t), "=m" (*p) \
355 : "r" (p), "r" (v), "m" (*p) \
357 /* __atomic_subtract_long */
360 #define _ATOMIC_SUBTRACT(type) \
361 static __inline void \
362 atomic_subtract_##type(volatile u_##type *p, u_##type v) { \
364 __atomic_subtract_##type(p, v, t); \
367 static __inline void \
368 atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) { \
370 __atomic_subtract_##type(p, v, t); \
374 static __inline void \
375 atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) { \
378 __atomic_subtract_##type(p, v, t); \
380 /* _ATOMIC_SUBTRACT */
382 _ATOMIC_SUBTRACT(int)
383 _ATOMIC_SUBTRACT(long)
385 #define atomic_subtract_32 atomic_subtract_int
386 #define atomic_subtract_acq_32 atomic_subtract_acq_int
387 #define atomic_subtract_rel_32 atomic_subtract_rel_int
390 #define atomic_subtract_64 atomic_subtract_long
391 #define atomic_subtract_acq_64 atomic_subract_acq_long
392 #define atomic_subtract_rel_64 atomic_subtract_rel_long
394 #define atomic_subtract_ptr atomic_subtract_long
395 #define atomic_subtract_acq_ptr atomic_subtract_acq_long
396 #define atomic_subtract_rel_ptr atomic_subtract_rel_long
398 #define atomic_subtract_ptr atomic_subtract_int
399 #define atomic_subtract_acq_ptr atomic_subtract_acq_int
400 #define atomic_subtract_rel_ptr atomic_subtract_rel_int
402 #undef _ATOMIC_SUBTRACT
403 #undef __atomic_subtract_long
404 #undef __atomic_subtract_int
407 * atomic_store_rel(p, v)
409 /* TODO -- see below */
412 * Old/original implementations that still need revisiting.
415 static __inline u_int
416 atomic_readandclear_int(volatile u_int *addr)
420 #ifdef __GNUCLIKE_ASM
422 "\tsync\n" /* drain writes */
423 "1:\tlwarx %0, 0, %3\n\t" /* load old value */
424 "li %1, 0\n\t" /* load new value */
425 "stwcx. %1, 0, %3\n\t" /* attempt to store */
426 "bne- 1b\n\t" /* spin if failed */
427 : "=&r"(result), "=&r"(temp), "=m" (*addr)
428 : "r" (addr), "m" (*addr)
436 static __inline u_long
437 atomic_readandclear_long(volatile u_long *addr)
441 #ifdef __GNUCLIKE_ASM
443 "\tsync\n" /* drain writes */
444 "1:\tldarx %0, 0, %3\n\t" /* load old value */
445 "li %1, 0\n\t" /* load new value */
446 "stdcx. %1, 0, %3\n\t" /* attempt to store */
447 "bne- 1b\n\t" /* spin if failed */
448 : "=&r"(result), "=&r"(temp), "=m" (*addr)
449 : "r" (addr), "m" (*addr)
457 #define atomic_readandclear_32 atomic_readandclear_int
460 #define atomic_readandclear_64 atomic_readandclear_long
462 #define atomic_readandclear_ptr atomic_readandclear_long
464 static __inline u_long
465 atomic_readandclear_long(volatile u_long *addr)
468 return ((u_long)atomic_readandclear_int((volatile u_int *)addr));
471 #define atomic_readandclear_ptr atomic_readandclear_int
475 * We assume that a = b will do atomic loads and stores.
477 #define ATOMIC_STORE_LOAD(TYPE) \
478 static __inline u_##TYPE \
479 atomic_load_acq_##TYPE(volatile u_##TYPE *p) \
488 static __inline void \
489 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \
495 ATOMIC_STORE_LOAD(int)
497 #define atomic_load_acq_32 atomic_load_acq_int
498 #define atomic_store_rel_32 atomic_store_rel_int
501 ATOMIC_STORE_LOAD(long)
503 #define atomic_load_acq_64 atomic_load_acq_long
504 #define atomic_store_rel_64 atomic_store_rel_long
506 #define atomic_load_acq_ptr atomic_load_acq_long
507 #define atomic_store_rel_ptr atomic_store_rel_long
509 static __inline u_long
510 atomic_load_acq_long(volatile u_long *addr)
513 return ((u_long)atomic_load_acq_int((volatile u_int *)addr));
517 atomic_store_rel_long(volatile u_long *addr, u_long val)
520 atomic_store_rel_int((volatile u_int *)addr, (u_int)val);
523 #define atomic_load_acq_ptr atomic_load_acq_int
524 #define atomic_store_rel_ptr atomic_store_rel_int
526 #undef ATOMIC_STORE_LOAD
529 * Atomically compare the value stored at *p with cmpval and if the
530 * two values are equal, update the value of *p with newval. Returns
531 * zero if the compare failed, nonzero otherwise.
534 atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval)
538 #ifdef __GNUCLIKE_ASM
540 "1:\tlwarx %0, 0, %2\n\t" /* load old value */
541 "cmplw %3, %0\n\t" /* compare */
542 "bne 2f\n\t" /* exit if not equal */
543 "stwcx. %4, 0, %2\n\t" /* attempt to store */
544 "bne- 1b\n\t" /* spin if failed */
545 "li %0, 1\n\t" /* success - retval = 1 */
546 "b 3f\n\t" /* we've succeeded */
548 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
549 "li %0, 0\n\t" /* failure - retval = 0 */
551 : "=&r" (ret), "=m" (*p)
552 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
559 atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval)
563 #ifdef __GNUCLIKE_ASM
566 "1:\tldarx %0, 0, %2\n\t" /* load old value */
567 "cmpld %3, %0\n\t" /* compare */
568 "bne 2f\n\t" /* exit if not equal */
569 "stdcx. %4, 0, %2\n\t" /* attempt to store */
571 "1:\tlwarx %0, 0, %2\n\t" /* load old value */
572 "cmplw %3, %0\n\t" /* compare */
573 "bne 2f\n\t" /* exit if not equal */
574 "stwcx. %4, 0, %2\n\t" /* attempt to store */
576 "bne- 1b\n\t" /* spin if failed */
577 "li %0, 1\n\t" /* success - retval = 1 */
578 "b 3f\n\t" /* we've succeeded */
581 "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
583 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
585 "li %0, 0\n\t" /* failure - retval = 0 */
587 : "=&r" (ret), "=m" (*p)
588 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
596 atomic_cmpset_acq_int(volatile u_int *p, u_int cmpval, u_int newval)
600 retval = atomic_cmpset_int(p, cmpval, newval);
606 atomic_cmpset_rel_int(volatile u_int *p, u_int cmpval, u_int newval)
609 return (atomic_cmpset_int(p, cmpval, newval));
613 atomic_cmpset_acq_long(volatile u_long *p, u_long cmpval, u_long newval)
617 retval = atomic_cmpset_long(p, cmpval, newval);
623 atomic_cmpset_rel_long(volatile u_long *p, u_long cmpval, u_long newval)
626 return (atomic_cmpset_long(p, cmpval, newval));
629 #define atomic_cmpset_32 atomic_cmpset_int
630 #define atomic_cmpset_acq_32 atomic_cmpset_acq_int
631 #define atomic_cmpset_rel_32 atomic_cmpset_rel_int
634 #define atomic_cmpset_64 atomic_cmpset_long
635 #define atomic_cmpset_acq_64 atomic_cmpset_acq_long
636 #define atomic_cmpset_rel_64 atomic_cmpset_rel_long
638 #define atomic_cmpset_ptr atomic_cmpset_long
639 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long
640 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long
642 #define atomic_cmpset_ptr atomic_cmpset_int
643 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_int
644 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_int
647 static __inline u_int
648 atomic_fetchadd_int(volatile u_int *p, u_int v)
654 } while (!atomic_cmpset_int(p, value, value + v));
658 static __inline u_long
659 atomic_fetchadd_long(volatile u_long *p, u_long v)
665 } while (!atomic_cmpset_long(p, value, value + v));
669 #define atomic_fetchadd_32 atomic_fetchadd_int
672 #define atomic_fetchadd_64 atomic_fetchadd_long
675 #endif /* ! _MACHINE_ATOMIC_H_ */