2 * Copyright (c) 1998 Doug Rabson
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 #ifndef _MACHINE_ATOMIC_H_
29 #define _MACHINE_ATOMIC_H_
32 #error this file needs sys/cdefs.h as a prerequisite
35 #define mb() __asm __volatile("mfence;" : : : "memory")
36 #define wmb() __asm __volatile("sfence;" : : : "memory")
37 #define rmb() __asm __volatile("lfence;" : : : "memory")
40 * Various simple operations on memory, each of which is atomic in the
41 * presence of interrupts and multiple processors.
43 * atomic_set_char(P, V) (*(u_char *)(P) |= (V))
44 * atomic_clear_char(P, V) (*(u_char *)(P) &= ~(V))
45 * atomic_add_char(P, V) (*(u_char *)(P) += (V))
46 * atomic_subtract_char(P, V) (*(u_char *)(P) -= (V))
48 * atomic_set_short(P, V) (*(u_short *)(P) |= (V))
49 * atomic_clear_short(P, V) (*(u_short *)(P) &= ~(V))
50 * atomic_add_short(P, V) (*(u_short *)(P) += (V))
51 * atomic_subtract_short(P, V) (*(u_short *)(P) -= (V))
53 * atomic_set_int(P, V) (*(u_int *)(P) |= (V))
54 * atomic_clear_int(P, V) (*(u_int *)(P) &= ~(V))
55 * atomic_add_int(P, V) (*(u_int *)(P) += (V))
56 * atomic_subtract_int(P, V) (*(u_int *)(P) -= (V))
57 * atomic_swap_int(P, V) (return (*(u_int *)(P)); *(u_int *)(P) = (V);)
58 * atomic_readandclear_int(P) (return (*(u_int *)(P)); *(u_int *)(P) = 0;)
60 * atomic_set_long(P, V) (*(u_long *)(P) |= (V))
61 * atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V))
62 * atomic_add_long(P, V) (*(u_long *)(P) += (V))
63 * atomic_subtract_long(P, V) (*(u_long *)(P) -= (V))
64 * atomic_swap_long(P, V) (return (*(u_long *)(P)); *(u_long *)(P) = (V);)
65 * atomic_readandclear_long(P) (return (*(u_long *)(P)); *(u_long *)(P) = 0;)
69 * The above functions are expanded inline in the statically-linked
70 * kernel. Lock prefixes are generated if an SMP kernel is being
73 * Kernel modules call real functions which are built into the kernel.
74 * This allows kernel modules to be portable between UP and SMP systems.
76 #if defined(KLD_MODULE) || !defined(__GNUCLIKE_ASM)
77 #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \
78 void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); \
79 void atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)
81 int atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src);
82 int atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src);
83 u_int atomic_fetchadd_int(volatile u_int *p, u_int v);
84 u_long atomic_fetchadd_long(volatile u_long *p, u_long v);
85 int atomic_testandset_int(volatile u_int *p, u_int v);
86 int atomic_testandset_long(volatile u_long *p, u_int v);
88 #define ATOMIC_LOAD(TYPE, LOP) \
89 u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p)
90 #define ATOMIC_STORE(TYPE) \
91 void atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)
93 #else /* !KLD_MODULE && __GNUCLIKE_ASM */
96 * For userland, always use lock prefixes so that the binaries will run
97 * on both SMP and !SMP systems.
99 #if defined(SMP) || !defined(_KERNEL)
100 #define MPLOCKED "lock ; "
106 * The assembly is volatilized to avoid code chunk removal by the compiler.
107 * GCC aggressively reorders operations and memory clobbering is necessary
108 * in order to avoid that for memory barriers.
110 #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \
111 static __inline void \
112 atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\
114 __asm __volatile(MPLOCKED OP \
116 : CONS (V), "m" (*p) \
120 static __inline void \
121 atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\
123 __asm __volatile(MPLOCKED OP \
125 : CONS (V), "m" (*p) \
131 * Atomic compare and set, used by the mutex functions
133 * if (*dst == expect) *dst = src (all 32 bit words)
135 * Returns 0 on failure, non-zero on success
139 atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src)
148 "# atomic_cmpset_int"
149 : "=a" (res), /* 0 */
152 "a" (expect), /* 3 */
160 atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src)
169 "# atomic_cmpset_long"
170 : "=a" (res), /* 0 */
173 "a" (expect), /* 3 */
181 * Atomically add the value of v to the integer pointed to by p and return
182 * the previous value of *p.
184 static __inline u_int
185 atomic_fetchadd_int(volatile u_int *p, u_int v)
191 "# atomic_fetchadd_int"
192 : "+r" (v), /* 0 (result) */
200 * Atomically add the value of v to the long integer pointed to by p and return
201 * the previous value of *p.
203 static __inline u_long
204 atomic_fetchadd_long(volatile u_long *p, u_long v)
210 "# atomic_fetchadd_long"
211 : "+r" (v), /* 0 (result) */
219 atomic_testandset_int(volatile u_int *p, u_int v)
227 "# atomic_testandset_int"
228 : "=q" (res), /* 0 */
230 : "Ir" (v & 0x1f) /* 2 */
236 atomic_testandset_long(volatile u_long *p, u_int v)
244 "# atomic_testandset_long"
245 : "=q" (res), /* 0 */
247 : "Jr" ((u_long)(v & 0x3f)) /* 2 */
253 * We assume that a = b will do atomic loads and stores. Due to the
254 * IA32 memory model, a simple store guarantees release semantics.
256 * However, loads may pass stores, so for atomic_load_acq we have to
257 * ensure a Store/Load barrier to do the load in SMP kernels. We use
258 * "lock cmpxchg" as recommended by the AMD Software Optimization
259 * Guide, and not mfence. For UP kernels, however, the cache of the
260 * single processor is always consistent, so we only need to take care
263 #define ATOMIC_STORE(TYPE) \
264 static __inline void \
265 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\
267 __compiler_membar(); \
272 #if defined(_KERNEL) && !defined(SMP)
274 #define ATOMIC_LOAD(TYPE, LOP) \
275 static __inline u_##TYPE \
276 atomic_load_acq_##TYPE(volatile u_##TYPE *p) \
281 __compiler_membar(); \
286 #else /* !(_KERNEL && !SMP) */
288 #define ATOMIC_LOAD(TYPE, LOP) \
289 static __inline u_##TYPE \
290 atomic_load_acq_##TYPE(volatile u_##TYPE *p) \
294 __asm __volatile(MPLOCKED LOP \
295 : "=a" (res), /* 0 */ \
304 #endif /* _KERNEL && !SMP */
306 #endif /* KLD_MODULE || !__GNUCLIKE_ASM */
308 ATOMIC_ASM(set, char, "orb %b1,%0", "iq", v);
309 ATOMIC_ASM(clear, char, "andb %b1,%0", "iq", ~v);
310 ATOMIC_ASM(add, char, "addb %b1,%0", "iq", v);
311 ATOMIC_ASM(subtract, char, "subb %b1,%0", "iq", v);
313 ATOMIC_ASM(set, short, "orw %w1,%0", "ir", v);
314 ATOMIC_ASM(clear, short, "andw %w1,%0", "ir", ~v);
315 ATOMIC_ASM(add, short, "addw %w1,%0", "ir", v);
316 ATOMIC_ASM(subtract, short, "subw %w1,%0", "ir", v);
318 ATOMIC_ASM(set, int, "orl %1,%0", "ir", v);
319 ATOMIC_ASM(clear, int, "andl %1,%0", "ir", ~v);
320 ATOMIC_ASM(add, int, "addl %1,%0", "ir", v);
321 ATOMIC_ASM(subtract, int, "subl %1,%0", "ir", v);
323 ATOMIC_ASM(set, long, "orq %1,%0", "ir", v);
324 ATOMIC_ASM(clear, long, "andq %1,%0", "ir", ~v);
325 ATOMIC_ASM(add, long, "addq %1,%0", "ir", v);
326 ATOMIC_ASM(subtract, long, "subq %1,%0", "ir", v);
328 ATOMIC_LOAD(char, "cmpxchgb %b0,%1");
329 ATOMIC_LOAD(short, "cmpxchgw %w0,%1");
330 ATOMIC_LOAD(int, "cmpxchgl %0,%1");
331 ATOMIC_LOAD(long, "cmpxchgq %0,%1");
342 #ifndef WANT_FUNCTIONS
344 /* Read the current value and store a zero in the destination. */
345 #ifdef __GNUCLIKE_ASM
347 static __inline u_int
348 atomic_readandclear_int(volatile u_int *addr)
355 "# atomic_readandclear_int"
356 : "+r" (res), /* 0 */
363 static __inline u_long
364 atomic_readandclear_long(volatile u_long *addr)
371 "# atomic_readandclear_long"
372 : "+r" (res), /* 0 */
379 static __inline u_int
380 atomic_swap_int(volatile u_int *p, u_int v)
391 static __inline u_long
392 atomic_swap_long(volatile u_long *p, u_long v)
403 #else /* !__GNUCLIKE_ASM */
405 u_int atomic_readandclear_int(volatile u_int *addr);
406 u_long atomic_readandclear_long(volatile u_long *addr);
407 u_int atomic_swap_int(volatile u_int *p, u_int v);
408 u_long atomic_swap_long(volatile u_long *p, u_long v);
410 #endif /* __GNUCLIKE_ASM */
412 #define atomic_set_acq_char atomic_set_barr_char
413 #define atomic_set_rel_char atomic_set_barr_char
414 #define atomic_clear_acq_char atomic_clear_barr_char
415 #define atomic_clear_rel_char atomic_clear_barr_char
416 #define atomic_add_acq_char atomic_add_barr_char
417 #define atomic_add_rel_char atomic_add_barr_char
418 #define atomic_subtract_acq_char atomic_subtract_barr_char
419 #define atomic_subtract_rel_char atomic_subtract_barr_char
421 #define atomic_set_acq_short atomic_set_barr_short
422 #define atomic_set_rel_short atomic_set_barr_short
423 #define atomic_clear_acq_short atomic_clear_barr_short
424 #define atomic_clear_rel_short atomic_clear_barr_short
425 #define atomic_add_acq_short atomic_add_barr_short
426 #define atomic_add_rel_short atomic_add_barr_short
427 #define atomic_subtract_acq_short atomic_subtract_barr_short
428 #define atomic_subtract_rel_short atomic_subtract_barr_short
430 #define atomic_set_acq_int atomic_set_barr_int
431 #define atomic_set_rel_int atomic_set_barr_int
432 #define atomic_clear_acq_int atomic_clear_barr_int
433 #define atomic_clear_rel_int atomic_clear_barr_int
434 #define atomic_add_acq_int atomic_add_barr_int
435 #define atomic_add_rel_int atomic_add_barr_int
436 #define atomic_subtract_acq_int atomic_subtract_barr_int
437 #define atomic_subtract_rel_int atomic_subtract_barr_int
438 #define atomic_cmpset_acq_int atomic_cmpset_int
439 #define atomic_cmpset_rel_int atomic_cmpset_int
441 #define atomic_set_acq_long atomic_set_barr_long
442 #define atomic_set_rel_long atomic_set_barr_long
443 #define atomic_clear_acq_long atomic_clear_barr_long
444 #define atomic_clear_rel_long atomic_clear_barr_long
445 #define atomic_add_acq_long atomic_add_barr_long
446 #define atomic_add_rel_long atomic_add_barr_long
447 #define atomic_subtract_acq_long atomic_subtract_barr_long
448 #define atomic_subtract_rel_long atomic_subtract_barr_long
449 #define atomic_cmpset_acq_long atomic_cmpset_long
450 #define atomic_cmpset_rel_long atomic_cmpset_long
452 /* Operations on 8-bit bytes. */
453 #define atomic_set_8 atomic_set_char
454 #define atomic_set_acq_8 atomic_set_acq_char
455 #define atomic_set_rel_8 atomic_set_rel_char
456 #define atomic_clear_8 atomic_clear_char
457 #define atomic_clear_acq_8 atomic_clear_acq_char
458 #define atomic_clear_rel_8 atomic_clear_rel_char
459 #define atomic_add_8 atomic_add_char
460 #define atomic_add_acq_8 atomic_add_acq_char
461 #define atomic_add_rel_8 atomic_add_rel_char
462 #define atomic_subtract_8 atomic_subtract_char
463 #define atomic_subtract_acq_8 atomic_subtract_acq_char
464 #define atomic_subtract_rel_8 atomic_subtract_rel_char
465 #define atomic_load_acq_8 atomic_load_acq_char
466 #define atomic_store_rel_8 atomic_store_rel_char
468 /* Operations on 16-bit words. */
469 #define atomic_set_16 atomic_set_short
470 #define atomic_set_acq_16 atomic_set_acq_short
471 #define atomic_set_rel_16 atomic_set_rel_short
472 #define atomic_clear_16 atomic_clear_short
473 #define atomic_clear_acq_16 atomic_clear_acq_short
474 #define atomic_clear_rel_16 atomic_clear_rel_short
475 #define atomic_add_16 atomic_add_short
476 #define atomic_add_acq_16 atomic_add_acq_short
477 #define atomic_add_rel_16 atomic_add_rel_short
478 #define atomic_subtract_16 atomic_subtract_short
479 #define atomic_subtract_acq_16 atomic_subtract_acq_short
480 #define atomic_subtract_rel_16 atomic_subtract_rel_short
481 #define atomic_load_acq_16 atomic_load_acq_short
482 #define atomic_store_rel_16 atomic_store_rel_short
484 /* Operations on 32-bit double words. */
485 #define atomic_set_32 atomic_set_int
486 #define atomic_set_acq_32 atomic_set_acq_int
487 #define atomic_set_rel_32 atomic_set_rel_int
488 #define atomic_clear_32 atomic_clear_int
489 #define atomic_clear_acq_32 atomic_clear_acq_int
490 #define atomic_clear_rel_32 atomic_clear_rel_int
491 #define atomic_add_32 atomic_add_int
492 #define atomic_add_acq_32 atomic_add_acq_int
493 #define atomic_add_rel_32 atomic_add_rel_int
494 #define atomic_subtract_32 atomic_subtract_int
495 #define atomic_subtract_acq_32 atomic_subtract_acq_int
496 #define atomic_subtract_rel_32 atomic_subtract_rel_int
497 #define atomic_load_acq_32 atomic_load_acq_int
498 #define atomic_store_rel_32 atomic_store_rel_int
499 #define atomic_cmpset_32 atomic_cmpset_int
500 #define atomic_cmpset_acq_32 atomic_cmpset_acq_int
501 #define atomic_cmpset_rel_32 atomic_cmpset_rel_int
502 #define atomic_swap_32 atomic_swap_int
503 #define atomic_readandclear_32 atomic_readandclear_int
504 #define atomic_fetchadd_32 atomic_fetchadd_int
505 #define atomic_testandset_32 atomic_testandset_int
507 /* Operations on 64-bit quad words. */
508 #define atomic_set_64 atomic_set_long
509 #define atomic_set_acq_64 atomic_set_acq_long
510 #define atomic_set_rel_64 atomic_set_rel_long
511 #define atomic_clear_64 atomic_clear_long
512 #define atomic_clear_acq_64 atomic_clear_acq_long
513 #define atomic_clear_rel_64 atomic_clear_rel_long
514 #define atomic_add_64 atomic_add_long
515 #define atomic_add_acq_64 atomic_add_acq_long
516 #define atomic_add_rel_64 atomic_add_rel_long
517 #define atomic_subtract_64 atomic_subtract_long
518 #define atomic_subtract_acq_64 atomic_subtract_acq_long
519 #define atomic_subtract_rel_64 atomic_subtract_rel_long
520 #define atomic_load_acq_64 atomic_load_acq_long
521 #define atomic_store_rel_64 atomic_store_rel_long
522 #define atomic_cmpset_64 atomic_cmpset_long
523 #define atomic_cmpset_acq_64 atomic_cmpset_acq_long
524 #define atomic_cmpset_rel_64 atomic_cmpset_rel_long
525 #define atomic_swap_64 atomic_swap_long
526 #define atomic_readandclear_64 atomic_readandclear_long
527 #define atomic_fetchadd_64 atomic_fetchadd_long
528 #define atomic_testandset_64 atomic_testandset_long
530 /* Operations on pointers. */
531 #define atomic_set_ptr atomic_set_long
532 #define atomic_set_acq_ptr atomic_set_acq_long
533 #define atomic_set_rel_ptr atomic_set_rel_long
534 #define atomic_clear_ptr atomic_clear_long
535 #define atomic_clear_acq_ptr atomic_clear_acq_long
536 #define atomic_clear_rel_ptr atomic_clear_rel_long
537 #define atomic_add_ptr atomic_add_long
538 #define atomic_add_acq_ptr atomic_add_acq_long
539 #define atomic_add_rel_ptr atomic_add_rel_long
540 #define atomic_subtract_ptr atomic_subtract_long
541 #define atomic_subtract_acq_ptr atomic_subtract_acq_long
542 #define atomic_subtract_rel_ptr atomic_subtract_rel_long
543 #define atomic_load_acq_ptr atomic_load_acq_long
544 #define atomic_store_rel_ptr atomic_store_rel_long
545 #define atomic_cmpset_ptr atomic_cmpset_long
546 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long
547 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long
548 #define atomic_swap_ptr atomic_swap_long
549 #define atomic_readandclear_ptr atomic_readandclear_long
551 #endif /* !WANT_FUNCTIONS */
553 #endif /* !_MACHINE_ATOMIC_H_ */