1 /***********************license start***************
2 * Copyright (c) 2003-2010 Cavium Inc. (support@cavium.com). All rights
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are
10 * * Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above
14 * copyright notice, this list of conditions and the following
15 * disclaimer in the documentation and/or other materials provided
16 * with the distribution.
18 * * Neither the name of Cavium Inc. nor the names of
19 * its contributors may be used to endorse or promote products
20 * derived from this software without specific prior written
23 * This Software, including technical data, may be subject to U.S. export control
24 * laws, including the U.S. Export Administration Act and its associated
25 * regulations, and may be subject to export or import regulations in other
28 * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
29 * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR
30 * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO
31 * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR
32 * DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
33 * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
34 * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
35 * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
36 * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR
37 * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
38 ***********************license end**************************************/
49 * This file provides atomic operations
51 * <hr>$Revision: 70030 $<hr>
57 #ifndef __CVMX_ATOMIC_H__
58 #define __CVMX_ATOMIC_H__
66 * Atomically adds a signed value to a 32 bit (aligned) memory location.
68 * This version does not perform 'sync' operations to enforce memory
69 * operations. This should only be used when there are no memory operation
70 * ordering constraints. (This should NOT be used for reference counting -
71 * use the standard version instead.)
73 * @param ptr address in memory to add incr to
74 * @param incr amount to increment memory location by (signed)
76 static inline void cvmx_atomic_add32_nosync(int32_t *ptr, int32_t incr)
78 if (OCTEON_IS_MODEL(OCTEON_CN3XXX))
84 "1: ll %[tmp], %[val] \n"
85 " addu %[tmp], %[inc] \n"
86 " sc %[tmp], %[val] \n"
90 : [val] "+m" (*ptr), [tmp] "=&r" (tmp)
97 " saa %[inc], (%[base]) \n"
99 : [inc] "r" (incr), [base] "r" (ptr)
105 * Atomically adds a signed value to a 32 bit (aligned) memory location.
107 * Memory access ordering is enforced before/after the atomic operation,
108 * so no additional 'sync' instructions are required.
111 * @param ptr address in memory to add incr to
112 * @param incr amount to increment memory location by (signed)
114 static inline void cvmx_atomic_add32(int32_t *ptr, int32_t incr)
117 cvmx_atomic_add32_nosync(ptr, incr);
122 * Atomically sets a 32 bit (aligned) memory location to a value
124 * @param ptr address of memory to set
125 * @param value value to set memory location to.
127 static inline void cvmx_atomic_set32(int32_t *ptr, int32_t value)
135 * Returns the current value of a 32 bit (aligned) memory
138 * @param ptr Address of memory to get
139 * @return Value of the memory
141 static inline int32_t cvmx_atomic_get32(int32_t *ptr)
143 return *(volatile int32_t *)ptr;
147 * Atomically adds a signed value to a 64 bit (aligned) memory location.
149 * This version does not perform 'sync' operations to enforce memory
150 * operations. This should only be used when there are no memory operation
151 * ordering constraints. (This should NOT be used for reference counting -
152 * use the standard version instead.)
154 * @param ptr address in memory to add incr to
155 * @param incr amount to increment memory location by (signed)
157 static inline void cvmx_atomic_add64_nosync(int64_t *ptr, int64_t incr)
159 if (OCTEON_IS_MODEL(OCTEON_CN3XXX))
162 __asm__ __volatile__(
164 "1: lld %[tmp], %[val] \n"
165 " daddu %[tmp], %[inc] \n"
166 " scd %[tmp], %[val] \n"
167 " beqz %[tmp], 1b \n"
170 : [val] "+m" (*ptr), [tmp] "=&r" (tmp)
176 __asm__ __volatile__(
177 " saad %[inc], (%[base]) \n"
179 : [inc] "r" (incr), [base] "r" (ptr)
185 * Atomically adds a signed value to a 64 bit (aligned) memory location.
187 * Memory access ordering is enforced before/after the atomic operation,
188 * so no additional 'sync' instructions are required.
191 * @param ptr address in memory to add incr to
192 * @param incr amount to increment memory location by (signed)
194 static inline void cvmx_atomic_add64(int64_t *ptr, int64_t incr)
197 cvmx_atomic_add64_nosync(ptr, incr);
202 * Atomically sets a 64 bit (aligned) memory location to a value
204 * @param ptr address of memory to set
205 * @param value value to set memory location to.
207 static inline void cvmx_atomic_set64(int64_t *ptr, int64_t value)
215 * Returns the current value of a 64 bit (aligned) memory
218 * @param ptr Address of memory to get
219 * @return Value of the memory
221 static inline int64_t cvmx_atomic_get64(int64_t *ptr)
223 return *(volatile int64_t *)ptr;
227 * Atomically compares the old value with the value at ptr, and if they match,
228 * stores new_val to ptr.
229 * If *ptr and old don't match, function returns failure immediately.
230 * If *ptr and old match, function spins until *ptr updated to new atomically, or
231 * until *ptr and old no longer match
233 * Does no memory synchronization.
235 * @return 1 on success (match and store)
238 static inline uint32_t cvmx_atomic_compare_and_store32_nosync(uint32_t *ptr, uint32_t old_val, uint32_t new_val)
242 __asm__ __volatile__(
244 "1: ll %[tmp], %[val] \n"
246 " bne %[tmp], %[old], 2f \n"
247 " move %[tmp], %[new_val] \n"
248 " sc %[tmp], %[val] \n"
249 " beqz %[tmp], 1b \n"
253 : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
254 : [old] "r" (old_val), [new_val] "r" (new_val)
262 * Atomically compares the old value with the value at ptr, and if they match,
263 * stores new_val to ptr.
264 * If *ptr and old don't match, function returns failure immediately.
265 * If *ptr and old match, function spins until *ptr updated to new atomically, or
266 * until *ptr and old no longer match
268 * Does memory synchronization that is required to use this as a locking primitive.
270 * @return 1 on success (match and store)
273 static inline uint32_t cvmx_atomic_compare_and_store32(uint32_t *ptr, uint32_t old_val, uint32_t new_val)
277 ret = cvmx_atomic_compare_and_store32_nosync(ptr, old_val, new_val);
285 * Atomically compares the old value with the value at ptr, and if they match,
286 * stores new_val to ptr.
287 * If *ptr and old don't match, function returns failure immediately.
288 * If *ptr and old match, function spins until *ptr updated to new atomically, or
289 * until *ptr and old no longer match
291 * Does no memory synchronization.
293 * @return 1 on success (match and store)
296 static inline uint64_t cvmx_atomic_compare_and_store64_nosync(uint64_t *ptr, uint64_t old_val, uint64_t new_val)
300 __asm__ __volatile__(
302 "1: lld %[tmp], %[val] \n"
304 " bne %[tmp], %[old], 2f \n"
305 " move %[tmp], %[new_val] \n"
306 " scd %[tmp], %[val] \n"
307 " beqz %[tmp], 1b \n"
311 : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
312 : [old] "r" (old_val), [new_val] "r" (new_val)
320 * Atomically compares the old value with the value at ptr, and if they match,
321 * stores new_val to ptr.
322 * If *ptr and old don't match, function returns failure immediately.
323 * If *ptr and old match, function spins until *ptr updated to new atomically, or
324 * until *ptr and old no longer match
326 * Does memory synchronization that is required to use this as a locking primitive.
328 * @return 1 on success (match and store)
331 static inline uint64_t cvmx_atomic_compare_and_store64(uint64_t *ptr, uint64_t old_val, uint64_t new_val)
335 ret = cvmx_atomic_compare_and_store64_nosync(ptr, old_val, new_val);
341 * Atomically adds a signed value to a 64 bit (aligned) memory location,
342 * and returns previous value.
344 * This version does not perform 'sync' operations to enforce memory
345 * operations. This should only be used when there are no memory operation
346 * ordering constraints. (This should NOT be used for reference counting -
347 * use the standard version instead.)
349 * @param ptr address in memory to add incr to
350 * @param incr amount to increment memory location by (signed)
352 * @return Value of memory location before increment
354 static inline int64_t cvmx_atomic_fetch_and_add64_nosync(int64_t *ptr, int64_t incr)
358 #if !defined(__FreeBSD__) || !defined(_KERNEL)
359 if (OCTEON_IS_MODEL(OCTEON_CN6XXX) || OCTEON_IS_MODEL(OCTEON_CNF7XXX))
362 if (__builtin_constant_p(incr) && incr == 1)
364 __asm__ __volatile__(
366 : "=r" (ret), "+m" (ptr) : "r" (ptr) : "memory");
368 else if (__builtin_constant_p(incr) && incr == -1)
370 __asm__ __volatile__(
372 : "=r" (ret), "+m" (ptr) : "r" (ptr) : "memory");
376 __asm__ __volatile__(
378 : "=r" (ret), "+m" (ptr) : "r" (ptr), "r" (incr) : "memory");
385 __asm__ __volatile__(
387 "1: lld %[tmp], %[val] \n"
388 " move %[ret], %[tmp] \n"
389 " daddu %[tmp], %[inc] \n"
390 " scd %[tmp], %[val] \n"
391 " beqz %[tmp], 1b \n"
394 : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
397 #if !defined(__FreeBSD__) || !defined(_KERNEL)
405 * Atomically adds a signed value to a 64 bit (aligned) memory location,
406 * and returns previous value.
408 * Memory access ordering is enforced before/after the atomic operation,
409 * so no additional 'sync' instructions are required.
411 * @param ptr address in memory to add incr to
412 * @param incr amount to increment memory location by (signed)
414 * @return Value of memory location before increment
416 static inline int64_t cvmx_atomic_fetch_and_add64(int64_t *ptr, int64_t incr)
420 ret = cvmx_atomic_fetch_and_add64_nosync(ptr, incr);
426 * Atomically adds a signed value to a 32 bit (aligned) memory location,
427 * and returns previous value.
429 * This version does not perform 'sync' operations to enforce memory
430 * operations. This should only be used when there are no memory operation
431 * ordering constraints. (This should NOT be used for reference counting -
432 * use the standard version instead.)
434 * @param ptr address in memory to add incr to
435 * @param incr amount to increment memory location by (signed)
437 * @return Value of memory location before increment
439 static inline int32_t cvmx_atomic_fetch_and_add32_nosync(int32_t *ptr, int32_t incr)
443 #if !defined(__FreeBSD__) || !defined(_KERNEL)
444 if (OCTEON_IS_MODEL(OCTEON_CN6XXX) || OCTEON_IS_MODEL(OCTEON_CNF7XXX))
447 if (__builtin_constant_p(incr) && incr == 1)
449 __asm__ __volatile__(
451 : "=r" (ret), "+m" (ptr) : "r" (ptr) : "memory");
453 else if (__builtin_constant_p(incr) && incr == -1)
455 __asm__ __volatile__(
457 : "=r" (ret), "+m" (ptr) : "r" (ptr) : "memory");
461 __asm__ __volatile__(
463 : "=r" (ret), "+m" (ptr) : "r" (ptr), "r" (incr) : "memory");
470 __asm__ __volatile__(
472 "1: ll %[tmp], %[val] \n"
473 " move %[ret], %[tmp] \n"
474 " addu %[tmp], %[inc] \n"
475 " sc %[tmp], %[val] \n"
476 " beqz %[tmp], 1b \n"
479 : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
482 #if !defined(__FreeBSD__) || !defined(_KERNEL)
490 * Atomically adds a signed value to a 32 bit (aligned) memory location,
491 * and returns previous value.
493 * Memory access ordering is enforced before/after the atomic operation,
494 * so no additional 'sync' instructions are required.
496 * @param ptr address in memory to add incr to
497 * @param incr amount to increment memory location by (signed)
499 * @return Value of memory location before increment
501 static inline int32_t cvmx_atomic_fetch_and_add32(int32_t *ptr, int32_t incr)
505 ret = cvmx_atomic_fetch_and_add32_nosync(ptr, incr);
511 * Atomically set bits in a 64 bit (aligned) memory location,
512 * and returns previous value.
514 * This version does not perform 'sync' operations to enforce memory
515 * operations. This should only be used when there are no memory operation
516 * ordering constraints.
518 * @param ptr address in memory
519 * @param mask mask of bits to set
521 * @return Value of memory location before setting bits
523 static inline uint64_t cvmx_atomic_fetch_and_bset64_nosync(uint64_t *ptr, uint64_t mask)
527 __asm__ __volatile__(
529 "1: lld %[tmp], %[val] \n"
530 " move %[ret], %[tmp] \n"
531 " or %[tmp], %[msk] \n"
532 " scd %[tmp], %[val] \n"
533 " beqz %[tmp], 1b \n"
536 : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
544 * Atomically set bits in a 32 bit (aligned) memory location,
545 * and returns previous value.
547 * This version does not perform 'sync' operations to enforce memory
548 * operations. This should only be used when there are no memory operation
549 * ordering constraints.
551 * @param ptr address in memory
552 * @param mask mask of bits to set
554 * @return Value of memory location before setting bits
556 static inline uint32_t cvmx_atomic_fetch_and_bset32_nosync(uint32_t *ptr, uint32_t mask)
560 __asm__ __volatile__(
562 "1: ll %[tmp], %[val] \n"
563 " move %[ret], %[tmp] \n"
564 " or %[tmp], %[msk] \n"
565 " sc %[tmp], %[val] \n"
566 " beqz %[tmp], 1b \n"
569 : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
577 * Atomically clear bits in a 64 bit (aligned) memory location,
578 * and returns previous value.
580 * This version does not perform 'sync' operations to enforce memory
581 * operations. This should only be used when there are no memory operation
582 * ordering constraints.
584 * @param ptr address in memory
585 * @param mask mask of bits to clear
587 * @return Value of memory location before clearing bits
589 static inline uint64_t cvmx_atomic_fetch_and_bclr64_nosync(uint64_t *ptr, uint64_t mask)
593 __asm__ __volatile__(
596 "1: lld %[tmp], %[val] \n"
597 " move %[ret], %[tmp] \n"
598 " and %[tmp], %[msk] \n"
599 " scd %[tmp], %[val] \n"
600 " beqz %[tmp], 1b \n"
603 : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret), [msk] "+r" (mask)
610 * Atomically clear bits in a 32 bit (aligned) memory location,
611 * and returns previous value.
613 * This version does not perform 'sync' operations to enforce memory
614 * operations. This should only be used when there are no memory operation
615 * ordering constraints.
617 * @param ptr address in memory
618 * @param mask mask of bits to clear
620 * @return Value of memory location before clearing bits
622 static inline uint32_t cvmx_atomic_fetch_and_bclr32_nosync(uint32_t *ptr, uint32_t mask)
626 __asm__ __volatile__(
629 "1: ll %[tmp], %[val] \n"
630 " move %[ret], %[tmp] \n"
631 " and %[tmp], %[msk] \n"
632 " sc %[tmp], %[val] \n"
633 " beqz %[tmp], 1b \n"
636 : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret), [msk] "+r" (mask)
643 * Atomically swaps value in 64 bit (aligned) memory location,
644 * and returns previous value.
646 * This version does not perform 'sync' operations to enforce memory
647 * operations. This should only be used when there are no memory operation
648 * ordering constraints.
650 * @param ptr address in memory
651 * @param new_val new value to write
653 * @return Value of memory location before swap operation
655 static inline uint64_t cvmx_atomic_swap64_nosync(uint64_t *ptr, uint64_t new_val)
659 #if !defined(__FreeBSD__) || !defined(_KERNEL)
660 if (OCTEON_IS_MODEL(OCTEON_CN6XXX) || OCTEON_IS_MODEL(OCTEON_CNF7XXX))
663 if (__builtin_constant_p(new_val) && new_val == 0)
665 __asm__ __volatile__(
667 : "=r" (ret) : "r" (ptr) : "memory");
669 else if (__builtin_constant_p(new_val) && new_val == ~0ull)
671 __asm__ __volatile__(
673 : "=r" (ret) : "r" (ptr) : "memory");
677 __asm__ __volatile__(
679 : "=r" (ret) : "r" (ptr), "r" (new_val) : "memory");
686 __asm__ __volatile__(
688 "1: lld %[ret], %[val] \n"
689 " move %[tmp], %[new_val] \n"
690 " scd %[tmp], %[val] \n"
691 " beqz %[tmp], 1b \n"
694 : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
695 : [new_val] "r" (new_val)
697 #if !defined(__FreeBSD__) || !defined(_KERNEL)
705 * Atomically swaps value in 32 bit (aligned) memory location,
706 * and returns previous value.
708 * This version does not perform 'sync' operations to enforce memory
709 * operations. This should only be used when there are no memory operation
710 * ordering constraints.
712 * @param ptr address in memory
713 * @param new_val new value to write
715 * @return Value of memory location before swap operation
717 static inline uint32_t cvmx_atomic_swap32_nosync(uint32_t *ptr, uint32_t new_val)
721 #if !defined(__FreeBSD__) || !defined(_KERNEL)
722 if (OCTEON_IS_MODEL(OCTEON_CN6XXX) || OCTEON_IS_MODEL(OCTEON_CNF7XXX))
725 if (__builtin_constant_p(new_val) && new_val == 0)
727 __asm__ __volatile__(
729 : "=r" (ret) : "r" (ptr) : "memory");
731 else if (__builtin_constant_p(new_val) && new_val == ~0u)
733 __asm__ __volatile__(
735 : "=r" (ret) : "r" (ptr) : "memory");
739 __asm__ __volatile__(
741 : "=r" (ret) : "r" (ptr), "r" (new_val) : "memory");
748 __asm__ __volatile__(
750 "1: ll %[ret], %[val] \n"
751 " move %[tmp], %[new_val] \n"
752 " sc %[tmp], %[val] \n"
753 " beqz %[tmp], 1b \n"
756 : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
757 : [new_val] "r" (new_val)
759 #if !defined(__FreeBSD__) || !defined(_KERNEL)
770 #endif /* __CVMX_ATOMIC_H__ */