2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2019 Kyle Evans <kevans@FreeBSD.org>
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 #ifndef _SYS__ATOMIC_SUBWORD_H_
30 #define _SYS__ATOMIC_SUBWORD_H_
33 * This header is specifically for platforms that either do not have ways to or
34 * simply do not do sub-word atomic operations. These are not ideal as they
35 * require a little more effort to make sure our atomic operations are failing
36 * because of the bits of the word we're trying to write rather than the rest
39 #ifndef _MACHINE_ATOMIC_H_
40 #error do not include this header, use machine/atomic.h
43 #include <machine/endian.h>
52 #define _ATOMIC_WORD_ALIGNED(p) \
53 (uint32_t *)((__uintptr_t)(p) - ((__uintptr_t)(p) % 4))
55 #if _BYTE_ORDER == _BIG_ENDIAN
56 #define _ATOMIC_BYTE_SHIFT(p) \
57 ((3 - ((__uintptr_t)(p) % 4)) * NBBY)
59 #define _ATOMIC_HWORD_SHIFT(p) \
60 ((2 - ((__uintptr_t)(p) % 4)) * NBBY)
62 #define _ATOMIC_BYTE_SHIFT(p) \
63 ((((__uintptr_t)(p) % 4)) * NBBY)
65 #define _ATOMIC_HWORD_SHIFT(p) \
66 ((((__uintptr_t)(p) % 4)) * NBBY)
69 #ifndef _atomic_cmpset_masked_word
71 * Pass these bad boys a couple words and a mask of the bits you care about,
72 * they'll loop until we either succeed or fail because of those bits rather
73 * than the ones we're not masking. old and val should already be preshifted to
74 * the proper position.
77 _atomic_cmpset_masked_word(uint32_t *addr, uint32_t old, uint32_t val,
86 * We'll attempt the cmpset on the entire word. Loop here in case the
87 * operation fails due to the other half-word resident in that word,
88 * rather than the half-word we're trying to operate on. Ideally we
89 * only take one trip through here. We'll have to recalculate the old
90 * value since it's the other part of the word changing.
93 old = (*addr & ~mask) | wcomp;
94 ret = atomic_fcmpset_32(addr, &old, (old & ~mask) | val);
95 } while (ret == 0 && (old & mask) == wcomp);
101 #ifndef _atomic_fcmpset_masked_word
103 _atomic_fcmpset_masked_word(uint32_t *addr, uint32_t *old, uint32_t val,
108 * fcmpset_* is documented in atomic(9) to allow spurious failures where
109 * *old == val on ll/sc architectures because the sc may fail due to
110 * parallel writes or other reasons. We take advantage of that here
111 * and only attempt once, because the caller should be compensating for
114 *old = (*addr & ~mask) | *old;
115 return (atomic_fcmpset_32(addr, old, (*old & ~mask) | val));
119 #ifndef atomic_cmpset_8
121 atomic_cmpset_8(__volatile uint8_t *addr, uint8_t old, uint8_t val)
125 shift = _ATOMIC_BYTE_SHIFT(addr);
127 return (_atomic_cmpset_masked_word(_ATOMIC_WORD_ALIGNED(addr),
128 old << shift, val << shift, 0xff << shift));
132 #ifndef atomic_fcmpset_8
134 atomic_fcmpset_8(__volatile uint8_t *addr, uint8_t *old, uint8_t val)
139 shift = _ATOMIC_BYTE_SHIFT(addr);
140 wold = *old << shift;
141 ret = _atomic_fcmpset_masked_word(_ATOMIC_WORD_ALIGNED(addr),
142 &wold, val << shift, 0xff << shift);
144 *old = (wold >> shift) & 0xff;
149 #ifndef atomic_cmpset_16
151 atomic_cmpset_16(__volatile uint16_t *addr, uint16_t old, uint16_t val)
155 shift = _ATOMIC_HWORD_SHIFT(addr);
157 return (_atomic_cmpset_masked_word(_ATOMIC_WORD_ALIGNED(addr),
158 old << shift, val << shift, 0xffff << shift));
162 #ifndef atomic_fcmpset_16
164 atomic_fcmpset_16(__volatile uint16_t *addr, uint16_t *old, uint16_t val)
169 shift = _ATOMIC_HWORD_SHIFT(addr);
170 wold = *old << shift;
171 ret = _atomic_fcmpset_masked_word(_ATOMIC_WORD_ALIGNED(addr),
172 &wold, val << shift, 0xffff << shift);
174 *old = (wold >> shift) & 0xffff;
179 #ifndef atomic_load_acq_8
180 static __inline uint8_t
181 atomic_load_acq_8(volatile uint8_t *p)
186 shift = _ATOMIC_BYTE_SHIFT(p);
187 ret = (atomic_load_acq_32(_ATOMIC_WORD_ALIGNED(p)) >> shift) & 0xff;
192 #ifndef atomic_load_acq_16
193 static __inline uint16_t
194 atomic_load_acq_16(volatile uint16_t *p)
199 shift = _ATOMIC_HWORD_SHIFT(p);
200 ret = (atomic_load_acq_32(_ATOMIC_WORD_ALIGNED(p)) >> shift) &
206 #undef _ATOMIC_WORD_ALIGNED
207 #undef _ATOMIC_BYTE_SHIFT
208 #undef _ATOMIC_HWORD_SHIFT
211 * Provide generic testandset_long implementation based on fcmpset long
212 * primitive. It may not be ideal for any given arch, so machine/atomic.h
213 * should define the macro atomic_testandset_long to override with an
214 * MD-specific version.
216 * (Organizationally, this isn't really subword atomics. But atomic_common is
217 * included too early in machine/atomic.h, so it isn't a good place for derived
218 * primitives like this.)
220 #ifndef atomic_testandset_acq_long
222 atomic_testandset_acq_long(volatile u_long *p, u_int v)
227 bit = (1ul << (v % (sizeof(*p) * NBBY)));
229 old = atomic_load_acq_long(p);
231 while (!ret && (old & bit) == 0)
232 ret = atomic_fcmpset_acq_long(p, &old, old | bit);
238 #ifndef atomic_testandset_long
240 atomic_testandset_long(volatile u_long *p, u_int v)
245 bit = (1ul << (v % (sizeof(*p) * NBBY)));
247 old = atomic_load_long(p);
249 while (!ret && (old & bit) == 0)
250 ret = atomic_fcmpset_long(p, &old, old | bit);
256 #ifndef atomic_testandclear_long
258 atomic_testandclear_long(volatile u_long *p, u_int v)
263 bit = (1ul << (v % (sizeof(*p) * NBBY)));
265 old = atomic_load_long(p);
267 while (!ret && (old & bit) != 0)
268 ret = atomic_fcmpset_long(p, &old, old & ~bit);
274 #endif /* _SYS__ATOMIC_SUBWORD_H_ */