2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2005 David Xu <davidxu@freebsd.org>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 #ifndef _THR_FBSD_UMTX_H_
32 #define _THR_FBSD_UMTX_H_
38 #define DEFAULT_UMUTEX {0,0,{0,0},0,{0,0}}
40 #define DEFAULT_UMUTEX {0,0,{0,0},0,0,{0,0}}
42 #define DEFAULT_URWLOCK {0,0,0,0,{0,0,0,0}}
44 int _umtx_op_err(void *, int op, u_long, void *, void *) __hidden;
45 int __thr_umutex_lock(struct umutex *mtx, uint32_t id) __hidden;
46 int __thr_umutex_lock_spin(struct umutex *mtx, uint32_t id) __hidden;
47 int __thr_umutex_timedlock(struct umutex *mtx, uint32_t id,
48 const struct timespec *timeout) __hidden;
49 int __thr_umutex_unlock(struct umutex *mtx) __hidden;
50 int __thr_umutex_trylock(struct umutex *mtx) __hidden;
51 int __thr_umutex_set_ceiling(struct umutex *mtx, uint32_t ceiling,
52 uint32_t *oldceiling) __hidden;
54 void _thr_umutex_init(struct umutex *mtx) __hidden;
55 void _thr_urwlock_init(struct urwlock *rwl) __hidden;
57 int _thr_umtx_wait(volatile long *mtx, long exp,
58 const struct timespec *timeout) __hidden;
59 int _thr_umtx_wait_uint(volatile u_int *mtx, u_int exp,
60 const struct timespec *timeout, int shared) __hidden;
61 int _thr_umtx_timedwait_uint(volatile u_int *mtx, u_int exp, int clockid,
62 const struct timespec *timeout, int shared) __hidden;
63 int _thr_umtx_wake(volatile void *mtx, int count, int shared) __hidden;
64 int _thr_ucond_wait(struct ucond *cv, struct umutex *m,
65 const struct timespec *timeout, int flags) __hidden;
66 void _thr_ucond_init(struct ucond *cv) __hidden;
67 int _thr_ucond_signal(struct ucond *cv) __hidden;
68 int _thr_ucond_broadcast(struct ucond *cv) __hidden;
70 int __thr_rwlock_rdlock(struct urwlock *rwlock, int flags,
71 const struct timespec *tsp) __hidden;
72 int __thr_rwlock_wrlock(struct urwlock *rwlock,
73 const struct timespec *tsp) __hidden;
74 int __thr_rwlock_unlock(struct urwlock *rwlock) __hidden;
76 /* Internal used only */
77 void _thr_rwl_rdlock(struct urwlock *rwlock) __hidden;
78 void _thr_rwl_wrlock(struct urwlock *rwlock) __hidden;
79 void _thr_rwl_unlock(struct urwlock *rwlock) __hidden;
82 _thr_umutex_trylock(struct umutex *mtx, uint32_t id)
85 if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id))
87 if (__predict_false((uint32_t)mtx->m_owner == UMUTEX_RB_OWNERDEAD) &&
88 atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_RB_OWNERDEAD,
89 id | UMUTEX_CONTESTED))
91 if (__predict_false((uint32_t)mtx->m_owner == UMUTEX_RB_NOTRECOV))
92 return (ENOTRECOVERABLE);
93 if ((mtx->m_flags & UMUTEX_PRIO_PROTECT) == 0)
95 return (__thr_umutex_trylock(mtx));
99 _thr_umutex_trylock2(struct umutex *mtx, uint32_t id)
102 if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id) != 0)
104 if ((uint32_t)mtx->m_owner == UMUTEX_CONTESTED &&
105 __predict_true((mtx->m_flags & (UMUTEX_PRIO_PROTECT |
106 UMUTEX_PRIO_INHERIT)) == 0) &&
107 atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_CONTESTED,
108 id | UMUTEX_CONTESTED))
110 if (__predict_false((uint32_t)mtx->m_owner == UMUTEX_RB_OWNERDEAD) &&
111 atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_RB_OWNERDEAD,
112 id | UMUTEX_CONTESTED))
114 if (__predict_false((uint32_t)mtx->m_owner == UMUTEX_RB_NOTRECOV))
115 return (ENOTRECOVERABLE);
120 _thr_umutex_lock(struct umutex *mtx, uint32_t id)
123 if (_thr_umutex_trylock2(mtx, id) == 0)
125 return (__thr_umutex_lock(mtx, id));
129 _thr_umutex_lock_spin(struct umutex *mtx, uint32_t id)
132 if (_thr_umutex_trylock2(mtx, id) == 0)
134 return (__thr_umutex_lock_spin(mtx, id));
138 _thr_umutex_timedlock(struct umutex *mtx, uint32_t id,
139 const struct timespec *timeout)
142 if (_thr_umutex_trylock2(mtx, id) == 0)
144 return (__thr_umutex_timedlock(mtx, id, timeout));
148 _thr_umutex_unlock2(struct umutex *mtx, uint32_t id, int *defer)
150 uint32_t flags, owner;
153 flags = mtx->m_flags;
154 noncst = (flags & UMUTEX_NONCONSISTENT) != 0;
156 if ((flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) != 0) {
157 if (atomic_cmpset_rel_32(&mtx->m_owner, id, noncst ?
158 UMUTEX_RB_NOTRECOV : UMUTEX_UNOWNED))
160 return (__thr_umutex_unlock(mtx));
164 owner = mtx->m_owner;
165 if (__predict_false((owner & ~UMUTEX_CONTESTED) != id))
167 } while (__predict_false(!atomic_cmpset_rel_32(&mtx->m_owner, owner,
168 noncst ? UMUTEX_RB_NOTRECOV : UMUTEX_UNOWNED)));
169 if ((owner & UMUTEX_CONTESTED) != 0) {
170 if (defer == NULL || noncst)
171 (void)_umtx_op_err(mtx, UMTX_OP_MUTEX_WAKE2,
180 _thr_umutex_unlock(struct umutex *mtx, uint32_t id)
183 return (_thr_umutex_unlock2(mtx, id, NULL));
187 _thr_rwlock_tryrdlock(struct urwlock *rwlock, int flags)
189 int32_t state, wrflags;
191 if ((flags & URWLOCK_PREFER_READER) != 0 ||
192 (rwlock->rw_flags & URWLOCK_PREFER_READER) != 0)
193 wrflags = URWLOCK_WRITE_OWNER;
195 wrflags = URWLOCK_WRITE_OWNER | URWLOCK_WRITE_WAITERS;
196 state = rwlock->rw_state;
197 while (!(state & wrflags)) {
198 if (__predict_false(URWLOCK_READER_COUNT(state) ==
199 URWLOCK_MAX_READERS))
201 if (atomic_cmpset_acq_32(&rwlock->rw_state, state, state + 1))
203 state = rwlock->rw_state;
210 _thr_rwlock_trywrlock(struct urwlock *rwlock)
214 state = rwlock->rw_state;
215 while ((state & URWLOCK_WRITE_OWNER) == 0 &&
216 URWLOCK_READER_COUNT(state) == 0) {
217 if (atomic_cmpset_acq_32(&rwlock->rw_state, state,
218 state | URWLOCK_WRITE_OWNER))
220 state = rwlock->rw_state;
227 _thr_rwlock_rdlock(struct urwlock *rwlock, int flags, struct timespec *tsp)
230 if (_thr_rwlock_tryrdlock(rwlock, flags) == 0)
232 return (__thr_rwlock_rdlock(rwlock, flags, tsp));
236 _thr_rwlock_wrlock(struct urwlock *rwlock, struct timespec *tsp)
239 if (_thr_rwlock_trywrlock(rwlock) == 0)
241 return (__thr_rwlock_wrlock(rwlock, tsp));
245 _thr_rwlock_unlock(struct urwlock *rwlock)
249 state = rwlock->rw_state;
250 if ((state & URWLOCK_WRITE_OWNER) != 0) {
251 if (atomic_cmpset_rel_32(&rwlock->rw_state,
252 URWLOCK_WRITE_OWNER, 0))
256 if (__predict_false(URWLOCK_READER_COUNT(state) == 0))
258 if (!((state & (URWLOCK_WRITE_WAITERS |
259 URWLOCK_READ_WAITERS)) != 0 &&
260 URWLOCK_READER_COUNT(state) == 1)) {
261 if (atomic_cmpset_rel_32(&rwlock->rw_state,
264 state = rwlock->rw_state;
270 return (__thr_rwlock_unlock(rwlock));