2 * Copyright (C) 2001 Jason Evans <jasone@freebsd.org>. All rights reserved.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
7 * 1. Redistributions of source code must retain the above copyright
8 * notice(s), this list of conditions and the following disclaimer as
9 * the first lines of this file unmodified other than the possible
10 * addition of one or more copyright notices.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice(s), this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
16 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
19 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
21 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
22 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
29 * Shared/exclusive locks. This implementation assures deterministic lock
30 * granting behavior, so that slocks and xlocks are interleaved.
32 * Priority propagation will not generally raise the priority of lock holders,
33 * so should not be relied upon in combination with sx locks.
36 #include <sys/cdefs.h>
37 __FBSDID("$FreeBSD$");
39 #include <sys/param.h>
40 #include <sys/systm.h>
42 #include <sys/condvar.h>
44 #include <sys/mutex.h>
47 struct lock_class lock_class_sx = {
49 LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE | LC_UPGRADABLE
53 #define _sx_assert(sx, what, file, line)
59 struct sx_args *sargs = arg;
61 sx_init(sargs->sa_sx, sargs->sa_desc);
65 sx_init(struct sx *sx, const char *description)
67 struct lock_object *lock;
69 lock = &sx->sx_object;
70 KASSERT((lock->lo_flags & LO_INITIALIZED) == 0,
71 ("sx lock %s %p already initialized", description, sx));
72 bzero(sx, sizeof(*sx));
73 lock->lo_class = &lock_class_sx;
74 lock->lo_type = lock->lo_name = description;
75 lock->lo_flags = LO_WITNESS | LO_RECURSABLE | LO_SLEEPABLE |
77 sx->sx_lock = mtx_pool_find(mtxpool_lockbuilder, sx);
79 cv_init(&sx->sx_shrd_cv, description);
81 cv_init(&sx->sx_excl_cv, description);
83 sx->sx_xholder = NULL;
85 LOCK_LOG_INIT(lock, 0);
91 sx_destroy(struct sx *sx)
94 LOCK_LOG_DESTROY(&sx->sx_object, 0);
96 KASSERT((sx->sx_cnt == 0 && sx->sx_shrd_wcnt == 0 && sx->sx_excl_wcnt ==
97 0), ("%s (%s): holders or waiters\n", __func__,
98 sx->sx_object.lo_name));
101 cv_destroy(&sx->sx_shrd_cv);
102 cv_destroy(&sx->sx_excl_cv);
104 WITNESS_DESTROY(&sx->sx_object);
108 _sx_slock(struct sx *sx, const char *file, int line)
111 mtx_lock(sx->sx_lock);
112 KASSERT(sx->sx_xholder != curthread,
113 ("%s (%s): slock while xlock is held @ %s:%d\n", __func__,
114 sx->sx_object.lo_name, file, line));
115 WITNESS_CHECKORDER(&sx->sx_object, LOP_NEWORDER, file, line);
118 * Loop in case we lose the race for lock acquisition.
120 while (sx->sx_cnt < 0) {
122 cv_wait(&sx->sx_shrd_cv, sx->sx_lock);
126 /* Acquire a shared lock. */
129 LOCK_LOG_LOCK("SLOCK", &sx->sx_object, 0, 0, file, line);
130 WITNESS_LOCK(&sx->sx_object, 0, file, line);
132 mtx_unlock(sx->sx_lock);
136 _sx_try_slock(struct sx *sx, const char *file, int line)
139 mtx_lock(sx->sx_lock);
140 if (sx->sx_cnt >= 0) {
142 LOCK_LOG_TRY("SLOCK", &sx->sx_object, 0, 1, file, line);
143 WITNESS_LOCK(&sx->sx_object, LOP_TRYLOCK, file, line);
144 mtx_unlock(sx->sx_lock);
147 LOCK_LOG_TRY("SLOCK", &sx->sx_object, 0, 0, file, line);
148 mtx_unlock(sx->sx_lock);
154 _sx_xlock(struct sx *sx, const char *file, int line)
157 mtx_lock(sx->sx_lock);
160 * With sx locks, we're absolutely not permitted to recurse on
161 * xlocks, as it is fatal (deadlock). Normally, recursion is handled
162 * by WITNESS, but as it is not semantically correct to hold the
163 * xlock while in here, we consider it API abuse and put it under
166 KASSERT(sx->sx_xholder != curthread,
167 ("%s (%s): xlock already held @ %s:%d", __func__,
168 sx->sx_object.lo_name, file, line));
169 WITNESS_CHECKORDER(&sx->sx_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
172 /* Loop in case we lose the race for lock acquisition. */
173 while (sx->sx_cnt != 0) {
175 cv_wait(&sx->sx_excl_cv, sx->sx_lock);
179 MPASS(sx->sx_cnt == 0);
181 /* Acquire an exclusive lock. */
183 sx->sx_xholder = curthread;
185 LOCK_LOG_LOCK("XLOCK", &sx->sx_object, 0, 0, file, line);
186 WITNESS_LOCK(&sx->sx_object, LOP_EXCLUSIVE, file, line);
188 mtx_unlock(sx->sx_lock);
192 _sx_try_xlock(struct sx *sx, const char *file, int line)
195 mtx_lock(sx->sx_lock);
196 if (sx->sx_cnt == 0) {
198 sx->sx_xholder = curthread;
199 LOCK_LOG_TRY("XLOCK", &sx->sx_object, 0, 1, file, line);
200 WITNESS_LOCK(&sx->sx_object, LOP_EXCLUSIVE | LOP_TRYLOCK, file,
202 mtx_unlock(sx->sx_lock);
205 LOCK_LOG_TRY("XLOCK", &sx->sx_object, 0, 0, file, line);
206 mtx_unlock(sx->sx_lock);
212 _sx_sunlock(struct sx *sx, const char *file, int line)
215 _sx_assert(sx, SX_SLOCKED, file, line);
216 mtx_lock(sx->sx_lock);
218 WITNESS_UNLOCK(&sx->sx_object, 0, file, line);
224 * If we just released the last shared lock, wake any waiters up, giving
225 * exclusive lockers precedence. In order to make sure that exclusive
226 * lockers won't be blocked forever, don't wake shared lock waiters if
227 * there are exclusive lock waiters.
229 if (sx->sx_excl_wcnt > 0) {
231 cv_signal(&sx->sx_excl_cv);
232 } else if (sx->sx_shrd_wcnt > 0)
233 cv_broadcast(&sx->sx_shrd_cv);
235 LOCK_LOG_LOCK("SUNLOCK", &sx->sx_object, 0, 0, file, line);
237 mtx_unlock(sx->sx_lock);
241 _sx_xunlock(struct sx *sx, const char *file, int line)
244 _sx_assert(sx, SX_XLOCKED, file, line);
245 mtx_lock(sx->sx_lock);
246 MPASS(sx->sx_cnt == -1);
248 WITNESS_UNLOCK(&sx->sx_object, LOP_EXCLUSIVE, file, line);
252 sx->sx_xholder = NULL;
255 * Wake up waiters if there are any. Give precedence to slock waiters.
257 if (sx->sx_shrd_wcnt > 0)
258 cv_broadcast(&sx->sx_shrd_cv);
259 else if (sx->sx_excl_wcnt > 0)
260 cv_signal(&sx->sx_excl_cv);
262 LOCK_LOG_LOCK("XUNLOCK", &sx->sx_object, 0, 0, file, line);
264 mtx_unlock(sx->sx_lock);
268 _sx_try_upgrade(struct sx *sx, const char *file, int line)
271 _sx_assert(sx, SX_SLOCKED, file, line);
272 mtx_lock(sx->sx_lock);
274 if (sx->sx_cnt == 1) {
276 sx->sx_xholder = curthread;
278 LOCK_LOG_TRY("XUPGRADE", &sx->sx_object, 0, 1, file, line);
279 WITNESS_UPGRADE(&sx->sx_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
282 mtx_unlock(sx->sx_lock);
285 LOCK_LOG_TRY("XUPGRADE", &sx->sx_object, 0, 0, file, line);
286 mtx_unlock(sx->sx_lock);
292 _sx_downgrade(struct sx *sx, const char *file, int line)
295 _sx_assert(sx, SX_XLOCKED, file, line);
296 mtx_lock(sx->sx_lock);
297 MPASS(sx->sx_cnt == -1);
299 WITNESS_DOWNGRADE(&sx->sx_object, 0, file, line);
302 sx->sx_xholder = NULL;
303 if (sx->sx_shrd_wcnt > 0)
304 cv_broadcast(&sx->sx_shrd_cv);
306 LOCK_LOG_LOCK("XDOWNGRADE", &sx->sx_object, 0, 0, file, line);
308 mtx_unlock(sx->sx_lock);
311 #ifdef INVARIANT_SUPPORT
317 * In the non-WITNESS case, sx_assert() can only detect that at least
318 * *some* thread owns an slock, but it cannot guarantee that *this*
319 * thread owns an slock.
322 _sx_assert(struct sx *sx, int what, const char *file, int line)
325 if (panicstr != NULL)
331 witness_assert(&sx->sx_object, what, file, line);
333 mtx_lock(sx->sx_lock);
334 if (sx->sx_cnt <= 0 &&
335 (what == SX_SLOCKED || sx->sx_xholder != curthread))
336 panic("Lock %s not %slocked @ %s:%d\n",
337 sx->sx_object.lo_name, (what == SX_SLOCKED) ?
338 "share " : "", file, line);
339 mtx_unlock(sx->sx_lock);
343 mtx_lock(sx->sx_lock);
344 if (sx->sx_xholder != curthread)
345 panic("Lock %s not exclusively locked @ %s:%d\n",
346 sx->sx_object.lo_name, file, line);
347 mtx_unlock(sx->sx_lock);
351 witness_assert(&sx->sx_object, what, file, line);
354 * We are able to check only exclusive lock here,
355 * we cannot assert that *this* thread owns slock.
357 mtx_lock(sx->sx_lock);
358 if (sx->sx_xholder == curthread)
359 panic("Lock %s exclusively locked @ %s:%d\n",
360 sx->sx_object.lo_name, file, line);
361 mtx_unlock(sx->sx_lock);
365 panic("Unknown sx lock assertion: %d @ %s:%d", what, file,
369 #endif /* INVARIANT_SUPPORT */