2 * Copyright (c) 2004, David Xu <davidxu@freebsd.org>
3 * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org>
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice unmodified, this list of conditions, and the following
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
31 #include "opt_compat.h"
32 #include "opt_umtx_profiling.h"
34 #include <sys/param.h>
35 #include <sys/kernel.h>
36 #include <sys/limits.h>
38 #include <sys/malloc.h>
39 #include <sys/mutex.h>
42 #include <sys/sched.h>
44 #include <sys/sysctl.h>
45 #include <sys/sysent.h>
46 #include <sys/systm.h>
47 #include <sys/sysproto.h>
48 #include <sys/syscallsubr.h>
49 #include <sys/eventhandler.h>
53 #include <vm/vm_param.h>
55 #include <vm/vm_map.h>
56 #include <vm/vm_object.h>
58 #include <machine/cpu.h>
60 #ifdef COMPAT_FREEBSD32
61 #include <compat/freebsd32/freebsd32_proto.h>
65 #define _UMUTEX_WAIT 2
67 /* Priority inheritance mutex info. */
70 struct thread *pi_owner;
75 /* List entry to link umtx holding by thread */
76 TAILQ_ENTRY(umtx_pi) pi_link;
78 /* List entry in hash */
79 TAILQ_ENTRY(umtx_pi) pi_hashlink;
81 /* List for waiters */
82 TAILQ_HEAD(,umtx_q) pi_blocked;
84 /* Identify a userland lock object */
85 struct umtx_key pi_key;
88 /* A userland synchronous object user. */
90 /* Linked list for the hash. */
91 TAILQ_ENTRY(umtx_q) uq_link;
94 struct umtx_key uq_key;
98 #define UQF_UMTXQ 0x0001
100 /* The thread waits on. */
101 struct thread *uq_thread;
104 * Blocked on PI mutex. read can use chain lock
105 * or umtx_lock, write must have both chain lock and
106 * umtx_lock being hold.
108 struct umtx_pi *uq_pi_blocked;
110 /* On blocked list */
111 TAILQ_ENTRY(umtx_q) uq_lockq;
113 /* Thread contending with us */
114 TAILQ_HEAD(,umtx_pi) uq_pi_contested;
116 /* Inherited priority from PP mutex */
117 u_char uq_inherited_pri;
119 /* Spare queue ready to be reused */
120 struct umtxq_queue *uq_spare_queue;
122 /* The queue we on */
123 struct umtxq_queue *uq_cur_queue;
126 TAILQ_HEAD(umtxq_head, umtx_q);
128 /* Per-key wait-queue */
130 struct umtxq_head head;
132 LIST_ENTRY(umtxq_queue) link;
136 LIST_HEAD(umtxq_list, umtxq_queue);
138 /* Userland lock object's wait-queue chain */
140 /* Lock for this chain. */
143 /* List of sleep queues. */
144 struct umtxq_list uc_queue[2];
145 #define UMTX_SHARED_QUEUE 0
146 #define UMTX_EXCLUSIVE_QUEUE 1
148 LIST_HEAD(, umtxq_queue) uc_spare_queue;
153 /* Chain lock waiters */
156 /* All PI in the list */
157 TAILQ_HEAD(,umtx_pi) uc_pi_list;
159 #ifdef UMTX_PROFILING
165 #define UMTXQ_LOCKED_ASSERT(uc) mtx_assert(&(uc)->uc_lock, MA_OWNED)
166 #define UMTXQ_BUSY_ASSERT(uc) KASSERT(&(uc)->uc_busy, ("umtx chain is not busy"))
169 * Don't propagate time-sharing priority, there is a security reason,
170 * a user can simply introduce PI-mutex, let thread A lock the mutex,
171 * and let another thread B block on the mutex, because B is
172 * sleeping, its priority will be boosted, this causes A's priority to
173 * be boosted via priority propagating too and will never be lowered even
174 * if it is using 100%CPU, this is unfair to other processes.
177 #define UPRI(td) (((td)->td_user_pri >= PRI_MIN_TIMESHARE &&\
178 (td)->td_user_pri <= PRI_MAX_TIMESHARE) ?\
179 PRI_MAX_TIMESHARE : (td)->td_user_pri)
181 #define GOLDEN_RATIO_PRIME 2654404609U
182 #define UMTX_CHAINS 512
183 #define UMTX_SHIFTS (__WORD_BIT - 9)
185 #define GET_SHARE(flags) \
186 (((flags) & USYNC_PROCESS_SHARED) == 0 ? THREAD_SHARE : PROCESS_SHARE)
188 #define BUSY_SPINS 200
190 static uma_zone_t umtx_pi_zone;
191 static struct umtxq_chain umtxq_chains[2][UMTX_CHAINS];
192 static MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory");
193 static int umtx_pi_allocated;
195 static SYSCTL_NODE(_debug, OID_AUTO, umtx, CTLFLAG_RW, 0, "umtx debug");
196 SYSCTL_INT(_debug_umtx, OID_AUTO, umtx_pi_allocated, CTLFLAG_RD,
197 &umtx_pi_allocated, 0, "Allocated umtx_pi");
199 #ifdef UMTX_PROFILING
200 static long max_length;
201 SYSCTL_LONG(_debug_umtx, OID_AUTO, max_length, CTLFLAG_RD, &max_length, 0, "max_length");
202 static SYSCTL_NODE(_debug_umtx, OID_AUTO, chains, CTLFLAG_RD, 0, "umtx chain stats");
205 static void umtxq_sysinit(void *);
206 static void umtxq_hash(struct umtx_key *key);
207 static struct umtxq_chain *umtxq_getchain(struct umtx_key *key);
208 static void umtxq_lock(struct umtx_key *key);
209 static void umtxq_unlock(struct umtx_key *key);
210 static void umtxq_busy(struct umtx_key *key);
211 static void umtxq_unbusy(struct umtx_key *key);
212 static void umtxq_insert_queue(struct umtx_q *uq, int q);
213 static void umtxq_remove_queue(struct umtx_q *uq, int q);
214 static int umtxq_sleep(struct umtx_q *uq, const char *wmesg, int timo);
215 static int umtxq_count(struct umtx_key *key);
216 static struct umtx_pi *umtx_pi_alloc(int);
217 static void umtx_pi_free(struct umtx_pi *pi);
218 static int do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags);
219 static void umtx_thread_cleanup(struct thread *td);
220 static void umtx_exec_hook(void *arg __unused, struct proc *p __unused,
221 struct image_params *imgp __unused);
222 SYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_sysinit, NULL);
224 #define umtxq_signal(key, nwake) umtxq_signal_queue((key), (nwake), UMTX_SHARED_QUEUE)
225 #define umtxq_insert(uq) umtxq_insert_queue((uq), UMTX_SHARED_QUEUE)
226 #define umtxq_remove(uq) umtxq_remove_queue((uq), UMTX_SHARED_QUEUE)
228 static struct mtx umtx_lock;
230 #ifdef UMTX_PROFILING
232 umtx_init_profiling(void)
234 struct sysctl_oid *chain_oid;
238 for (i = 0; i < UMTX_CHAINS; ++i) {
239 snprintf(chain_name, sizeof(chain_name), "%d", i);
240 chain_oid = SYSCTL_ADD_NODE(NULL,
241 SYSCTL_STATIC_CHILDREN(_debug_umtx_chains), OID_AUTO,
242 chain_name, CTLFLAG_RD, NULL, "umtx hash stats");
243 SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(chain_oid), OID_AUTO,
244 "max_length0", CTLFLAG_RD, &umtxq_chains[0][i].max_length, 0, NULL);
245 SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(chain_oid), OID_AUTO,
246 "max_length1", CTLFLAG_RD, &umtxq_chains[1][i].max_length, 0, NULL);
252 umtxq_sysinit(void *arg __unused)
256 umtx_pi_zone = uma_zcreate("umtx pi", sizeof(struct umtx_pi),
257 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
258 for (i = 0; i < 2; ++i) {
259 for (j = 0; j < UMTX_CHAINS; ++j) {
260 mtx_init(&umtxq_chains[i][j].uc_lock, "umtxql", NULL,
261 MTX_DEF | MTX_DUPOK);
262 LIST_INIT(&umtxq_chains[i][j].uc_queue[0]);
263 LIST_INIT(&umtxq_chains[i][j].uc_queue[1]);
264 LIST_INIT(&umtxq_chains[i][j].uc_spare_queue);
265 TAILQ_INIT(&umtxq_chains[i][j].uc_pi_list);
266 umtxq_chains[i][j].uc_busy = 0;
267 umtxq_chains[i][j].uc_waiters = 0;
268 #ifdef UMTX_PROFILING
269 umtxq_chains[i][j].length = 0;
270 umtxq_chains[i][j].max_length = 0;
274 #ifdef UMTX_PROFILING
275 umtx_init_profiling();
277 mtx_init(&umtx_lock, "umtx lock", NULL, MTX_SPIN);
278 EVENTHANDLER_REGISTER(process_exec, umtx_exec_hook, NULL,
279 EVENTHANDLER_PRI_ANY);
287 uq = malloc(sizeof(struct umtx_q), M_UMTX, M_WAITOK | M_ZERO);
288 uq->uq_spare_queue = malloc(sizeof(struct umtxq_queue), M_UMTX, M_WAITOK | M_ZERO);
289 TAILQ_INIT(&uq->uq_spare_queue->head);
290 TAILQ_INIT(&uq->uq_pi_contested);
291 uq->uq_inherited_pri = PRI_MAX;
296 umtxq_free(struct umtx_q *uq)
298 MPASS(uq->uq_spare_queue != NULL);
299 free(uq->uq_spare_queue, M_UMTX);
304 umtxq_hash(struct umtx_key *key)
306 unsigned n = (uintptr_t)key->info.both.a + key->info.both.b;
307 key->hash = ((n * GOLDEN_RATIO_PRIME) >> UMTX_SHIFTS) % UMTX_CHAINS;
310 static inline struct umtxq_chain *
311 umtxq_getchain(struct umtx_key *key)
313 if (key->type <= TYPE_SEM)
314 return (&umtxq_chains[1][key->hash]);
315 return (&umtxq_chains[0][key->hash]);
322 umtxq_lock(struct umtx_key *key)
324 struct umtxq_chain *uc;
326 uc = umtxq_getchain(key);
327 mtx_lock(&uc->uc_lock);
334 umtxq_unlock(struct umtx_key *key)
336 struct umtxq_chain *uc;
338 uc = umtxq_getchain(key);
339 mtx_unlock(&uc->uc_lock);
343 * Set chain to busy state when following operation
344 * may be blocked (kernel mutex can not be used).
347 umtxq_busy(struct umtx_key *key)
349 struct umtxq_chain *uc;
351 uc = umtxq_getchain(key);
352 mtx_assert(&uc->uc_lock, MA_OWNED);
356 int count = BUSY_SPINS;
359 while (uc->uc_busy && --count > 0)
365 while (uc->uc_busy) {
367 msleep(uc, &uc->uc_lock, 0, "umtxqb", 0);
378 umtxq_unbusy(struct umtx_key *key)
380 struct umtxq_chain *uc;
382 uc = umtxq_getchain(key);
383 mtx_assert(&uc->uc_lock, MA_OWNED);
384 KASSERT(uc->uc_busy != 0, ("not busy"));
390 static struct umtxq_queue *
391 umtxq_queue_lookup(struct umtx_key *key, int q)
393 struct umtxq_queue *uh;
394 struct umtxq_chain *uc;
396 uc = umtxq_getchain(key);
397 UMTXQ_LOCKED_ASSERT(uc);
398 LIST_FOREACH(uh, &uc->uc_queue[q], link) {
399 if (umtx_key_match(&uh->key, key))
407 umtxq_insert_queue(struct umtx_q *uq, int q)
409 struct umtxq_queue *uh;
410 struct umtxq_chain *uc;
412 uc = umtxq_getchain(&uq->uq_key);
413 UMTXQ_LOCKED_ASSERT(uc);
414 KASSERT((uq->uq_flags & UQF_UMTXQ) == 0, ("umtx_q is already on queue"));
415 uh = umtxq_queue_lookup(&uq->uq_key, q);
417 LIST_INSERT_HEAD(&uc->uc_spare_queue, uq->uq_spare_queue, link);
419 uh = uq->uq_spare_queue;
420 uh->key = uq->uq_key;
421 LIST_INSERT_HEAD(&uc->uc_queue[q], uh, link);
423 uq->uq_spare_queue = NULL;
425 TAILQ_INSERT_TAIL(&uh->head, uq, uq_link);
427 #ifdef UMTX_PROFILING
429 if (uc->length > uc->max_length) {
430 uc->max_length = uc->length;
431 if (uc->max_length > max_length)
432 max_length = uc->max_length;
435 uq->uq_flags |= UQF_UMTXQ;
436 uq->uq_cur_queue = uh;
441 umtxq_remove_queue(struct umtx_q *uq, int q)
443 struct umtxq_chain *uc;
444 struct umtxq_queue *uh;
446 uc = umtxq_getchain(&uq->uq_key);
447 UMTXQ_LOCKED_ASSERT(uc);
448 if (uq->uq_flags & UQF_UMTXQ) {
449 uh = uq->uq_cur_queue;
450 TAILQ_REMOVE(&uh->head, uq, uq_link);
452 #ifdef UMTX_PROFILING
455 uq->uq_flags &= ~UQF_UMTXQ;
456 if (TAILQ_EMPTY(&uh->head)) {
457 KASSERT(uh->length == 0,
458 ("inconsistent umtxq_queue length"));
459 LIST_REMOVE(uh, link);
461 uh = LIST_FIRST(&uc->uc_spare_queue);
462 KASSERT(uh != NULL, ("uc_spare_queue is empty"));
463 LIST_REMOVE(uh, link);
465 uq->uq_spare_queue = uh;
466 uq->uq_cur_queue = NULL;
471 * Check if there are multiple waiters
474 umtxq_count(struct umtx_key *key)
476 struct umtxq_chain *uc;
477 struct umtxq_queue *uh;
479 uc = umtxq_getchain(key);
480 UMTXQ_LOCKED_ASSERT(uc);
481 uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE);
488 * Check if there are multiple PI waiters and returns first
492 umtxq_count_pi(struct umtx_key *key, struct umtx_q **first)
494 struct umtxq_chain *uc;
495 struct umtxq_queue *uh;
498 uc = umtxq_getchain(key);
499 UMTXQ_LOCKED_ASSERT(uc);
500 uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE);
502 *first = TAILQ_FIRST(&uh->head);
509 umtxq_check_susp(struct thread *td)
515 * The check for TDF_NEEDSUSPCHK is racy, but it is enough to
516 * eventually break the lockstep loop.
518 if ((td->td_flags & TDF_NEEDSUSPCHK) == 0)
523 if (P_SHOULDSTOP(p) ||
524 ((p->p_flag & P_TRACED) && (td->td_dbgflags & TDB_SUSPEND))) {
525 if (p->p_flag & P_SINGLE_EXIT)
535 * Wake up threads waiting on an userland object.
539 umtxq_signal_queue(struct umtx_key *key, int n_wake, int q)
541 struct umtxq_chain *uc;
542 struct umtxq_queue *uh;
547 uc = umtxq_getchain(key);
548 UMTXQ_LOCKED_ASSERT(uc);
549 uh = umtxq_queue_lookup(key, q);
551 while ((uq = TAILQ_FIRST(&uh->head)) != NULL) {
552 umtxq_remove_queue(uq, q);
563 * Wake up specified thread.
566 umtxq_signal_thread(struct umtx_q *uq)
568 struct umtxq_chain *uc;
570 uc = umtxq_getchain(&uq->uq_key);
571 UMTXQ_LOCKED_ASSERT(uc);
577 * Put thread into sleep state, before sleeping, check if
578 * thread was removed from umtx queue.
581 umtxq_sleep(struct umtx_q *uq, const char *wmesg, int timo)
583 struct umtxq_chain *uc;
586 uc = umtxq_getchain(&uq->uq_key);
587 UMTXQ_LOCKED_ASSERT(uc);
588 if (!(uq->uq_flags & UQF_UMTXQ))
590 error = msleep(uq, &uc->uc_lock, PCATCH, wmesg, timo);
591 if (error == EWOULDBLOCK)
597 * Convert userspace address into unique logical address.
600 umtx_key_get(void *addr, int type, int share, struct umtx_key *key)
602 struct thread *td = curthread;
604 vm_map_entry_t entry;
610 if (share == THREAD_SHARE) {
612 key->info.private.vs = td->td_proc->p_vmspace;
613 key->info.private.addr = (uintptr_t)addr;
615 MPASS(share == PROCESS_SHARE || share == AUTO_SHARE);
616 map = &td->td_proc->p_vmspace->vm_map;
617 if (vm_map_lookup(&map, (vm_offset_t)addr, VM_PROT_WRITE,
618 &entry, &key->info.shared.object, &pindex, &prot,
619 &wired) != KERN_SUCCESS) {
623 if ((share == PROCESS_SHARE) ||
624 (share == AUTO_SHARE &&
625 VM_INHERIT_SHARE == entry->inheritance)) {
627 key->info.shared.offset = entry->offset + entry->start -
629 vm_object_reference(key->info.shared.object);
632 key->info.private.vs = td->td_proc->p_vmspace;
633 key->info.private.addr = (uintptr_t)addr;
635 vm_map_lookup_done(map, entry);
646 umtx_key_release(struct umtx_key *key)
649 vm_object_deallocate(key->info.shared.object);
653 * Lock a umtx object.
656 _do_lock_umtx(struct thread *td, struct umtx *umtx, u_long id, int timo)
666 * Care must be exercised when dealing with umtx structure. It
667 * can fault on any access.
671 * Try the uncontested case. This should be done in userland.
673 owner = casuword(&umtx->u_owner, UMTX_UNOWNED, id);
675 /* The acquire succeeded. */
676 if (owner == UMTX_UNOWNED)
679 /* The address was invalid. */
683 /* If no one owns it but it is contested try to acquire it. */
684 if (owner == UMTX_CONTESTED) {
685 owner = casuword(&umtx->u_owner,
686 UMTX_CONTESTED, id | UMTX_CONTESTED);
688 if (owner == UMTX_CONTESTED)
691 /* The address was invalid. */
695 error = umtxq_check_susp(td);
699 /* If this failed the lock has changed, restart. */
704 * If we caught a signal, we have retried and now
710 if ((error = umtx_key_get(umtx, TYPE_SIMPLE_LOCK,
711 AUTO_SHARE, &uq->uq_key)) != 0)
714 umtxq_lock(&uq->uq_key);
715 umtxq_busy(&uq->uq_key);
717 umtxq_unbusy(&uq->uq_key);
718 umtxq_unlock(&uq->uq_key);
721 * Set the contested bit so that a release in user space
722 * knows to use the system call for unlock. If this fails
723 * either some one else has acquired the lock or it has been
726 old = casuword(&umtx->u_owner, owner, owner | UMTX_CONTESTED);
728 /* The address was invalid. */
730 umtxq_lock(&uq->uq_key);
732 umtxq_unlock(&uq->uq_key);
733 umtx_key_release(&uq->uq_key);
738 * We set the contested bit, sleep. Otherwise the lock changed
739 * and we need to retry or we lost a race to the thread
740 * unlocking the umtx.
742 umtxq_lock(&uq->uq_key);
744 error = umtxq_sleep(uq, "umtx", timo);
746 umtxq_unlock(&uq->uq_key);
747 umtx_key_release(&uq->uq_key);
750 error = umtxq_check_susp(td);
757 * Lock a umtx object.
760 do_lock_umtx(struct thread *td, struct umtx *umtx, u_long id,
761 struct timespec *timeout)
763 struct timespec ts, ts2, ts3;
767 if (timeout == NULL) {
768 error = _do_lock_umtx(td, umtx, id, 0);
769 /* Mutex locking is restarted if it is interrupted. */
774 timespecadd(&ts, timeout);
775 TIMESPEC_TO_TIMEVAL(&tv, timeout);
777 error = _do_lock_umtx(td, umtx, id, tvtohz(&tv));
778 if (error != ETIMEDOUT)
781 if (timespeccmp(&ts2, &ts, >=)) {
786 timespecsub(&ts3, &ts2);
787 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
789 /* Timed-locking is not restarted. */
790 if (error == ERESTART)
797 * Unlock a umtx object.
800 do_unlock_umtx(struct thread *td, struct umtx *umtx, u_long id)
809 * Make sure we own this mtx.
811 owner = fuword(__DEVOLATILE(u_long *, &umtx->u_owner));
815 if ((owner & ~UMTX_CONTESTED) != id)
818 /* This should be done in userland */
819 if ((owner & UMTX_CONTESTED) == 0) {
820 old = casuword(&umtx->u_owner, owner, UMTX_UNOWNED);
828 /* We should only ever be in here for contested locks */
829 if ((error = umtx_key_get(umtx, TYPE_SIMPLE_LOCK, AUTO_SHARE,
835 count = umtxq_count(&key);
839 * When unlocking the umtx, it must be marked as unowned if
840 * there is zero or one thread only waiting for it.
841 * Otherwise, it must be marked as contested.
843 old = casuword(&umtx->u_owner, owner,
844 count <= 1 ? UMTX_UNOWNED : UMTX_CONTESTED);
846 umtxq_signal(&key,1);
849 umtx_key_release(&key);
857 #ifdef COMPAT_FREEBSD32
860 * Lock a umtx object.
863 _do_lock_umtx32(struct thread *td, uint32_t *m, uint32_t id, int timo)
873 * Care must be exercised when dealing with umtx structure. It
874 * can fault on any access.
878 * Try the uncontested case. This should be done in userland.
880 owner = casuword32(m, UMUTEX_UNOWNED, id);
882 /* The acquire succeeded. */
883 if (owner == UMUTEX_UNOWNED)
886 /* The address was invalid. */
890 /* If no one owns it but it is contested try to acquire it. */
891 if (owner == UMUTEX_CONTESTED) {
892 owner = casuword32(m,
893 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
894 if (owner == UMUTEX_CONTESTED)
897 /* The address was invalid. */
901 error = umtxq_check_susp(td);
905 /* If this failed the lock has changed, restart. */
910 * If we caught a signal, we have retried and now
916 if ((error = umtx_key_get(m, TYPE_SIMPLE_LOCK,
917 AUTO_SHARE, &uq->uq_key)) != 0)
920 umtxq_lock(&uq->uq_key);
921 umtxq_busy(&uq->uq_key);
923 umtxq_unbusy(&uq->uq_key);
924 umtxq_unlock(&uq->uq_key);
927 * Set the contested bit so that a release in user space
928 * knows to use the system call for unlock. If this fails
929 * either some one else has acquired the lock or it has been
932 old = casuword32(m, owner, owner | UMUTEX_CONTESTED);
934 /* The address was invalid. */
936 umtxq_lock(&uq->uq_key);
938 umtxq_unlock(&uq->uq_key);
939 umtx_key_release(&uq->uq_key);
944 * We set the contested bit, sleep. Otherwise the lock changed
945 * and we need to retry or we lost a race to the thread
946 * unlocking the umtx.
948 umtxq_lock(&uq->uq_key);
950 error = umtxq_sleep(uq, "umtx", timo);
952 umtxq_unlock(&uq->uq_key);
953 umtx_key_release(&uq->uq_key);
956 error = umtxq_check_susp(td);
963 * Lock a umtx object.
966 do_lock_umtx32(struct thread *td, void *m, uint32_t id,
967 struct timespec *timeout)
969 struct timespec ts, ts2, ts3;
973 if (timeout == NULL) {
974 error = _do_lock_umtx32(td, m, id, 0);
975 /* Mutex locking is restarted if it is interrupted. */
980 timespecadd(&ts, timeout);
981 TIMESPEC_TO_TIMEVAL(&tv, timeout);
983 error = _do_lock_umtx32(td, m, id, tvtohz(&tv));
984 if (error != ETIMEDOUT)
987 if (timespeccmp(&ts2, &ts, >=)) {
992 timespecsub(&ts3, &ts2);
993 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
995 /* Timed-locking is not restarted. */
996 if (error == ERESTART)
1003 * Unlock a umtx object.
1006 do_unlock_umtx32(struct thread *td, uint32_t *m, uint32_t id)
1008 struct umtx_key key;
1015 * Make sure we own this mtx.
1017 owner = fuword32(m);
1021 if ((owner & ~UMUTEX_CONTESTED) != id)
1024 /* This should be done in userland */
1025 if ((owner & UMUTEX_CONTESTED) == 0) {
1026 old = casuword32(m, owner, UMUTEX_UNOWNED);
1034 /* We should only ever be in here for contested locks */
1035 if ((error = umtx_key_get(m, TYPE_SIMPLE_LOCK, AUTO_SHARE,
1041 count = umtxq_count(&key);
1045 * When unlocking the umtx, it must be marked as unowned if
1046 * there is zero or one thread only waiting for it.
1047 * Otherwise, it must be marked as contested.
1049 old = casuword32(m, owner,
1050 count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
1052 umtxq_signal(&key,1);
1055 umtx_key_release(&key);
1065 * Fetch and compare value, sleep on the address if value is not changed.
1068 do_wait(struct thread *td, void *addr, u_long id,
1069 struct timespec *timeout, int compat32, int is_private)
1072 struct timespec ts, ts2, ts3;
1078 if ((error = umtx_key_get(addr, TYPE_SIMPLE_WAIT,
1079 is_private ? THREAD_SHARE : AUTO_SHARE, &uq->uq_key)) != 0)
1082 umtxq_lock(&uq->uq_key);
1084 umtxq_unlock(&uq->uq_key);
1088 tmp = (unsigned int)fuword32(addr);
1090 umtxq_lock(&uq->uq_key);
1092 umtxq_unlock(&uq->uq_key);
1093 } else if (timeout == NULL) {
1094 umtxq_lock(&uq->uq_key);
1095 error = umtxq_sleep(uq, "uwait", 0);
1097 umtxq_unlock(&uq->uq_key);
1100 timespecadd(&ts, timeout);
1101 TIMESPEC_TO_TIMEVAL(&tv, timeout);
1102 umtxq_lock(&uq->uq_key);
1104 error = umtxq_sleep(uq, "uwait", tvtohz(&tv));
1105 if (!(uq->uq_flags & UQF_UMTXQ)) {
1109 if (error != ETIMEDOUT)
1111 umtxq_unlock(&uq->uq_key);
1112 getnanouptime(&ts2);
1113 if (timespeccmp(&ts2, &ts, >=)) {
1115 umtxq_lock(&uq->uq_key);
1119 timespecsub(&ts3, &ts2);
1120 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
1121 umtxq_lock(&uq->uq_key);
1124 umtxq_unlock(&uq->uq_key);
1126 umtx_key_release(&uq->uq_key);
1127 if (error == ERESTART)
1133 * Wake up threads sleeping on the specified address.
1136 kern_umtx_wake(struct thread *td, void *uaddr, int n_wake, int is_private)
1138 struct umtx_key key;
1141 if ((ret = umtx_key_get(uaddr, TYPE_SIMPLE_WAIT,
1142 is_private ? THREAD_SHARE : AUTO_SHARE, &key)) != 0)
1145 ret = umtxq_signal(&key, n_wake);
1147 umtx_key_release(&key);
1152 * Lock PTHREAD_PRIO_NONE protocol POSIX mutex.
1155 _do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, int timo,
1159 uint32_t owner, old, id;
1166 * Care must be exercised when dealing with umtx structure. It
1167 * can fault on any access.
1170 owner = fuword32(__DEVOLATILE(void *, &m->m_owner));
1171 if (mode == _UMUTEX_WAIT) {
1172 if (owner == UMUTEX_UNOWNED || owner == UMUTEX_CONTESTED)
1176 * Try the uncontested case. This should be done in userland.
1178 owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id);
1180 /* The acquire succeeded. */
1181 if (owner == UMUTEX_UNOWNED)
1184 /* The address was invalid. */
1188 /* If no one owns it but it is contested try to acquire it. */
1189 if (owner == UMUTEX_CONTESTED) {
1190 owner = casuword32(&m->m_owner,
1191 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
1193 if (owner == UMUTEX_CONTESTED)
1196 /* The address was invalid. */
1200 error = umtxq_check_susp(td);
1204 /* If this failed the lock has changed, restart. */
1209 if ((flags & UMUTEX_ERROR_CHECK) != 0 &&
1210 (owner & ~UMUTEX_CONTESTED) == id)
1213 if (mode == _UMUTEX_TRY)
1217 * If we caught a signal, we have retried and now
1223 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX,
1224 GET_SHARE(flags), &uq->uq_key)) != 0)
1227 umtxq_lock(&uq->uq_key);
1228 umtxq_busy(&uq->uq_key);
1230 umtxq_unlock(&uq->uq_key);
1233 * Set the contested bit so that a release in user space
1234 * knows to use the system call for unlock. If this fails
1235 * either some one else has acquired the lock or it has been
1238 old = casuword32(&m->m_owner, owner, owner | UMUTEX_CONTESTED);
1240 /* The address was invalid. */
1242 umtxq_lock(&uq->uq_key);
1244 umtxq_unbusy(&uq->uq_key);
1245 umtxq_unlock(&uq->uq_key);
1246 umtx_key_release(&uq->uq_key);
1251 * We set the contested bit, sleep. Otherwise the lock changed
1252 * and we need to retry or we lost a race to the thread
1253 * unlocking the umtx.
1255 umtxq_lock(&uq->uq_key);
1256 umtxq_unbusy(&uq->uq_key);
1258 error = umtxq_sleep(uq, "umtxn", timo);
1260 umtxq_unlock(&uq->uq_key);
1261 umtx_key_release(&uq->uq_key);
1264 error = umtxq_check_susp(td);
1271 * Lock PTHREAD_PRIO_NONE protocol POSIX mutex.
1274 * Unlock PTHREAD_PRIO_NONE protocol POSIX mutex.
1277 do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags)
1279 struct umtx_key key;
1280 uint32_t owner, old, id;
1286 * Make sure we own this mtx.
1288 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
1292 if ((owner & ~UMUTEX_CONTESTED) != id)
1295 if ((owner & UMUTEX_CONTESTED) == 0) {
1296 old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED);
1304 /* We should only ever be in here for contested locks */
1305 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags),
1311 count = umtxq_count(&key);
1315 * When unlocking the umtx, it must be marked as unowned if
1316 * there is zero or one thread only waiting for it.
1317 * Otherwise, it must be marked as contested.
1319 old = casuword32(&m->m_owner, owner,
1320 count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
1322 umtxq_signal(&key,1);
1325 umtx_key_release(&key);
1334 * Check if the mutex is available and wake up a waiter,
1335 * only for simple mutex.
1338 do_wake_umutex(struct thread *td, struct umutex *m)
1340 struct umtx_key key;
1346 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
1350 if ((owner & ~UMUTEX_CONTESTED) != 0)
1353 flags = fuword32(&m->m_flags);
1355 /* We should only ever be in here for contested locks */
1356 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags),
1362 count = umtxq_count(&key);
1366 owner = casuword32(&m->m_owner, UMUTEX_CONTESTED, UMUTEX_UNOWNED);
1369 if (count != 0 && (owner & ~UMUTEX_CONTESTED) == 0)
1370 umtxq_signal(&key, 1);
1373 umtx_key_release(&key);
1378 * Check if the mutex has waiters and tries to fix contention bit.
1381 do_wake2_umutex(struct thread *td, struct umutex *m, uint32_t flags)
1383 struct umtx_key key;
1384 uint32_t owner, old;
1389 switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) {
1391 type = TYPE_NORMAL_UMUTEX;
1393 case UMUTEX_PRIO_INHERIT:
1394 type = TYPE_PI_UMUTEX;
1396 case UMUTEX_PRIO_PROTECT:
1397 type = TYPE_PP_UMUTEX;
1402 if ((error = umtx_key_get(m, type, GET_SHARE(flags),
1409 count = umtxq_count(&key);
1412 * Only repair contention bit if there is a waiter, this means the mutex
1413 * is still being referenced by userland code, otherwise don't update
1417 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
1418 while ((owner & UMUTEX_CONTESTED) ==0) {
1419 old = casuword32(&m->m_owner, owner,
1420 owner|UMUTEX_CONTESTED);
1426 error = umtxq_check_susp(td);
1430 } else if (count == 1) {
1431 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
1432 while ((owner & ~UMUTEX_CONTESTED) != 0 &&
1433 (owner & UMUTEX_CONTESTED) == 0) {
1434 old = casuword32(&m->m_owner, owner,
1435 owner|UMUTEX_CONTESTED);
1441 error = umtxq_check_susp(td);
1449 umtxq_signal(&key, INT_MAX);
1451 else if (count != 0 && (owner & ~UMUTEX_CONTESTED) == 0)
1452 umtxq_signal(&key, 1);
1455 umtx_key_release(&key);
1459 static inline struct umtx_pi *
1460 umtx_pi_alloc(int flags)
1464 pi = uma_zalloc(umtx_pi_zone, M_ZERO | flags);
1465 TAILQ_INIT(&pi->pi_blocked);
1466 atomic_add_int(&umtx_pi_allocated, 1);
1471 umtx_pi_free(struct umtx_pi *pi)
1473 uma_zfree(umtx_pi_zone, pi);
1474 atomic_add_int(&umtx_pi_allocated, -1);
1478 * Adjust the thread's position on a pi_state after its priority has been
1482 umtx_pi_adjust_thread(struct umtx_pi *pi, struct thread *td)
1484 struct umtx_q *uq, *uq1, *uq2;
1487 mtx_assert(&umtx_lock, MA_OWNED);
1494 * Check if the thread needs to be moved on the blocked chain.
1495 * It needs to be moved if either its priority is lower than
1496 * the previous thread or higher than the next thread.
1498 uq1 = TAILQ_PREV(uq, umtxq_head, uq_lockq);
1499 uq2 = TAILQ_NEXT(uq, uq_lockq);
1500 if ((uq1 != NULL && UPRI(td) < UPRI(uq1->uq_thread)) ||
1501 (uq2 != NULL && UPRI(td) > UPRI(uq2->uq_thread))) {
1503 * Remove thread from blocked chain and determine where
1504 * it should be moved to.
1506 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
1507 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) {
1508 td1 = uq1->uq_thread;
1509 MPASS(td1->td_proc->p_magic == P_MAGIC);
1510 if (UPRI(td1) > UPRI(td))
1515 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq);
1517 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq);
1523 * Propagate priority when a thread is blocked on POSIX
1527 umtx_propagate_priority(struct thread *td)
1533 mtx_assert(&umtx_lock, MA_OWNED);
1536 pi = uq->uq_pi_blocked;
1542 if (td == NULL || td == curthread)
1545 MPASS(td->td_proc != NULL);
1546 MPASS(td->td_proc->p_magic == P_MAGIC);
1549 if (td->td_lend_user_pri > pri)
1550 sched_lend_user_prio(td, pri);
1558 * Pick up the lock that td is blocked on.
1561 pi = uq->uq_pi_blocked;
1564 /* Resort td on the list if needed. */
1565 umtx_pi_adjust_thread(pi, td);
1570 * Unpropagate priority for a PI mutex when a thread blocked on
1571 * it is interrupted by signal or resumed by others.
1574 umtx_repropagate_priority(struct umtx_pi *pi)
1576 struct umtx_q *uq, *uq_owner;
1577 struct umtx_pi *pi2;
1580 mtx_assert(&umtx_lock, MA_OWNED);
1582 while (pi != NULL && pi->pi_owner != NULL) {
1584 uq_owner = pi->pi_owner->td_umtxq;
1586 TAILQ_FOREACH(pi2, &uq_owner->uq_pi_contested, pi_link) {
1587 uq = TAILQ_FIRST(&pi2->pi_blocked);
1589 if (pri > UPRI(uq->uq_thread))
1590 pri = UPRI(uq->uq_thread);
1594 if (pri > uq_owner->uq_inherited_pri)
1595 pri = uq_owner->uq_inherited_pri;
1596 thread_lock(pi->pi_owner);
1597 sched_lend_user_prio(pi->pi_owner, pri);
1598 thread_unlock(pi->pi_owner);
1599 if ((pi = uq_owner->uq_pi_blocked) != NULL)
1600 umtx_pi_adjust_thread(pi, uq_owner->uq_thread);
1605 * Insert a PI mutex into owned list.
1608 umtx_pi_setowner(struct umtx_pi *pi, struct thread *owner)
1610 struct umtx_q *uq_owner;
1612 uq_owner = owner->td_umtxq;
1613 mtx_assert(&umtx_lock, MA_OWNED);
1614 if (pi->pi_owner != NULL)
1615 panic("pi_ower != NULL");
1616 pi->pi_owner = owner;
1617 TAILQ_INSERT_TAIL(&uq_owner->uq_pi_contested, pi, pi_link);
1621 * Claim ownership of a PI mutex.
1624 umtx_pi_claim(struct umtx_pi *pi, struct thread *owner)
1626 struct umtx_q *uq, *uq_owner;
1628 uq_owner = owner->td_umtxq;
1629 mtx_lock_spin(&umtx_lock);
1630 if (pi->pi_owner == owner) {
1631 mtx_unlock_spin(&umtx_lock);
1635 if (pi->pi_owner != NULL) {
1637 * userland may have already messed the mutex, sigh.
1639 mtx_unlock_spin(&umtx_lock);
1642 umtx_pi_setowner(pi, owner);
1643 uq = TAILQ_FIRST(&pi->pi_blocked);
1647 pri = UPRI(uq->uq_thread);
1649 if (pri < UPRI(owner))
1650 sched_lend_user_prio(owner, pri);
1651 thread_unlock(owner);
1653 mtx_unlock_spin(&umtx_lock);
1658 * Adjust a thread's order position in its blocked PI mutex,
1659 * this may result new priority propagating process.
1662 umtx_pi_adjust(struct thread *td, u_char oldpri)
1668 mtx_lock_spin(&umtx_lock);
1670 * Pick up the lock that td is blocked on.
1672 pi = uq->uq_pi_blocked;
1674 umtx_pi_adjust_thread(pi, td);
1675 umtx_repropagate_priority(pi);
1677 mtx_unlock_spin(&umtx_lock);
1681 * Sleep on a PI mutex.
1684 umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi,
1685 uint32_t owner, const char *wmesg, int timo)
1687 struct umtxq_chain *uc;
1688 struct thread *td, *td1;
1694 KASSERT(td == curthread, ("inconsistent uq_thread"));
1695 uc = umtxq_getchain(&uq->uq_key);
1696 UMTXQ_LOCKED_ASSERT(uc);
1697 UMTXQ_BUSY_ASSERT(uc);
1699 mtx_lock_spin(&umtx_lock);
1700 if (pi->pi_owner == NULL) {
1701 mtx_unlock_spin(&umtx_lock);
1702 /* XXX Only look up thread in current process. */
1703 td1 = tdfind(owner, curproc->p_pid);
1704 mtx_lock_spin(&umtx_lock);
1706 if (pi->pi_owner == NULL)
1707 umtx_pi_setowner(pi, td1);
1708 PROC_UNLOCK(td1->td_proc);
1712 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) {
1713 pri = UPRI(uq1->uq_thread);
1719 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq);
1721 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq);
1723 uq->uq_pi_blocked = pi;
1725 td->td_flags |= TDF_UPIBLOCKED;
1727 umtx_propagate_priority(td);
1728 mtx_unlock_spin(&umtx_lock);
1729 umtxq_unbusy(&uq->uq_key);
1731 if (uq->uq_flags & UQF_UMTXQ) {
1732 error = msleep(uq, &uc->uc_lock, PCATCH, wmesg, timo);
1733 if (error == EWOULDBLOCK)
1735 if (uq->uq_flags & UQF_UMTXQ) {
1739 mtx_lock_spin(&umtx_lock);
1740 uq->uq_pi_blocked = NULL;
1742 td->td_flags &= ~TDF_UPIBLOCKED;
1744 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
1745 umtx_repropagate_priority(pi);
1746 mtx_unlock_spin(&umtx_lock);
1747 umtxq_unlock(&uq->uq_key);
1753 * Add reference count for a PI mutex.
1756 umtx_pi_ref(struct umtx_pi *pi)
1758 struct umtxq_chain *uc;
1760 uc = umtxq_getchain(&pi->pi_key);
1761 UMTXQ_LOCKED_ASSERT(uc);
1766 * Decrease reference count for a PI mutex, if the counter
1767 * is decreased to zero, its memory space is freed.
1770 umtx_pi_unref(struct umtx_pi *pi)
1772 struct umtxq_chain *uc;
1774 uc = umtxq_getchain(&pi->pi_key);
1775 UMTXQ_LOCKED_ASSERT(uc);
1776 KASSERT(pi->pi_refcount > 0, ("invalid reference count"));
1777 if (--pi->pi_refcount == 0) {
1778 mtx_lock_spin(&umtx_lock);
1779 if (pi->pi_owner != NULL) {
1780 TAILQ_REMOVE(&pi->pi_owner->td_umtxq->uq_pi_contested,
1782 pi->pi_owner = NULL;
1784 KASSERT(TAILQ_EMPTY(&pi->pi_blocked),
1785 ("blocked queue not empty"));
1786 mtx_unlock_spin(&umtx_lock);
1787 TAILQ_REMOVE(&uc->uc_pi_list, pi, pi_hashlink);
1793 * Find a PI mutex in hash table.
1795 static struct umtx_pi *
1796 umtx_pi_lookup(struct umtx_key *key)
1798 struct umtxq_chain *uc;
1801 uc = umtxq_getchain(key);
1802 UMTXQ_LOCKED_ASSERT(uc);
1804 TAILQ_FOREACH(pi, &uc->uc_pi_list, pi_hashlink) {
1805 if (umtx_key_match(&pi->pi_key, key)) {
1813 * Insert a PI mutex into hash table.
1816 umtx_pi_insert(struct umtx_pi *pi)
1818 struct umtxq_chain *uc;
1820 uc = umtxq_getchain(&pi->pi_key);
1821 UMTXQ_LOCKED_ASSERT(uc);
1822 TAILQ_INSERT_TAIL(&uc->uc_pi_list, pi, pi_hashlink);
1829 _do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, int timo,
1833 struct umtx_pi *pi, *new_pi;
1834 uint32_t id, owner, old;
1840 if ((error = umtx_key_get(m, TYPE_PI_UMUTEX, GET_SHARE(flags),
1843 umtxq_lock(&uq->uq_key);
1844 pi = umtx_pi_lookup(&uq->uq_key);
1846 new_pi = umtx_pi_alloc(M_NOWAIT);
1847 if (new_pi == NULL) {
1848 umtxq_unlock(&uq->uq_key);
1849 new_pi = umtx_pi_alloc(M_WAITOK);
1850 umtxq_lock(&uq->uq_key);
1851 pi = umtx_pi_lookup(&uq->uq_key);
1853 umtx_pi_free(new_pi);
1857 if (new_pi != NULL) {
1858 new_pi->pi_key = uq->uq_key;
1859 umtx_pi_insert(new_pi);
1864 umtxq_unlock(&uq->uq_key);
1867 * Care must be exercised when dealing with umtx structure. It
1868 * can fault on any access.
1872 * Try the uncontested case. This should be done in userland.
1874 owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id);
1876 /* The acquire succeeded. */
1877 if (owner == UMUTEX_UNOWNED) {
1882 /* The address was invalid. */
1888 /* If no one owns it but it is contested try to acquire it. */
1889 if (owner == UMUTEX_CONTESTED) {
1890 owner = casuword32(&m->m_owner,
1891 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
1893 if (owner == UMUTEX_CONTESTED) {
1894 umtxq_lock(&uq->uq_key);
1895 umtxq_busy(&uq->uq_key);
1896 error = umtx_pi_claim(pi, td);
1897 umtxq_unbusy(&uq->uq_key);
1898 umtxq_unlock(&uq->uq_key);
1902 /* The address was invalid. */
1908 error = umtxq_check_susp(td);
1912 /* If this failed the lock has changed, restart. */
1916 if ((flags & UMUTEX_ERROR_CHECK) != 0 &&
1917 (owner & ~UMUTEX_CONTESTED) == id) {
1928 * If we caught a signal, we have retried and now
1934 umtxq_lock(&uq->uq_key);
1935 umtxq_busy(&uq->uq_key);
1936 umtxq_unlock(&uq->uq_key);
1939 * Set the contested bit so that a release in user space
1940 * knows to use the system call for unlock. If this fails
1941 * either some one else has acquired the lock or it has been
1944 old = casuword32(&m->m_owner, owner, owner | UMUTEX_CONTESTED);
1946 /* The address was invalid. */
1948 umtxq_lock(&uq->uq_key);
1949 umtxq_unbusy(&uq->uq_key);
1950 umtxq_unlock(&uq->uq_key);
1955 umtxq_lock(&uq->uq_key);
1957 * We set the contested bit, sleep. Otherwise the lock changed
1958 * and we need to retry or we lost a race to the thread
1959 * unlocking the umtx.
1962 error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED,
1965 umtxq_unbusy(&uq->uq_key);
1966 umtxq_unlock(&uq->uq_key);
1969 error = umtxq_check_susp(td);
1974 umtxq_lock(&uq->uq_key);
1976 umtxq_unlock(&uq->uq_key);
1978 umtx_key_release(&uq->uq_key);
1983 * Unlock a PI mutex.
1986 do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags)
1988 struct umtx_key key;
1989 struct umtx_q *uq_first, *uq_first2, *uq_me;
1990 struct umtx_pi *pi, *pi2;
1991 uint32_t owner, old, id;
1998 * Make sure we own this mtx.
2000 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
2004 if ((owner & ~UMUTEX_CONTESTED) != id)
2007 /* This should be done in userland */
2008 if ((owner & UMUTEX_CONTESTED) == 0) {
2009 old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED);
2017 /* We should only ever be in here for contested locks */
2018 if ((error = umtx_key_get(m, TYPE_PI_UMUTEX, GET_SHARE(flags),
2024 count = umtxq_count_pi(&key, &uq_first);
2025 if (uq_first != NULL) {
2026 mtx_lock_spin(&umtx_lock);
2027 pi = uq_first->uq_pi_blocked;
2028 KASSERT(pi != NULL, ("pi == NULL?"));
2029 if (pi->pi_owner != curthread) {
2030 mtx_unlock_spin(&umtx_lock);
2033 umtx_key_release(&key);
2034 /* userland messed the mutex */
2037 uq_me = curthread->td_umtxq;
2038 pi->pi_owner = NULL;
2039 TAILQ_REMOVE(&uq_me->uq_pi_contested, pi, pi_link);
2040 /* get highest priority thread which is still sleeping. */
2041 uq_first = TAILQ_FIRST(&pi->pi_blocked);
2042 while (uq_first != NULL &&
2043 (uq_first->uq_flags & UQF_UMTXQ) == 0) {
2044 uq_first = TAILQ_NEXT(uq_first, uq_lockq);
2047 TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) {
2048 uq_first2 = TAILQ_FIRST(&pi2->pi_blocked);
2049 if (uq_first2 != NULL) {
2050 if (pri > UPRI(uq_first2->uq_thread))
2051 pri = UPRI(uq_first2->uq_thread);
2054 thread_lock(curthread);
2055 sched_lend_user_prio(curthread, pri);
2056 thread_unlock(curthread);
2057 mtx_unlock_spin(&umtx_lock);
2059 umtxq_signal_thread(uq_first);
2064 * When unlocking the umtx, it must be marked as unowned if
2065 * there is zero or one thread only waiting for it.
2066 * Otherwise, it must be marked as contested.
2068 old = casuword32(&m->m_owner, owner,
2069 count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
2074 umtx_key_release(&key);
2086 _do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, int timo,
2089 struct umtx_q *uq, *uq2;
2093 int error, pri, old_inherited_pri, su;
2097 if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags),
2100 su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0);
2102 old_inherited_pri = uq->uq_inherited_pri;
2103 umtxq_lock(&uq->uq_key);
2104 umtxq_busy(&uq->uq_key);
2105 umtxq_unlock(&uq->uq_key);
2107 ceiling = RTP_PRIO_MAX - fuword32(&m->m_ceilings[0]);
2108 if (ceiling > RTP_PRIO_MAX) {
2113 mtx_lock_spin(&umtx_lock);
2114 if (UPRI(td) < PRI_MIN_REALTIME + ceiling) {
2115 mtx_unlock_spin(&umtx_lock);
2119 if (su && PRI_MIN_REALTIME + ceiling < uq->uq_inherited_pri) {
2120 uq->uq_inherited_pri = PRI_MIN_REALTIME + ceiling;
2122 if (uq->uq_inherited_pri < UPRI(td))
2123 sched_lend_user_prio(td, uq->uq_inherited_pri);
2126 mtx_unlock_spin(&umtx_lock);
2128 owner = casuword32(&m->m_owner,
2129 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
2131 if (owner == UMUTEX_CONTESTED) {
2136 /* The address was invalid. */
2142 if ((flags & UMUTEX_ERROR_CHECK) != 0 &&
2143 (owner & ~UMUTEX_CONTESTED) == id) {
2154 * If we caught a signal, we have retried and now
2160 umtxq_lock(&uq->uq_key);
2162 umtxq_unbusy(&uq->uq_key);
2163 error = umtxq_sleep(uq, "umtxpp", timo);
2165 umtxq_unlock(&uq->uq_key);
2167 mtx_lock_spin(&umtx_lock);
2168 uq->uq_inherited_pri = old_inherited_pri;
2170 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2171 uq2 = TAILQ_FIRST(&pi->pi_blocked);
2173 if (pri > UPRI(uq2->uq_thread))
2174 pri = UPRI(uq2->uq_thread);
2177 if (pri > uq->uq_inherited_pri)
2178 pri = uq->uq_inherited_pri;
2180 sched_lend_user_prio(td, pri);
2182 mtx_unlock_spin(&umtx_lock);
2186 mtx_lock_spin(&umtx_lock);
2187 uq->uq_inherited_pri = old_inherited_pri;
2189 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2190 uq2 = TAILQ_FIRST(&pi->pi_blocked);
2192 if (pri > UPRI(uq2->uq_thread))
2193 pri = UPRI(uq2->uq_thread);
2196 if (pri > uq->uq_inherited_pri)
2197 pri = uq->uq_inherited_pri;
2199 sched_lend_user_prio(td, pri);
2201 mtx_unlock_spin(&umtx_lock);
2205 umtxq_lock(&uq->uq_key);
2206 umtxq_unbusy(&uq->uq_key);
2207 umtxq_unlock(&uq->uq_key);
2208 umtx_key_release(&uq->uq_key);
2213 * Unlock a PP mutex.
2216 do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags)
2218 struct umtx_key key;
2219 struct umtx_q *uq, *uq2;
2223 int error, pri, new_inherited_pri, su;
2227 su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0);
2230 * Make sure we own this mtx.
2232 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
2236 if ((owner & ~UMUTEX_CONTESTED) != id)
2239 error = copyin(&m->m_ceilings[1], &rceiling, sizeof(uint32_t));
2244 new_inherited_pri = PRI_MAX;
2246 rceiling = RTP_PRIO_MAX - rceiling;
2247 if (rceiling > RTP_PRIO_MAX)
2249 new_inherited_pri = PRI_MIN_REALTIME + rceiling;
2252 if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags),
2259 * For priority protected mutex, always set unlocked state
2260 * to UMUTEX_CONTESTED, so that userland always enters kernel
2261 * to lock the mutex, it is necessary because thread priority
2262 * has to be adjusted for such mutex.
2264 error = suword32(__DEVOLATILE(uint32_t *, &m->m_owner),
2269 umtxq_signal(&key, 1);
2276 mtx_lock_spin(&umtx_lock);
2278 uq->uq_inherited_pri = new_inherited_pri;
2280 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2281 uq2 = TAILQ_FIRST(&pi->pi_blocked);
2283 if (pri > UPRI(uq2->uq_thread))
2284 pri = UPRI(uq2->uq_thread);
2287 if (pri > uq->uq_inherited_pri)
2288 pri = uq->uq_inherited_pri;
2290 sched_lend_user_prio(td, pri);
2292 mtx_unlock_spin(&umtx_lock);
2294 umtx_key_release(&key);
2299 do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling,
2300 uint32_t *old_ceiling)
2303 uint32_t save_ceiling;
2308 flags = fuword32(&m->m_flags);
2309 if ((flags & UMUTEX_PRIO_PROTECT) == 0)
2311 if (ceiling > RTP_PRIO_MAX)
2315 if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags),
2319 umtxq_lock(&uq->uq_key);
2320 umtxq_busy(&uq->uq_key);
2321 umtxq_unlock(&uq->uq_key);
2323 save_ceiling = fuword32(&m->m_ceilings[0]);
2325 owner = casuword32(&m->m_owner,
2326 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
2328 if (owner == UMUTEX_CONTESTED) {
2329 suword32(&m->m_ceilings[0], ceiling);
2330 suword32(__DEVOLATILE(uint32_t *, &m->m_owner),
2336 /* The address was invalid. */
2342 if ((owner & ~UMUTEX_CONTESTED) == id) {
2343 suword32(&m->m_ceilings[0], ceiling);
2349 * If we caught a signal, we have retried and now
2356 * We set the contested bit, sleep. Otherwise the lock changed
2357 * and we need to retry or we lost a race to the thread
2358 * unlocking the umtx.
2360 umtxq_lock(&uq->uq_key);
2362 umtxq_unbusy(&uq->uq_key);
2363 error = umtxq_sleep(uq, "umtxpp", 0);
2365 umtxq_unlock(&uq->uq_key);
2367 umtxq_lock(&uq->uq_key);
2369 umtxq_signal(&uq->uq_key, INT_MAX);
2370 umtxq_unbusy(&uq->uq_key);
2371 umtxq_unlock(&uq->uq_key);
2372 umtx_key_release(&uq->uq_key);
2373 if (error == 0 && old_ceiling != NULL)
2374 suword32(old_ceiling, save_ceiling);
2379 _do_lock_umutex(struct thread *td, struct umutex *m, int flags, int timo,
2382 switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) {
2384 return (_do_lock_normal(td, m, flags, timo, mode));
2385 case UMUTEX_PRIO_INHERIT:
2386 return (_do_lock_pi(td, m, flags, timo, mode));
2387 case UMUTEX_PRIO_PROTECT:
2388 return (_do_lock_pp(td, m, flags, timo, mode));
2394 * Lock a userland POSIX mutex.
2397 do_lock_umutex(struct thread *td, struct umutex *m,
2398 struct timespec *timeout, int mode)
2400 struct timespec ts, ts2, ts3;
2405 flags = fuword32(&m->m_flags);
2409 if (timeout == NULL) {
2410 error = _do_lock_umutex(td, m, flags, 0, mode);
2411 /* Mutex locking is restarted if it is interrupted. */
2412 if (error == EINTR && mode != _UMUTEX_WAIT)
2416 timespecadd(&ts, timeout);
2417 TIMESPEC_TO_TIMEVAL(&tv, timeout);
2419 error = _do_lock_umutex(td, m, flags, tvtohz(&tv), mode);
2420 if (error != ETIMEDOUT)
2422 getnanouptime(&ts2);
2423 if (timespeccmp(&ts2, &ts, >=)) {
2428 timespecsub(&ts3, &ts2);
2429 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
2431 /* Timed-locking is not restarted. */
2432 if (error == ERESTART)
2439 * Unlock a userland POSIX mutex.
2442 do_unlock_umutex(struct thread *td, struct umutex *m)
2446 flags = fuword32(&m->m_flags);
2450 switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) {
2452 return (do_unlock_normal(td, m, flags));
2453 case UMUTEX_PRIO_INHERIT:
2454 return (do_unlock_pi(td, m, flags));
2455 case UMUTEX_PRIO_PROTECT:
2456 return (do_unlock_pp(td, m, flags));
2463 do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m,
2464 struct timespec *timeout, u_long wflags)
2468 struct timespec cts, ets, tts;
2474 flags = fuword32(&cv->c_flags);
2475 error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key);
2479 if ((wflags & CVWAIT_CLOCKID) != 0) {
2480 clockid = fuword32(&cv->c_clockid);
2481 if (clockid < CLOCK_REALTIME ||
2482 clockid >= CLOCK_THREAD_CPUTIME_ID) {
2483 /* hmm, only HW clock id will work. */
2487 clockid = CLOCK_REALTIME;
2490 umtxq_lock(&uq->uq_key);
2491 umtxq_busy(&uq->uq_key);
2493 umtxq_unlock(&uq->uq_key);
2496 * Set c_has_waiters to 1 before releasing user mutex, also
2497 * don't modify cache line when unnecessary.
2499 if (fuword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters)) == 0)
2500 suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 1);
2502 umtxq_lock(&uq->uq_key);
2503 umtxq_unbusy(&uq->uq_key);
2504 umtxq_unlock(&uq->uq_key);
2506 error = do_unlock_umutex(td, m);
2508 umtxq_lock(&uq->uq_key);
2510 if (timeout == NULL) {
2511 error = umtxq_sleep(uq, "ucond", 0);
2513 if ((wflags & CVWAIT_ABSTIME) == 0) {
2514 kern_clock_gettime(td, clockid, &ets);
2515 timespecadd(&ets, timeout);
2517 } else { /* absolute time */
2520 kern_clock_gettime(td, clockid, &cts);
2521 timespecsub(&tts, &cts);
2523 TIMESPEC_TO_TIMEVAL(&tv, &tts);
2525 error = umtxq_sleep(uq, "ucond", tvtohz(&tv));
2526 if (error != ETIMEDOUT)
2528 kern_clock_gettime(td, clockid, &cts);
2529 if (timespeccmp(&cts, &ets, >=)) {
2534 timespecsub(&tts, &cts);
2535 TIMESPEC_TO_TIMEVAL(&tv, &tts);
2540 if ((uq->uq_flags & UQF_UMTXQ) == 0)
2544 * This must be timeout,interrupted by signal or
2545 * surprious wakeup, clear c_has_waiter flag when
2548 umtxq_busy(&uq->uq_key);
2549 if ((uq->uq_flags & UQF_UMTXQ) != 0) {
2550 int oldlen = uq->uq_cur_queue->length;
2553 umtxq_unlock(&uq->uq_key);
2555 __DEVOLATILE(uint32_t *,
2556 &cv->c_has_waiters), 0);
2557 umtxq_lock(&uq->uq_key);
2560 umtxq_unbusy(&uq->uq_key);
2561 if (error == ERESTART)
2565 umtxq_unlock(&uq->uq_key);
2566 umtx_key_release(&uq->uq_key);
2571 * Signal a userland condition variable.
2574 do_cv_signal(struct thread *td, struct ucond *cv)
2576 struct umtx_key key;
2577 int error, cnt, nwake;
2580 flags = fuword32(&cv->c_flags);
2581 if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0)
2585 cnt = umtxq_count(&key);
2586 nwake = umtxq_signal(&key, 1);
2590 __DEVOLATILE(uint32_t *, &cv->c_has_waiters), 0);
2595 umtx_key_release(&key);
2600 do_cv_broadcast(struct thread *td, struct ucond *cv)
2602 struct umtx_key key;
2606 flags = fuword32(&cv->c_flags);
2607 if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0)
2612 umtxq_signal(&key, INT_MAX);
2615 error = suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 0);
2621 umtx_key_release(&key);
2626 do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag, int timo)
2629 uint32_t flags, wrflags;
2630 int32_t state, oldstate;
2631 int32_t blocked_readers;
2635 flags = fuword32(&rwlock->rw_flags);
2636 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
2640 wrflags = URWLOCK_WRITE_OWNER;
2641 if (!(fflag & URWLOCK_PREFER_READER) && !(flags & URWLOCK_PREFER_READER))
2642 wrflags |= URWLOCK_WRITE_WAITERS;
2645 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2646 /* try to lock it */
2647 while (!(state & wrflags)) {
2648 if (__predict_false(URWLOCK_READER_COUNT(state) == URWLOCK_MAX_READERS)) {
2649 umtx_key_release(&uq->uq_key);
2652 oldstate = casuword32(&rwlock->rw_state, state, state + 1);
2653 if (oldstate == -1) {
2654 umtx_key_release(&uq->uq_key);
2657 if (oldstate == state) {
2658 umtx_key_release(&uq->uq_key);
2661 error = umtxq_check_susp(td);
2670 /* grab monitor lock */
2671 umtxq_lock(&uq->uq_key);
2672 umtxq_busy(&uq->uq_key);
2673 umtxq_unlock(&uq->uq_key);
2676 * re-read the state, in case it changed between the try-lock above
2677 * and the check below
2679 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2681 /* set read contention bit */
2682 while ((state & wrflags) && !(state & URWLOCK_READ_WAITERS)) {
2683 oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_READ_WAITERS);
2684 if (oldstate == -1) {
2688 if (oldstate == state)
2691 error = umtxq_check_susp(td);
2696 umtxq_lock(&uq->uq_key);
2697 umtxq_unbusy(&uq->uq_key);
2698 umtxq_unlock(&uq->uq_key);
2702 /* state is changed while setting flags, restart */
2703 if (!(state & wrflags)) {
2704 umtxq_lock(&uq->uq_key);
2705 umtxq_unbusy(&uq->uq_key);
2706 umtxq_unlock(&uq->uq_key);
2707 error = umtxq_check_susp(td);
2714 /* contention bit is set, before sleeping, increase read waiter count */
2715 blocked_readers = fuword32(&rwlock->rw_blocked_readers);
2716 suword32(&rwlock->rw_blocked_readers, blocked_readers+1);
2718 while (state & wrflags) {
2719 umtxq_lock(&uq->uq_key);
2721 umtxq_unbusy(&uq->uq_key);
2723 error = umtxq_sleep(uq, "urdlck", timo);
2725 umtxq_busy(&uq->uq_key);
2727 umtxq_unlock(&uq->uq_key);
2730 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2733 /* decrease read waiter count, and may clear read contention bit */
2734 blocked_readers = fuword32(&rwlock->rw_blocked_readers);
2735 suword32(&rwlock->rw_blocked_readers, blocked_readers-1);
2736 if (blocked_readers == 1) {
2737 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2739 oldstate = casuword32(&rwlock->rw_state, state,
2740 state & ~URWLOCK_READ_WAITERS);
2741 if (oldstate == -1) {
2745 if (oldstate == state)
2748 error = umtxq_check_susp(td);
2754 umtxq_lock(&uq->uq_key);
2755 umtxq_unbusy(&uq->uq_key);
2756 umtxq_unlock(&uq->uq_key);
2760 umtx_key_release(&uq->uq_key);
2765 do_rw_rdlock2(struct thread *td, void *obj, long val, struct timespec *timeout)
2767 struct timespec ts, ts2, ts3;
2772 timespecadd(&ts, timeout);
2773 TIMESPEC_TO_TIMEVAL(&tv, timeout);
2775 error = do_rw_rdlock(td, obj, val, tvtohz(&tv));
2776 if (error != ETIMEDOUT)
2778 getnanouptime(&ts2);
2779 if (timespeccmp(&ts2, &ts, >=)) {
2784 timespecsub(&ts3, &ts2);
2785 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
2787 if (error == ERESTART)
2793 do_rw_wrlock(struct thread *td, struct urwlock *rwlock, int timo)
2797 int32_t state, oldstate;
2798 int32_t blocked_writers;
2799 int32_t blocked_readers;
2803 flags = fuword32(&rwlock->rw_flags);
2804 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
2808 blocked_readers = 0;
2810 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2811 while (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) {
2812 oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_WRITE_OWNER);
2813 if (oldstate == -1) {
2814 umtx_key_release(&uq->uq_key);
2817 if (oldstate == state) {
2818 umtx_key_release(&uq->uq_key);
2822 error = umtxq_check_susp(td);
2828 if (!(state & (URWLOCK_WRITE_OWNER|URWLOCK_WRITE_WAITERS)) &&
2829 blocked_readers != 0) {
2830 umtxq_lock(&uq->uq_key);
2831 umtxq_busy(&uq->uq_key);
2832 umtxq_signal_queue(&uq->uq_key, INT_MAX, UMTX_SHARED_QUEUE);
2833 umtxq_unbusy(&uq->uq_key);
2834 umtxq_unlock(&uq->uq_key);
2840 /* grab monitor lock */
2841 umtxq_lock(&uq->uq_key);
2842 umtxq_busy(&uq->uq_key);
2843 umtxq_unlock(&uq->uq_key);
2846 * re-read the state, in case it changed between the try-lock above
2847 * and the check below
2849 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2851 while (((state & URWLOCK_WRITE_OWNER) || URWLOCK_READER_COUNT(state) != 0) &&
2852 (state & URWLOCK_WRITE_WAITERS) == 0) {
2853 oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_WRITE_WAITERS);
2854 if (oldstate == -1) {
2858 if (oldstate == state)
2861 error = umtxq_check_susp(td);
2866 umtxq_lock(&uq->uq_key);
2867 umtxq_unbusy(&uq->uq_key);
2868 umtxq_unlock(&uq->uq_key);
2872 if (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) {
2873 umtxq_lock(&uq->uq_key);
2874 umtxq_unbusy(&uq->uq_key);
2875 umtxq_unlock(&uq->uq_key);
2876 error = umtxq_check_susp(td);
2882 blocked_writers = fuword32(&rwlock->rw_blocked_writers);
2883 suword32(&rwlock->rw_blocked_writers, blocked_writers+1);
2885 while ((state & URWLOCK_WRITE_OWNER) || URWLOCK_READER_COUNT(state) != 0) {
2886 umtxq_lock(&uq->uq_key);
2887 umtxq_insert_queue(uq, UMTX_EXCLUSIVE_QUEUE);
2888 umtxq_unbusy(&uq->uq_key);
2890 error = umtxq_sleep(uq, "uwrlck", timo);
2892 umtxq_busy(&uq->uq_key);
2893 umtxq_remove_queue(uq, UMTX_EXCLUSIVE_QUEUE);
2894 umtxq_unlock(&uq->uq_key);
2897 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2900 blocked_writers = fuword32(&rwlock->rw_blocked_writers);
2901 suword32(&rwlock->rw_blocked_writers, blocked_writers-1);
2902 if (blocked_writers == 1) {
2903 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2905 oldstate = casuword32(&rwlock->rw_state, state,
2906 state & ~URWLOCK_WRITE_WAITERS);
2907 if (oldstate == -1) {
2911 if (oldstate == state)
2914 error = umtxq_check_susp(td);
2916 * We are leaving the URWLOCK_WRITE_WAITERS
2917 * behind, but this should not harm the
2923 blocked_readers = fuword32(&rwlock->rw_blocked_readers);
2925 blocked_readers = 0;
2927 umtxq_lock(&uq->uq_key);
2928 umtxq_unbusy(&uq->uq_key);
2929 umtxq_unlock(&uq->uq_key);
2932 umtx_key_release(&uq->uq_key);
2937 do_rw_wrlock2(struct thread *td, void *obj, struct timespec *timeout)
2939 struct timespec ts, ts2, ts3;
2944 timespecadd(&ts, timeout);
2945 TIMESPEC_TO_TIMEVAL(&tv, timeout);
2947 error = do_rw_wrlock(td, obj, tvtohz(&tv));
2948 if (error != ETIMEDOUT)
2950 getnanouptime(&ts2);
2951 if (timespeccmp(&ts2, &ts, >=)) {
2956 timespecsub(&ts3, &ts2);
2957 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
2959 if (error == ERESTART)
2965 do_rw_unlock(struct thread *td, struct urwlock *rwlock)
2969 int32_t state, oldstate;
2970 int error, q, count;
2973 flags = fuword32(&rwlock->rw_flags);
2974 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
2978 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2979 if (state & URWLOCK_WRITE_OWNER) {
2981 oldstate = casuword32(&rwlock->rw_state, state,
2982 state & ~URWLOCK_WRITE_OWNER);
2983 if (oldstate == -1) {
2987 if (oldstate != state) {
2989 if (!(oldstate & URWLOCK_WRITE_OWNER)) {
2993 error = umtxq_check_susp(td);
2999 } else if (URWLOCK_READER_COUNT(state) != 0) {
3001 oldstate = casuword32(&rwlock->rw_state, state,
3003 if (oldstate == -1) {
3007 if (oldstate != state) {
3009 if (URWLOCK_READER_COUNT(oldstate) == 0) {
3013 error = umtxq_check_susp(td);
3026 if (!(flags & URWLOCK_PREFER_READER)) {
3027 if (state & URWLOCK_WRITE_WAITERS) {
3029 q = UMTX_EXCLUSIVE_QUEUE;
3030 } else if (state & URWLOCK_READ_WAITERS) {
3032 q = UMTX_SHARED_QUEUE;
3035 if (state & URWLOCK_READ_WAITERS) {
3037 q = UMTX_SHARED_QUEUE;
3038 } else if (state & URWLOCK_WRITE_WAITERS) {
3040 q = UMTX_EXCLUSIVE_QUEUE;
3045 umtxq_lock(&uq->uq_key);
3046 umtxq_busy(&uq->uq_key);
3047 umtxq_signal_queue(&uq->uq_key, count, q);
3048 umtxq_unbusy(&uq->uq_key);
3049 umtxq_unlock(&uq->uq_key);
3052 umtx_key_release(&uq->uq_key);
3057 do_sem_wait(struct thread *td, struct _usem *sem, struct timespec *timeout)
3061 struct timespec cts, ets, tts;
3062 uint32_t flags, count;
3066 flags = fuword32(&sem->_flags);
3067 error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key);
3070 umtxq_lock(&uq->uq_key);
3071 umtxq_busy(&uq->uq_key);
3073 umtxq_unlock(&uq->uq_key);
3075 if (fuword32(__DEVOLATILE(uint32_t *, &sem->_has_waiters)) == 0)
3076 casuword32(__DEVOLATILE(uint32_t *, &sem->_has_waiters), 0, 1);
3078 count = fuword32(__DEVOLATILE(uint32_t *, &sem->_count));
3080 umtxq_lock(&uq->uq_key);
3081 umtxq_unbusy(&uq->uq_key);
3083 umtxq_unlock(&uq->uq_key);
3084 umtx_key_release(&uq->uq_key);
3088 umtxq_lock(&uq->uq_key);
3089 umtxq_unbusy(&uq->uq_key);
3090 umtxq_unlock(&uq->uq_key);
3092 umtxq_lock(&uq->uq_key);
3093 if (timeout == NULL) {
3094 error = umtxq_sleep(uq, "usem", 0);
3096 getnanouptime(&ets);
3097 timespecadd(&ets, timeout);
3098 TIMESPEC_TO_TIMEVAL(&tv, timeout);
3100 error = umtxq_sleep(uq, "usem", tvtohz(&tv));
3101 if (error != ETIMEDOUT)
3103 getnanouptime(&cts);
3104 if (timespeccmp(&cts, &ets, >=)) {
3109 timespecsub(&tts, &cts);
3110 TIMESPEC_TO_TIMEVAL(&tv, &tts);
3114 if ((uq->uq_flags & UQF_UMTXQ) == 0)
3118 /* A relative timeout cannot be restarted. */
3119 if (error == ERESTART && timeout != NULL)
3122 umtxq_unlock(&uq->uq_key);
3123 umtx_key_release(&uq->uq_key);
3128 * Signal a userland condition variable.
3131 do_sem_wake(struct thread *td, struct _usem *sem)
3133 struct umtx_key key;
3134 int error, cnt, nwake;
3137 flags = fuword32(&sem->_flags);
3138 if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0)
3142 cnt = umtxq_count(&key);
3143 nwake = umtxq_signal(&key, 1);
3147 __DEVOLATILE(uint32_t *, &sem->_has_waiters), 0);
3152 umtx_key_release(&key);
3157 sys__umtx_lock(struct thread *td, struct _umtx_lock_args *uap)
3158 /* struct umtx *umtx */
3160 return _do_lock_umtx(td, uap->umtx, td->td_tid, 0);
3164 sys__umtx_unlock(struct thread *td, struct _umtx_unlock_args *uap)
3165 /* struct umtx *umtx */
3167 return do_unlock_umtx(td, uap->umtx, td->td_tid);
3171 umtx_copyin_timeout(const void *addr, struct timespec *tsp)
3175 error = copyin(addr, tsp, sizeof(struct timespec));
3177 if (tsp->tv_sec < 0 ||
3178 tsp->tv_nsec >= 1000000000 ||
3186 __umtx_op_lock_umtx(struct thread *td, struct _umtx_op_args *uap)
3188 struct timespec *ts, timeout;
3191 /* Allow a null timespec (wait forever). */
3192 if (uap->uaddr2 == NULL)
3195 error = umtx_copyin_timeout(uap->uaddr2, &timeout);
3200 return (do_lock_umtx(td, uap->obj, uap->val, ts));
3204 __umtx_op_unlock_umtx(struct thread *td, struct _umtx_op_args *uap)
3206 return (do_unlock_umtx(td, uap->obj, uap->val));
3210 __umtx_op_wait(struct thread *td, struct _umtx_op_args *uap)
3212 struct timespec *ts, timeout;
3215 if (uap->uaddr2 == NULL)
3218 error = umtx_copyin_timeout(uap->uaddr2, &timeout);
3223 return do_wait(td, uap->obj, uap->val, ts, 0, 0);
3227 __umtx_op_wait_uint(struct thread *td, struct _umtx_op_args *uap)
3229 struct timespec *ts, timeout;
3232 if (uap->uaddr2 == NULL)
3235 error = umtx_copyin_timeout(uap->uaddr2, &timeout);
3240 return do_wait(td, uap->obj, uap->val, ts, 1, 0);
3244 __umtx_op_wait_uint_private(struct thread *td, struct _umtx_op_args *uap)
3246 struct timespec *ts, timeout;
3249 if (uap->uaddr2 == NULL)
3252 error = umtx_copyin_timeout(uap->uaddr2, &timeout);
3257 return do_wait(td, uap->obj, uap->val, ts, 1, 1);
3261 __umtx_op_wake(struct thread *td, struct _umtx_op_args *uap)
3263 return (kern_umtx_wake(td, uap->obj, uap->val, 0));
3266 #define BATCH_SIZE 128
3268 __umtx_op_nwake_private(struct thread *td, struct _umtx_op_args *uap)
3270 int count = uap->val;
3271 void *uaddrs[BATCH_SIZE];
3272 char **upp = (char **)uap->obj;
3279 if (tocopy > BATCH_SIZE)
3280 tocopy = BATCH_SIZE;
3281 error = copyin(upp+pos, uaddrs, tocopy * sizeof(char *));
3284 for (i = 0; i < tocopy; ++i)
3285 kern_umtx_wake(td, uaddrs[i], INT_MAX, 1);
3293 __umtx_op_wake_private(struct thread *td, struct _umtx_op_args *uap)
3295 return (kern_umtx_wake(td, uap->obj, uap->val, 1));
3299 __umtx_op_lock_umutex(struct thread *td, struct _umtx_op_args *uap)
3301 struct timespec *ts, timeout;
3304 /* Allow a null timespec (wait forever). */
3305 if (uap->uaddr2 == NULL)
3308 error = umtx_copyin_timeout(uap->uaddr2, &timeout);
3313 return do_lock_umutex(td, uap->obj, ts, 0);
3317 __umtx_op_trylock_umutex(struct thread *td, struct _umtx_op_args *uap)
3319 return do_lock_umutex(td, uap->obj, NULL, _UMUTEX_TRY);
3323 __umtx_op_wait_umutex(struct thread *td, struct _umtx_op_args *uap)
3325 struct timespec *ts, timeout;
3328 /* Allow a null timespec (wait forever). */
3329 if (uap->uaddr2 == NULL)
3332 error = umtx_copyin_timeout(uap->uaddr2, &timeout);
3337 return do_lock_umutex(td, uap->obj, ts, _UMUTEX_WAIT);
3341 __umtx_op_wake_umutex(struct thread *td, struct _umtx_op_args *uap)
3343 return do_wake_umutex(td, uap->obj);
3347 __umtx_op_unlock_umutex(struct thread *td, struct _umtx_op_args *uap)
3349 return do_unlock_umutex(td, uap->obj);
3353 __umtx_op_set_ceiling(struct thread *td, struct _umtx_op_args *uap)
3355 return do_set_ceiling(td, uap->obj, uap->val, uap->uaddr1);
3359 __umtx_op_cv_wait(struct thread *td, struct _umtx_op_args *uap)
3361 struct timespec *ts, timeout;
3364 /* Allow a null timespec (wait forever). */
3365 if (uap->uaddr2 == NULL)
3368 error = umtx_copyin_timeout(uap->uaddr2, &timeout);
3373 return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val));
3377 __umtx_op_cv_signal(struct thread *td, struct _umtx_op_args *uap)
3379 return do_cv_signal(td, uap->obj);
3383 __umtx_op_cv_broadcast(struct thread *td, struct _umtx_op_args *uap)
3385 return do_cv_broadcast(td, uap->obj);
3389 __umtx_op_rw_rdlock(struct thread *td, struct _umtx_op_args *uap)
3391 struct timespec timeout;
3394 /* Allow a null timespec (wait forever). */
3395 if (uap->uaddr2 == NULL) {
3396 error = do_rw_rdlock(td, uap->obj, uap->val, 0);
3398 error = umtx_copyin_timeout(uap->uaddr2, &timeout);
3401 error = do_rw_rdlock2(td, uap->obj, uap->val, &timeout);
3407 __umtx_op_rw_wrlock(struct thread *td, struct _umtx_op_args *uap)
3409 struct timespec timeout;
3412 /* Allow a null timespec (wait forever). */
3413 if (uap->uaddr2 == NULL) {
3414 error = do_rw_wrlock(td, uap->obj, 0);
3416 error = umtx_copyin_timeout(uap->uaddr2, &timeout);
3420 error = do_rw_wrlock2(td, uap->obj, &timeout);
3426 __umtx_op_rw_unlock(struct thread *td, struct _umtx_op_args *uap)
3428 return do_rw_unlock(td, uap->obj);
3432 __umtx_op_sem_wait(struct thread *td, struct _umtx_op_args *uap)
3434 struct timespec *ts, timeout;
3437 /* Allow a null timespec (wait forever). */
3438 if (uap->uaddr2 == NULL)
3441 error = umtx_copyin_timeout(uap->uaddr2, &timeout);
3446 return (do_sem_wait(td, uap->obj, ts));
3450 __umtx_op_sem_wake(struct thread *td, struct _umtx_op_args *uap)
3452 return do_sem_wake(td, uap->obj);
3456 __umtx_op_wake2_umutex(struct thread *td, struct _umtx_op_args *uap)
3458 return do_wake2_umutex(td, uap->obj, uap->val);
3461 typedef int (*_umtx_op_func)(struct thread *td, struct _umtx_op_args *uap);
3463 static _umtx_op_func op_table[] = {
3464 __umtx_op_lock_umtx, /* UMTX_OP_LOCK */
3465 __umtx_op_unlock_umtx, /* UMTX_OP_UNLOCK */
3466 __umtx_op_wait, /* UMTX_OP_WAIT */
3467 __umtx_op_wake, /* UMTX_OP_WAKE */
3468 __umtx_op_trylock_umutex, /* UMTX_OP_MUTEX_TRYLOCK */
3469 __umtx_op_lock_umutex, /* UMTX_OP_MUTEX_LOCK */
3470 __umtx_op_unlock_umutex, /* UMTX_OP_MUTEX_UNLOCK */
3471 __umtx_op_set_ceiling, /* UMTX_OP_SET_CEILING */
3472 __umtx_op_cv_wait, /* UMTX_OP_CV_WAIT*/
3473 __umtx_op_cv_signal, /* UMTX_OP_CV_SIGNAL */
3474 __umtx_op_cv_broadcast, /* UMTX_OP_CV_BROADCAST */
3475 __umtx_op_wait_uint, /* UMTX_OP_WAIT_UINT */
3476 __umtx_op_rw_rdlock, /* UMTX_OP_RW_RDLOCK */
3477 __umtx_op_rw_wrlock, /* UMTX_OP_RW_WRLOCK */
3478 __umtx_op_rw_unlock, /* UMTX_OP_RW_UNLOCK */
3479 __umtx_op_wait_uint_private, /* UMTX_OP_WAIT_UINT_PRIVATE */
3480 __umtx_op_wake_private, /* UMTX_OP_WAKE_PRIVATE */
3481 __umtx_op_wait_umutex, /* UMTX_OP_UMUTEX_WAIT */
3482 __umtx_op_wake_umutex, /* UMTX_OP_UMUTEX_WAKE */
3483 __umtx_op_sem_wait, /* UMTX_OP_SEM_WAIT */
3484 __umtx_op_sem_wake, /* UMTX_OP_SEM_WAKE */
3485 __umtx_op_nwake_private, /* UMTX_OP_NWAKE_PRIVATE */
3486 __umtx_op_wake2_umutex /* UMTX_OP_UMUTEX_WAKE2 */
3490 sys__umtx_op(struct thread *td, struct _umtx_op_args *uap)
3492 if ((unsigned)uap->op < UMTX_OP_MAX)
3493 return (*op_table[uap->op])(td, uap);
3497 #ifdef COMPAT_FREEBSD32
3499 freebsd32_umtx_lock(struct thread *td, struct freebsd32_umtx_lock_args *uap)
3500 /* struct umtx *umtx */
3502 return (do_lock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid, NULL));
3506 freebsd32_umtx_unlock(struct thread *td, struct freebsd32_umtx_unlock_args *uap)
3507 /* struct umtx *umtx */
3509 return (do_unlock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid));
3518 umtx_copyin_timeout32(void *addr, struct timespec *tsp)
3520 struct timespec32 ts32;
3523 error = copyin(addr, &ts32, sizeof(struct timespec32));
3525 if (ts32.tv_sec < 0 ||
3526 ts32.tv_nsec >= 1000000000 ||
3530 tsp->tv_sec = ts32.tv_sec;
3531 tsp->tv_nsec = ts32.tv_nsec;
3538 __umtx_op_lock_umtx_compat32(struct thread *td, struct _umtx_op_args *uap)
3540 struct timespec *ts, timeout;
3543 /* Allow a null timespec (wait forever). */
3544 if (uap->uaddr2 == NULL)
3547 error = umtx_copyin_timeout32(uap->uaddr2, &timeout);
3552 return (do_lock_umtx32(td, uap->obj, uap->val, ts));
3556 __umtx_op_unlock_umtx_compat32(struct thread *td, struct _umtx_op_args *uap)
3558 return (do_unlock_umtx32(td, uap->obj, (uint32_t)uap->val));
3562 __umtx_op_wait_compat32(struct thread *td, struct _umtx_op_args *uap)
3564 struct timespec *ts, timeout;
3567 if (uap->uaddr2 == NULL)
3570 error = umtx_copyin_timeout32(uap->uaddr2, &timeout);
3575 return do_wait(td, uap->obj, uap->val, ts, 1, 0);
3579 __umtx_op_lock_umutex_compat32(struct thread *td, struct _umtx_op_args *uap)
3581 struct timespec *ts, timeout;
3584 /* Allow a null timespec (wait forever). */
3585 if (uap->uaddr2 == NULL)
3588 error = umtx_copyin_timeout32(uap->uaddr2, &timeout);
3593 return do_lock_umutex(td, uap->obj, ts, 0);
3597 __umtx_op_wait_umutex_compat32(struct thread *td, struct _umtx_op_args *uap)
3599 struct timespec *ts, timeout;
3602 /* Allow a null timespec (wait forever). */
3603 if (uap->uaddr2 == NULL)
3606 error = umtx_copyin_timeout32(uap->uaddr2, &timeout);
3611 return do_lock_umutex(td, uap->obj, ts, _UMUTEX_WAIT);
3615 __umtx_op_cv_wait_compat32(struct thread *td, struct _umtx_op_args *uap)
3617 struct timespec *ts, timeout;
3620 /* Allow a null timespec (wait forever). */
3621 if (uap->uaddr2 == NULL)
3624 error = umtx_copyin_timeout32(uap->uaddr2, &timeout);
3629 return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val));
3633 __umtx_op_rw_rdlock_compat32(struct thread *td, struct _umtx_op_args *uap)
3635 struct timespec timeout;
3638 /* Allow a null timespec (wait forever). */
3639 if (uap->uaddr2 == NULL) {
3640 error = do_rw_rdlock(td, uap->obj, uap->val, 0);
3642 error = umtx_copyin_timeout32(uap->uaddr2, &timeout);
3645 error = do_rw_rdlock2(td, uap->obj, uap->val, &timeout);
3651 __umtx_op_rw_wrlock_compat32(struct thread *td, struct _umtx_op_args *uap)
3653 struct timespec timeout;
3656 /* Allow a null timespec (wait forever). */
3657 if (uap->uaddr2 == NULL) {
3658 error = do_rw_wrlock(td, uap->obj, 0);
3660 error = umtx_copyin_timeout32(uap->uaddr2, &timeout);
3664 error = do_rw_wrlock2(td, uap->obj, &timeout);
3670 __umtx_op_wait_uint_private_compat32(struct thread *td, struct _umtx_op_args *uap)
3672 struct timespec *ts, timeout;
3675 if (uap->uaddr2 == NULL)
3678 error = umtx_copyin_timeout32(uap->uaddr2, &timeout);
3683 return do_wait(td, uap->obj, uap->val, ts, 1, 1);
3687 __umtx_op_sem_wait_compat32(struct thread *td, struct _umtx_op_args *uap)
3689 struct timespec *ts, timeout;
3692 /* Allow a null timespec (wait forever). */
3693 if (uap->uaddr2 == NULL)
3696 error = umtx_copyin_timeout32(uap->uaddr2, &timeout);
3701 return (do_sem_wait(td, uap->obj, ts));
3705 __umtx_op_nwake_private32(struct thread *td, struct _umtx_op_args *uap)
3707 int count = uap->val;
3708 uint32_t uaddrs[BATCH_SIZE];
3709 uint32_t **upp = (uint32_t **)uap->obj;
3716 if (tocopy > BATCH_SIZE)
3717 tocopy = BATCH_SIZE;
3718 error = copyin(upp+pos, uaddrs, tocopy * sizeof(uint32_t));
3721 for (i = 0; i < tocopy; ++i)
3722 kern_umtx_wake(td, (void *)(intptr_t)uaddrs[i],
3730 static _umtx_op_func op_table_compat32[] = {
3731 __umtx_op_lock_umtx_compat32, /* UMTX_OP_LOCK */
3732 __umtx_op_unlock_umtx_compat32, /* UMTX_OP_UNLOCK */
3733 __umtx_op_wait_compat32, /* UMTX_OP_WAIT */
3734 __umtx_op_wake, /* UMTX_OP_WAKE */
3735 __umtx_op_trylock_umutex, /* UMTX_OP_MUTEX_LOCK */
3736 __umtx_op_lock_umutex_compat32, /* UMTX_OP_MUTEX_TRYLOCK */
3737 __umtx_op_unlock_umutex, /* UMTX_OP_MUTEX_UNLOCK */
3738 __umtx_op_set_ceiling, /* UMTX_OP_SET_CEILING */
3739 __umtx_op_cv_wait_compat32, /* UMTX_OP_CV_WAIT*/
3740 __umtx_op_cv_signal, /* UMTX_OP_CV_SIGNAL */
3741 __umtx_op_cv_broadcast, /* UMTX_OP_CV_BROADCAST */
3742 __umtx_op_wait_compat32, /* UMTX_OP_WAIT_UINT */
3743 __umtx_op_rw_rdlock_compat32, /* UMTX_OP_RW_RDLOCK */
3744 __umtx_op_rw_wrlock_compat32, /* UMTX_OP_RW_WRLOCK */
3745 __umtx_op_rw_unlock, /* UMTX_OP_RW_UNLOCK */
3746 __umtx_op_wait_uint_private_compat32, /* UMTX_OP_WAIT_UINT_PRIVATE */
3747 __umtx_op_wake_private, /* UMTX_OP_WAKE_PRIVATE */
3748 __umtx_op_wait_umutex_compat32, /* UMTX_OP_UMUTEX_WAIT */
3749 __umtx_op_wake_umutex, /* UMTX_OP_UMUTEX_WAKE */
3750 __umtx_op_sem_wait_compat32, /* UMTX_OP_SEM_WAIT */
3751 __umtx_op_sem_wake, /* UMTX_OP_SEM_WAKE */
3752 __umtx_op_nwake_private32, /* UMTX_OP_NWAKE_PRIVATE */
3753 __umtx_op_wake2_umutex /* UMTX_OP_UMUTEX_WAKE2 */
3757 freebsd32_umtx_op(struct thread *td, struct freebsd32_umtx_op_args *uap)
3759 if ((unsigned)uap->op < UMTX_OP_MAX)
3760 return (*op_table_compat32[uap->op])(td,
3761 (struct _umtx_op_args *)uap);
3767 umtx_thread_init(struct thread *td)
3769 td->td_umtxq = umtxq_alloc();
3770 td->td_umtxq->uq_thread = td;
3774 umtx_thread_fini(struct thread *td)
3776 umtxq_free(td->td_umtxq);
3780 * It will be called when new thread is created, e.g fork().
3783 umtx_thread_alloc(struct thread *td)
3788 uq->uq_inherited_pri = PRI_MAX;
3790 KASSERT(uq->uq_flags == 0, ("uq_flags != 0"));
3791 KASSERT(uq->uq_thread == td, ("uq_thread != td"));
3792 KASSERT(uq->uq_pi_blocked == NULL, ("uq_pi_blocked != NULL"));
3793 KASSERT(TAILQ_EMPTY(&uq->uq_pi_contested), ("uq_pi_contested is not empty"));
3800 umtx_exec_hook(void *arg __unused, struct proc *p __unused,
3801 struct image_params *imgp __unused)
3803 umtx_thread_cleanup(curthread);
3807 * thread_exit() hook.
3810 umtx_thread_exit(struct thread *td)
3812 umtx_thread_cleanup(td);
3816 * clean up umtx data.
3819 umtx_thread_cleanup(struct thread *td)
3824 if ((uq = td->td_umtxq) == NULL)
3827 mtx_lock_spin(&umtx_lock);
3828 uq->uq_inherited_pri = PRI_MAX;
3829 while ((pi = TAILQ_FIRST(&uq->uq_pi_contested)) != NULL) {
3830 pi->pi_owner = NULL;
3831 TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link);
3833 mtx_unlock_spin(&umtx_lock);
3835 sched_lend_user_prio(td, PRI_MAX);