2 * Copyright (c) 2004, David Xu <davidxu@freebsd.org>
3 * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org>
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice unmodified, this list of conditions, and the following
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
31 #include "opt_compat.h"
32 #include <sys/param.h>
33 #include <sys/kernel.h>
34 #include <sys/limits.h>
36 #include <sys/malloc.h>
37 #include <sys/mutex.h>
40 #include <sys/sched.h>
42 #include <sys/sysctl.h>
43 #include <sys/sysent.h>
44 #include <sys/systm.h>
45 #include <sys/sysproto.h>
46 #include <sys/eventhandler.h>
50 #include <vm/vm_param.h>
52 #include <vm/vm_map.h>
53 #include <vm/vm_object.h>
55 #include <machine/cpu.h>
58 #include <compat/freebsd32/freebsd32_proto.h>
73 #define _UMUTEX_WAIT 2
75 /* Key to represent a unique userland synchronous object */
96 /* Priority inheritance mutex info. */
99 struct thread *pi_owner;
101 /* Reference count */
104 /* List entry to link umtx holding by thread */
105 TAILQ_ENTRY(umtx_pi) pi_link;
107 /* List entry in hash */
108 TAILQ_ENTRY(umtx_pi) pi_hashlink;
110 /* List for waiters */
111 TAILQ_HEAD(,umtx_q) pi_blocked;
113 /* Identify a userland lock object */
114 struct umtx_key pi_key;
117 /* A userland synchronous object user. */
119 /* Linked list for the hash. */
120 TAILQ_ENTRY(umtx_q) uq_link;
123 struct umtx_key uq_key;
127 #define UQF_UMTXQ 0x0001
129 /* The thread waits on. */
130 struct thread *uq_thread;
133 * Blocked on PI mutex. read can use chain lock
134 * or umtx_lock, write must have both chain lock and
135 * umtx_lock being hold.
137 struct umtx_pi *uq_pi_blocked;
139 /* On blocked list */
140 TAILQ_ENTRY(umtx_q) uq_lockq;
142 /* Thread contending with us */
143 TAILQ_HEAD(,umtx_pi) uq_pi_contested;
145 /* Inherited priority from PP mutex */
146 u_char uq_inherited_pri;
148 /* Spare queue ready to be reused */
149 struct umtxq_queue *uq_spare_queue;
151 /* The queue we on */
152 struct umtxq_queue *uq_cur_queue;
155 TAILQ_HEAD(umtxq_head, umtx_q);
157 /* Per-key wait-queue */
159 struct umtxq_head head;
161 LIST_ENTRY(umtxq_queue) link;
165 LIST_HEAD(umtxq_list, umtxq_queue);
167 /* Userland lock object's wait-queue chain */
169 /* Lock for this chain. */
172 /* List of sleep queues. */
173 struct umtxq_list uc_queue[2];
174 #define UMTX_SHARED_QUEUE 0
175 #define UMTX_EXCLUSIVE_QUEUE 1
177 LIST_HEAD(, umtxq_queue) uc_spare_queue;
182 /* Chain lock waiters */
185 /* All PI in the list */
186 TAILQ_HEAD(,umtx_pi) uc_pi_list;
190 #define UMTXQ_LOCKED_ASSERT(uc) mtx_assert(&(uc)->uc_lock, MA_OWNED)
191 #define UMTXQ_BUSY_ASSERT(uc) KASSERT(&(uc)->uc_busy, ("umtx chain is not busy"))
194 * Don't propagate time-sharing priority, there is a security reason,
195 * a user can simply introduce PI-mutex, let thread A lock the mutex,
196 * and let another thread B block on the mutex, because B is
197 * sleeping, its priority will be boosted, this causes A's priority to
198 * be boosted via priority propagating too and will never be lowered even
199 * if it is using 100%CPU, this is unfair to other processes.
202 #define UPRI(td) (((td)->td_user_pri >= PRI_MIN_TIMESHARE &&\
203 (td)->td_user_pri <= PRI_MAX_TIMESHARE) ?\
204 PRI_MAX_TIMESHARE : (td)->td_user_pri)
206 #define GOLDEN_RATIO_PRIME 2654404609U
207 #define UMTX_CHAINS 128
208 #define UMTX_SHIFTS (__WORD_BIT - 7)
210 #define THREAD_SHARE 0
211 #define PROCESS_SHARE 1
214 #define GET_SHARE(flags) \
215 (((flags) & USYNC_PROCESS_SHARED) == 0 ? THREAD_SHARE : PROCESS_SHARE)
217 #define BUSY_SPINS 200
219 static uma_zone_t umtx_pi_zone;
220 static struct umtxq_chain umtxq_chains[2][UMTX_CHAINS];
221 static MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory");
222 static int umtx_pi_allocated;
224 SYSCTL_NODE(_debug, OID_AUTO, umtx, CTLFLAG_RW, 0, "umtx debug");
225 SYSCTL_INT(_debug_umtx, OID_AUTO, umtx_pi_allocated, CTLFLAG_RD,
226 &umtx_pi_allocated, 0, "Allocated umtx_pi");
228 static void umtxq_sysinit(void *);
229 static void umtxq_hash(struct umtx_key *key);
230 static struct umtxq_chain *umtxq_getchain(struct umtx_key *key);
231 static void umtxq_lock(struct umtx_key *key);
232 static void umtxq_unlock(struct umtx_key *key);
233 static void umtxq_busy(struct umtx_key *key);
234 static void umtxq_unbusy(struct umtx_key *key);
235 static void umtxq_insert_queue(struct umtx_q *uq, int q);
236 static void umtxq_remove_queue(struct umtx_q *uq, int q);
237 static int umtxq_sleep(struct umtx_q *uq, const char *wmesg, int timo);
238 static int umtxq_count(struct umtx_key *key);
239 static int umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2);
240 static int umtx_key_get(void *addr, int type, int share,
241 struct umtx_key *key);
242 static void umtx_key_release(struct umtx_key *key);
243 static struct umtx_pi *umtx_pi_alloc(int);
244 static void umtx_pi_free(struct umtx_pi *pi);
245 static void umtx_pi_adjust_locked(struct thread *td, u_char oldpri);
246 static int do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags);
247 static void umtx_thread_cleanup(struct thread *td);
248 static void umtx_exec_hook(void *arg __unused, struct proc *p __unused,
249 struct image_params *imgp __unused);
250 SYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_sysinit, NULL);
252 #define umtxq_signal(key, nwake) umtxq_signal_queue((key), (nwake), UMTX_SHARED_QUEUE)
253 #define umtxq_insert(uq) umtxq_insert_queue((uq), UMTX_SHARED_QUEUE)
254 #define umtxq_remove(uq) umtxq_remove_queue((uq), UMTX_SHARED_QUEUE)
256 static struct mtx umtx_lock;
259 umtxq_sysinit(void *arg __unused)
263 umtx_pi_zone = uma_zcreate("umtx pi", sizeof(struct umtx_pi),
264 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
265 for (i = 0; i < 2; ++i) {
266 for (j = 0; j < UMTX_CHAINS; ++j) {
267 mtx_init(&umtxq_chains[i][j].uc_lock, "umtxql", NULL,
268 MTX_DEF | MTX_DUPOK);
269 LIST_INIT(&umtxq_chains[i][j].uc_queue[0]);
270 LIST_INIT(&umtxq_chains[i][j].uc_queue[1]);
271 LIST_INIT(&umtxq_chains[i][j].uc_spare_queue);
272 TAILQ_INIT(&umtxq_chains[i][j].uc_pi_list);
273 umtxq_chains[i][j].uc_busy = 0;
274 umtxq_chains[i][j].uc_waiters = 0;
277 mtx_init(&umtx_lock, "umtx lock", NULL, MTX_SPIN);
278 EVENTHANDLER_REGISTER(process_exec, umtx_exec_hook, NULL,
279 EVENTHANDLER_PRI_ANY);
287 uq = malloc(sizeof(struct umtx_q), M_UMTX, M_WAITOK | M_ZERO);
288 uq->uq_spare_queue = malloc(sizeof(struct umtxq_queue), M_UMTX, M_WAITOK | M_ZERO);
289 TAILQ_INIT(&uq->uq_spare_queue->head);
290 TAILQ_INIT(&uq->uq_pi_contested);
291 uq->uq_inherited_pri = PRI_MAX;
296 umtxq_free(struct umtx_q *uq)
298 MPASS(uq->uq_spare_queue != NULL);
299 free(uq->uq_spare_queue, M_UMTX);
304 umtxq_hash(struct umtx_key *key)
306 unsigned n = (uintptr_t)key->info.both.a + key->info.both.b;
307 key->hash = ((n * GOLDEN_RATIO_PRIME) >> UMTX_SHIFTS) % UMTX_CHAINS;
311 umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2)
313 return (k1->type == k2->type &&
314 k1->info.both.a == k2->info.both.a &&
315 k1->info.both.b == k2->info.both.b);
318 static inline struct umtxq_chain *
319 umtxq_getchain(struct umtx_key *key)
321 if (key->type <= TYPE_SEM)
322 return (&umtxq_chains[1][key->hash]);
323 return (&umtxq_chains[0][key->hash]);
330 umtxq_lock(struct umtx_key *key)
332 struct umtxq_chain *uc;
334 uc = umtxq_getchain(key);
335 mtx_lock(&uc->uc_lock);
342 umtxq_unlock(struct umtx_key *key)
344 struct umtxq_chain *uc;
346 uc = umtxq_getchain(key);
347 mtx_unlock(&uc->uc_lock);
351 * Set chain to busy state when following operation
352 * may be blocked (kernel mutex can not be used).
355 umtxq_busy(struct umtx_key *key)
357 struct umtxq_chain *uc;
359 uc = umtxq_getchain(key);
360 mtx_assert(&uc->uc_lock, MA_OWNED);
364 int count = BUSY_SPINS;
367 while (uc->uc_busy && --count > 0)
373 while (uc->uc_busy) {
375 msleep(uc, &uc->uc_lock, 0, "umtxqb", 0);
386 umtxq_unbusy(struct umtx_key *key)
388 struct umtxq_chain *uc;
390 uc = umtxq_getchain(key);
391 mtx_assert(&uc->uc_lock, MA_OWNED);
392 KASSERT(uc->uc_busy != 0, ("not busy"));
398 static struct umtxq_queue *
399 umtxq_queue_lookup(struct umtx_key *key, int q)
401 struct umtxq_queue *uh;
402 struct umtxq_chain *uc;
404 uc = umtxq_getchain(key);
405 UMTXQ_LOCKED_ASSERT(uc);
406 LIST_FOREACH(uh, &uc->uc_queue[q], link) {
407 if (umtx_key_match(&uh->key, key))
415 umtxq_insert_queue(struct umtx_q *uq, int q)
417 struct umtxq_queue *uh;
418 struct umtxq_chain *uc;
420 uc = umtxq_getchain(&uq->uq_key);
421 UMTXQ_LOCKED_ASSERT(uc);
422 KASSERT((uq->uq_flags & UQF_UMTXQ) == 0, ("umtx_q is already on queue"));
423 uh = umtxq_queue_lookup(&uq->uq_key, q);
425 LIST_INSERT_HEAD(&uc->uc_spare_queue, uq->uq_spare_queue, link);
427 uh = uq->uq_spare_queue;
428 uh->key = uq->uq_key;
429 LIST_INSERT_HEAD(&uc->uc_queue[q], uh, link);
431 uq->uq_spare_queue = NULL;
433 TAILQ_INSERT_TAIL(&uh->head, uq, uq_link);
435 uq->uq_flags |= UQF_UMTXQ;
436 uq->uq_cur_queue = uh;
441 umtxq_remove_queue(struct umtx_q *uq, int q)
443 struct umtxq_chain *uc;
444 struct umtxq_queue *uh;
446 uc = umtxq_getchain(&uq->uq_key);
447 UMTXQ_LOCKED_ASSERT(uc);
448 if (uq->uq_flags & UQF_UMTXQ) {
449 uh = uq->uq_cur_queue;
450 TAILQ_REMOVE(&uh->head, uq, uq_link);
452 uq->uq_flags &= ~UQF_UMTXQ;
453 if (TAILQ_EMPTY(&uh->head)) {
454 KASSERT(uh->length == 0,
455 ("inconsistent umtxq_queue length"));
456 LIST_REMOVE(uh, link);
458 uh = LIST_FIRST(&uc->uc_spare_queue);
459 KASSERT(uh != NULL, ("uc_spare_queue is empty"));
460 LIST_REMOVE(uh, link);
462 uq->uq_spare_queue = uh;
463 uq->uq_cur_queue = NULL;
468 * Check if there are multiple waiters
471 umtxq_count(struct umtx_key *key)
473 struct umtxq_chain *uc;
474 struct umtxq_queue *uh;
476 uc = umtxq_getchain(key);
477 UMTXQ_LOCKED_ASSERT(uc);
478 uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE);
485 * Check if there are multiple PI waiters and returns first
489 umtxq_count_pi(struct umtx_key *key, struct umtx_q **first)
491 struct umtxq_chain *uc;
492 struct umtxq_queue *uh;
495 uc = umtxq_getchain(key);
496 UMTXQ_LOCKED_ASSERT(uc);
497 uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE);
499 *first = TAILQ_FIRST(&uh->head);
506 * Wake up threads waiting on an userland object.
510 umtxq_signal_queue(struct umtx_key *key, int n_wake, int q)
512 struct umtxq_chain *uc;
513 struct umtxq_queue *uh;
518 uc = umtxq_getchain(key);
519 UMTXQ_LOCKED_ASSERT(uc);
520 uh = umtxq_queue_lookup(key, q);
522 while ((uq = TAILQ_FIRST(&uh->head)) != NULL) {
523 umtxq_remove_queue(uq, q);
534 * Wake up specified thread.
537 umtxq_signal_thread(struct umtx_q *uq)
539 struct umtxq_chain *uc;
541 uc = umtxq_getchain(&uq->uq_key);
542 UMTXQ_LOCKED_ASSERT(uc);
548 * Put thread into sleep state, before sleeping, check if
549 * thread was removed from umtx queue.
552 umtxq_sleep(struct umtx_q *uq, const char *wmesg, int timo)
554 struct umtxq_chain *uc;
557 uc = umtxq_getchain(&uq->uq_key);
558 UMTXQ_LOCKED_ASSERT(uc);
559 if (!(uq->uq_flags & UQF_UMTXQ))
561 error = msleep(uq, &uc->uc_lock, PCATCH, wmesg, timo);
562 if (error == EWOULDBLOCK)
568 * Convert userspace address into unique logical address.
571 umtx_key_get(void *addr, int type, int share, struct umtx_key *key)
573 struct thread *td = curthread;
575 vm_map_entry_t entry;
581 if (share == THREAD_SHARE) {
583 key->info.private.vs = td->td_proc->p_vmspace;
584 key->info.private.addr = (uintptr_t)addr;
586 MPASS(share == PROCESS_SHARE || share == AUTO_SHARE);
587 map = &td->td_proc->p_vmspace->vm_map;
588 if (vm_map_lookup(&map, (vm_offset_t)addr, VM_PROT_WRITE,
589 &entry, &key->info.shared.object, &pindex, &prot,
590 &wired) != KERN_SUCCESS) {
594 if ((share == PROCESS_SHARE) ||
595 (share == AUTO_SHARE &&
596 VM_INHERIT_SHARE == entry->inheritance)) {
598 key->info.shared.offset = entry->offset + entry->start -
600 vm_object_reference(key->info.shared.object);
603 key->info.private.vs = td->td_proc->p_vmspace;
604 key->info.private.addr = (uintptr_t)addr;
606 vm_map_lookup_done(map, entry);
617 umtx_key_release(struct umtx_key *key)
620 vm_object_deallocate(key->info.shared.object);
624 * Lock a umtx object.
627 _do_lock_umtx(struct thread *td, struct umtx *umtx, u_long id, int timo)
637 * Care must be exercised when dealing with umtx structure. It
638 * can fault on any access.
642 * Try the uncontested case. This should be done in userland.
644 owner = casuword(&umtx->u_owner, UMTX_UNOWNED, id);
646 /* The acquire succeeded. */
647 if (owner == UMTX_UNOWNED)
650 /* The address was invalid. */
654 /* If no one owns it but it is contested try to acquire it. */
655 if (owner == UMTX_CONTESTED) {
656 owner = casuword(&umtx->u_owner,
657 UMTX_CONTESTED, id | UMTX_CONTESTED);
659 if (owner == UMTX_CONTESTED)
662 /* The address was invalid. */
666 /* If this failed the lock has changed, restart. */
671 * If we caught a signal, we have retried and now
677 if ((error = umtx_key_get(umtx, TYPE_SIMPLE_LOCK,
678 AUTO_SHARE, &uq->uq_key)) != 0)
681 umtxq_lock(&uq->uq_key);
682 umtxq_busy(&uq->uq_key);
684 umtxq_unbusy(&uq->uq_key);
685 umtxq_unlock(&uq->uq_key);
688 * Set the contested bit so that a release in user space
689 * knows to use the system call for unlock. If this fails
690 * either some one else has acquired the lock or it has been
693 old = casuword(&umtx->u_owner, owner, owner | UMTX_CONTESTED);
695 /* The address was invalid. */
697 umtxq_lock(&uq->uq_key);
699 umtxq_unlock(&uq->uq_key);
700 umtx_key_release(&uq->uq_key);
705 * We set the contested bit, sleep. Otherwise the lock changed
706 * and we need to retry or we lost a race to the thread
707 * unlocking the umtx.
709 umtxq_lock(&uq->uq_key);
711 error = umtxq_sleep(uq, "umtx", timo);
713 umtxq_unlock(&uq->uq_key);
714 umtx_key_release(&uq->uq_key);
721 * Lock a umtx object.
724 do_lock_umtx(struct thread *td, struct umtx *umtx, u_long id,
725 struct timespec *timeout)
727 struct timespec ts, ts2, ts3;
731 if (timeout == NULL) {
732 error = _do_lock_umtx(td, umtx, id, 0);
733 /* Mutex locking is restarted if it is interrupted. */
738 timespecadd(&ts, timeout);
739 TIMESPEC_TO_TIMEVAL(&tv, timeout);
741 error = _do_lock_umtx(td, umtx, id, tvtohz(&tv));
742 if (error != ETIMEDOUT)
745 if (timespeccmp(&ts2, &ts, >=)) {
750 timespecsub(&ts3, &ts2);
751 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
753 /* Timed-locking is not restarted. */
754 if (error == ERESTART)
761 * Unlock a umtx object.
764 do_unlock_umtx(struct thread *td, struct umtx *umtx, u_long id)
773 * Make sure we own this mtx.
775 owner = fuword(__DEVOLATILE(u_long *, &umtx->u_owner));
779 if ((owner & ~UMTX_CONTESTED) != id)
782 /* This should be done in userland */
783 if ((owner & UMTX_CONTESTED) == 0) {
784 old = casuword(&umtx->u_owner, owner, UMTX_UNOWNED);
792 /* We should only ever be in here for contested locks */
793 if ((error = umtx_key_get(umtx, TYPE_SIMPLE_LOCK, AUTO_SHARE,
799 count = umtxq_count(&key);
803 * When unlocking the umtx, it must be marked as unowned if
804 * there is zero or one thread only waiting for it.
805 * Otherwise, it must be marked as contested.
807 old = casuword(&umtx->u_owner, owner,
808 count <= 1 ? UMTX_UNOWNED : UMTX_CONTESTED);
810 umtxq_signal(&key,1);
813 umtx_key_release(&key);
824 * Lock a umtx object.
827 _do_lock_umtx32(struct thread *td, uint32_t *m, uint32_t id, int timo)
837 * Care must be exercised when dealing with umtx structure. It
838 * can fault on any access.
842 * Try the uncontested case. This should be done in userland.
844 owner = casuword32(m, UMUTEX_UNOWNED, id);
846 /* The acquire succeeded. */
847 if (owner == UMUTEX_UNOWNED)
850 /* The address was invalid. */
854 /* If no one owns it but it is contested try to acquire it. */
855 if (owner == UMUTEX_CONTESTED) {
856 owner = casuword32(m,
857 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
858 if (owner == UMUTEX_CONTESTED)
861 /* The address was invalid. */
865 /* If this failed the lock has changed, restart. */
870 * If we caught a signal, we have retried and now
876 if ((error = umtx_key_get(m, TYPE_SIMPLE_LOCK,
877 AUTO_SHARE, &uq->uq_key)) != 0)
880 umtxq_lock(&uq->uq_key);
881 umtxq_busy(&uq->uq_key);
883 umtxq_unbusy(&uq->uq_key);
884 umtxq_unlock(&uq->uq_key);
887 * Set the contested bit so that a release in user space
888 * knows to use the system call for unlock. If this fails
889 * either some one else has acquired the lock or it has been
892 old = casuword32(m, owner, owner | UMUTEX_CONTESTED);
894 /* The address was invalid. */
896 umtxq_lock(&uq->uq_key);
898 umtxq_unlock(&uq->uq_key);
899 umtx_key_release(&uq->uq_key);
904 * We set the contested bit, sleep. Otherwise the lock changed
905 * and we need to retry or we lost a race to the thread
906 * unlocking the umtx.
908 umtxq_lock(&uq->uq_key);
910 error = umtxq_sleep(uq, "umtx", timo);
912 umtxq_unlock(&uq->uq_key);
913 umtx_key_release(&uq->uq_key);
920 * Lock a umtx object.
923 do_lock_umtx32(struct thread *td, void *m, uint32_t id,
924 struct timespec *timeout)
926 struct timespec ts, ts2, ts3;
930 if (timeout == NULL) {
931 error = _do_lock_umtx32(td, m, id, 0);
932 /* Mutex locking is restarted if it is interrupted. */
937 timespecadd(&ts, timeout);
938 TIMESPEC_TO_TIMEVAL(&tv, timeout);
940 error = _do_lock_umtx32(td, m, id, tvtohz(&tv));
941 if (error != ETIMEDOUT)
944 if (timespeccmp(&ts2, &ts, >=)) {
949 timespecsub(&ts3, &ts2);
950 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
952 /* Timed-locking is not restarted. */
953 if (error == ERESTART)
960 * Unlock a umtx object.
963 do_unlock_umtx32(struct thread *td, uint32_t *m, uint32_t id)
972 * Make sure we own this mtx.
978 if ((owner & ~UMUTEX_CONTESTED) != id)
981 /* This should be done in userland */
982 if ((owner & UMUTEX_CONTESTED) == 0) {
983 old = casuword32(m, owner, UMUTEX_UNOWNED);
991 /* We should only ever be in here for contested locks */
992 if ((error = umtx_key_get(m, TYPE_SIMPLE_LOCK, AUTO_SHARE,
998 count = umtxq_count(&key);
1002 * When unlocking the umtx, it must be marked as unowned if
1003 * there is zero or one thread only waiting for it.
1004 * Otherwise, it must be marked as contested.
1006 old = casuword32(m, owner,
1007 count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
1009 umtxq_signal(&key,1);
1012 umtx_key_release(&key);
1022 * Fetch and compare value, sleep on the address if value is not changed.
1025 do_wait(struct thread *td, void *addr, u_long id,
1026 struct timespec *timeout, int compat32, int is_private)
1029 struct timespec ts, ts2, ts3;
1035 if ((error = umtx_key_get(addr, TYPE_SIMPLE_WAIT,
1036 is_private ? THREAD_SHARE : AUTO_SHARE, &uq->uq_key)) != 0)
1039 umtxq_lock(&uq->uq_key);
1041 umtxq_unlock(&uq->uq_key);
1045 tmp = (unsigned int)fuword32(addr);
1047 umtxq_lock(&uq->uq_key);
1049 umtxq_unlock(&uq->uq_key);
1050 } else if (timeout == NULL) {
1051 umtxq_lock(&uq->uq_key);
1052 error = umtxq_sleep(uq, "uwait", 0);
1054 umtxq_unlock(&uq->uq_key);
1057 timespecadd(&ts, timeout);
1058 TIMESPEC_TO_TIMEVAL(&tv, timeout);
1059 umtxq_lock(&uq->uq_key);
1061 error = umtxq_sleep(uq, "uwait", tvtohz(&tv));
1062 if (!(uq->uq_flags & UQF_UMTXQ))
1064 if (error != ETIMEDOUT)
1066 umtxq_unlock(&uq->uq_key);
1067 getnanouptime(&ts2);
1068 if (timespeccmp(&ts2, &ts, >=)) {
1070 umtxq_lock(&uq->uq_key);
1074 timespecsub(&ts3, &ts2);
1075 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
1076 umtxq_lock(&uq->uq_key);
1079 umtxq_unlock(&uq->uq_key);
1081 umtx_key_release(&uq->uq_key);
1082 if (error == ERESTART)
1088 * Wake up threads sleeping on the specified address.
1091 kern_umtx_wake(struct thread *td, void *uaddr, int n_wake, int is_private)
1093 struct umtx_key key;
1096 if ((ret = umtx_key_get(uaddr, TYPE_SIMPLE_WAIT,
1097 is_private ? THREAD_SHARE : AUTO_SHARE, &key)) != 0)
1100 ret = umtxq_signal(&key, n_wake);
1102 umtx_key_release(&key);
1107 * Lock PTHREAD_PRIO_NONE protocol POSIX mutex.
1110 _do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, int timo,
1114 uint32_t owner, old, id;
1121 * Care must be exercised when dealing with umtx structure. It
1122 * can fault on any access.
1125 owner = fuword32(__DEVOLATILE(void *, &m->m_owner));
1126 if (mode == _UMUTEX_WAIT) {
1127 if (owner == UMUTEX_UNOWNED || owner == UMUTEX_CONTESTED)
1131 * Try the uncontested case. This should be done in userland.
1133 owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id);
1135 /* The acquire succeeded. */
1136 if (owner == UMUTEX_UNOWNED)
1139 /* The address was invalid. */
1143 /* If no one owns it but it is contested try to acquire it. */
1144 if (owner == UMUTEX_CONTESTED) {
1145 owner = casuword32(&m->m_owner,
1146 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
1148 if (owner == UMUTEX_CONTESTED)
1151 /* The address was invalid. */
1155 /* If this failed the lock has changed, restart. */
1160 if ((flags & UMUTEX_ERROR_CHECK) != 0 &&
1161 (owner & ~UMUTEX_CONTESTED) == id)
1164 if (mode == _UMUTEX_TRY)
1168 * If we caught a signal, we have retried and now
1174 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX,
1175 GET_SHARE(flags), &uq->uq_key)) != 0)
1178 umtxq_lock(&uq->uq_key);
1179 umtxq_busy(&uq->uq_key);
1181 umtxq_unlock(&uq->uq_key);
1184 * Set the contested bit so that a release in user space
1185 * knows to use the system call for unlock. If this fails
1186 * either some one else has acquired the lock or it has been
1189 old = casuword32(&m->m_owner, owner, owner | UMUTEX_CONTESTED);
1191 /* The address was invalid. */
1193 umtxq_lock(&uq->uq_key);
1195 umtxq_unbusy(&uq->uq_key);
1196 umtxq_unlock(&uq->uq_key);
1197 umtx_key_release(&uq->uq_key);
1202 * We set the contested bit, sleep. Otherwise the lock changed
1203 * and we need to retry or we lost a race to the thread
1204 * unlocking the umtx.
1206 umtxq_lock(&uq->uq_key);
1207 umtxq_unbusy(&uq->uq_key);
1209 error = umtxq_sleep(uq, "umtxn", timo);
1211 umtxq_unlock(&uq->uq_key);
1212 umtx_key_release(&uq->uq_key);
1219 * Lock PTHREAD_PRIO_NONE protocol POSIX mutex.
1222 * Unlock PTHREAD_PRIO_NONE protocol POSIX mutex.
1225 do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags)
1227 struct umtx_key key;
1228 uint32_t owner, old, id;
1234 * Make sure we own this mtx.
1236 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
1240 if ((owner & ~UMUTEX_CONTESTED) != id)
1243 if ((owner & UMUTEX_CONTESTED) == 0) {
1244 old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED);
1252 /* We should only ever be in here for contested locks */
1253 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags),
1259 count = umtxq_count(&key);
1263 * When unlocking the umtx, it must be marked as unowned if
1264 * there is zero or one thread only waiting for it.
1265 * Otherwise, it must be marked as contested.
1267 old = casuword32(&m->m_owner, owner,
1268 count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
1270 umtxq_signal(&key,1);
1273 umtx_key_release(&key);
1282 * Check if the mutex is available and wake up a waiter,
1283 * only for simple mutex.
1286 do_wake_umutex(struct thread *td, struct umutex *m)
1288 struct umtx_key key;
1294 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
1298 if ((owner & ~UMUTEX_CONTESTED) != 0)
1301 flags = fuword32(&m->m_flags);
1303 /* We should only ever be in here for contested locks */
1304 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags),
1310 count = umtxq_count(&key);
1314 owner = casuword32(&m->m_owner, UMUTEX_CONTESTED, UMUTEX_UNOWNED);
1317 if (count != 0 && (owner & ~UMUTEX_CONTESTED) == 0)
1318 umtxq_signal(&key, 1);
1321 umtx_key_release(&key);
1325 static inline struct umtx_pi *
1326 umtx_pi_alloc(int flags)
1330 pi = uma_zalloc(umtx_pi_zone, M_ZERO | flags);
1331 TAILQ_INIT(&pi->pi_blocked);
1332 atomic_add_int(&umtx_pi_allocated, 1);
1337 umtx_pi_free(struct umtx_pi *pi)
1339 uma_zfree(umtx_pi_zone, pi);
1340 atomic_add_int(&umtx_pi_allocated, -1);
1344 * Adjust the thread's position on a pi_state after its priority has been
1348 umtx_pi_adjust_thread(struct umtx_pi *pi, struct thread *td)
1350 struct umtx_q *uq, *uq1, *uq2;
1353 mtx_assert(&umtx_lock, MA_OWNED);
1360 * Check if the thread needs to be moved on the blocked chain.
1361 * It needs to be moved if either its priority is lower than
1362 * the previous thread or higher than the next thread.
1364 uq1 = TAILQ_PREV(uq, umtxq_head, uq_lockq);
1365 uq2 = TAILQ_NEXT(uq, uq_lockq);
1366 if ((uq1 != NULL && UPRI(td) < UPRI(uq1->uq_thread)) ||
1367 (uq2 != NULL && UPRI(td) > UPRI(uq2->uq_thread))) {
1369 * Remove thread from blocked chain and determine where
1370 * it should be moved to.
1372 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
1373 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) {
1374 td1 = uq1->uq_thread;
1375 MPASS(td1->td_proc->p_magic == P_MAGIC);
1376 if (UPRI(td1) > UPRI(td))
1381 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq);
1383 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq);
1389 * Propagate priority when a thread is blocked on POSIX
1393 umtx_propagate_priority(struct thread *td)
1399 mtx_assert(&umtx_lock, MA_OWNED);
1402 pi = uq->uq_pi_blocked;
1411 MPASS(td->td_proc != NULL);
1412 MPASS(td->td_proc->p_magic == P_MAGIC);
1414 if (UPRI(td) <= pri)
1418 sched_lend_user_prio(td, pri);
1422 * Pick up the lock that td is blocked on.
1425 pi = uq->uq_pi_blocked;
1426 /* Resort td on the list if needed. */
1427 if (!umtx_pi_adjust_thread(pi, td))
1433 * Unpropagate priority for a PI mutex when a thread blocked on
1434 * it is interrupted by signal or resumed by others.
1437 umtx_unpropagate_priority(struct umtx_pi *pi)
1439 struct umtx_q *uq, *uq_owner;
1440 struct umtx_pi *pi2;
1443 mtx_assert(&umtx_lock, MA_OWNED);
1445 while (pi != NULL && pi->pi_owner != NULL) {
1447 uq_owner = pi->pi_owner->td_umtxq;
1449 TAILQ_FOREACH(pi2, &uq_owner->uq_pi_contested, pi_link) {
1450 uq = TAILQ_FIRST(&pi2->pi_blocked);
1452 if (pri > UPRI(uq->uq_thread))
1453 pri = UPRI(uq->uq_thread);
1457 if (pri > uq_owner->uq_inherited_pri)
1458 pri = uq_owner->uq_inherited_pri;
1459 thread_lock(pi->pi_owner);
1460 oldpri = pi->pi_owner->td_user_pri;
1461 sched_unlend_user_prio(pi->pi_owner, pri);
1462 thread_unlock(pi->pi_owner);
1463 if (uq_owner->uq_pi_blocked != NULL)
1464 umtx_pi_adjust_locked(pi->pi_owner, oldpri);
1465 pi = uq_owner->uq_pi_blocked;
1470 * Insert a PI mutex into owned list.
1473 umtx_pi_setowner(struct umtx_pi *pi, struct thread *owner)
1475 struct umtx_q *uq_owner;
1477 uq_owner = owner->td_umtxq;
1478 mtx_assert(&umtx_lock, MA_OWNED);
1479 if (pi->pi_owner != NULL)
1480 panic("pi_ower != NULL");
1481 pi->pi_owner = owner;
1482 TAILQ_INSERT_TAIL(&uq_owner->uq_pi_contested, pi, pi_link);
1486 * Claim ownership of a PI mutex.
1489 umtx_pi_claim(struct umtx_pi *pi, struct thread *owner)
1491 struct umtx_q *uq, *uq_owner;
1493 uq_owner = owner->td_umtxq;
1494 mtx_lock_spin(&umtx_lock);
1495 if (pi->pi_owner == owner) {
1496 mtx_unlock_spin(&umtx_lock);
1500 if (pi->pi_owner != NULL) {
1502 * userland may have already messed the mutex, sigh.
1504 mtx_unlock_spin(&umtx_lock);
1507 umtx_pi_setowner(pi, owner);
1508 uq = TAILQ_FIRST(&pi->pi_blocked);
1512 pri = UPRI(uq->uq_thread);
1514 if (pri < UPRI(owner))
1515 sched_lend_user_prio(owner, pri);
1516 thread_unlock(owner);
1518 mtx_unlock_spin(&umtx_lock);
1523 umtx_pi_adjust_locked(struct thread *td, u_char oldpri)
1530 * Pick up the lock that td is blocked on.
1532 pi = uq->uq_pi_blocked;
1535 /* Resort the turnstile on the list. */
1536 if (!umtx_pi_adjust_thread(pi, td))
1540 * If our priority was lowered and we are at the head of the
1541 * turnstile, then propagate our new priority up the chain.
1543 if (uq == TAILQ_FIRST(&pi->pi_blocked) && UPRI(td) < oldpri)
1544 umtx_propagate_priority(td);
1548 * Adjust a thread's order position in its blocked PI mutex,
1549 * this may result new priority propagating process.
1552 umtx_pi_adjust(struct thread *td, u_char oldpri)
1558 mtx_lock_spin(&umtx_lock);
1560 * Pick up the lock that td is blocked on.
1562 pi = uq->uq_pi_blocked;
1564 umtx_pi_adjust_locked(td, oldpri);
1565 mtx_unlock_spin(&umtx_lock);
1569 * Sleep on a PI mutex.
1572 umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi,
1573 uint32_t owner, const char *wmesg, int timo)
1575 struct umtxq_chain *uc;
1576 struct thread *td, *td1;
1582 KASSERT(td == curthread, ("inconsistent uq_thread"));
1583 uc = umtxq_getchain(&uq->uq_key);
1584 UMTXQ_LOCKED_ASSERT(uc);
1585 UMTXQ_BUSY_ASSERT(uc);
1587 mtx_lock_spin(&umtx_lock);
1588 if (pi->pi_owner == NULL) {
1590 * Current, We only support process private PI-mutex,
1591 * we need a faster way to find an owner thread for
1592 * process-shared mutex (not available yet).
1594 mtx_unlock_spin(&umtx_lock);
1596 td1 = thread_find(curproc, owner);
1597 mtx_lock_spin(&umtx_lock);
1598 if (td1 != NULL && pi->pi_owner == NULL) {
1599 uq1 = td1->td_umtxq;
1600 umtx_pi_setowner(pi, td1);
1602 PROC_UNLOCK(curproc);
1605 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) {
1606 pri = UPRI(uq1->uq_thread);
1612 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq);
1614 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq);
1616 uq->uq_pi_blocked = pi;
1618 td->td_flags |= TDF_UPIBLOCKED;
1620 umtx_propagate_priority(td);
1621 mtx_unlock_spin(&umtx_lock);
1622 umtxq_unbusy(&uq->uq_key);
1624 if (uq->uq_flags & UQF_UMTXQ) {
1625 error = msleep(uq, &uc->uc_lock, PCATCH, wmesg, timo);
1626 if (error == EWOULDBLOCK)
1628 if (uq->uq_flags & UQF_UMTXQ) {
1632 mtx_lock_spin(&umtx_lock);
1633 uq->uq_pi_blocked = NULL;
1635 td->td_flags &= ~TDF_UPIBLOCKED;
1637 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
1638 umtx_unpropagate_priority(pi);
1639 mtx_unlock_spin(&umtx_lock);
1640 umtxq_unlock(&uq->uq_key);
1646 * Add reference count for a PI mutex.
1649 umtx_pi_ref(struct umtx_pi *pi)
1651 struct umtxq_chain *uc;
1653 uc = umtxq_getchain(&pi->pi_key);
1654 UMTXQ_LOCKED_ASSERT(uc);
1659 * Decrease reference count for a PI mutex, if the counter
1660 * is decreased to zero, its memory space is freed.
1663 umtx_pi_unref(struct umtx_pi *pi)
1665 struct umtxq_chain *uc;
1667 uc = umtxq_getchain(&pi->pi_key);
1668 UMTXQ_LOCKED_ASSERT(uc);
1669 KASSERT(pi->pi_refcount > 0, ("invalid reference count"));
1670 if (--pi->pi_refcount == 0) {
1671 mtx_lock_spin(&umtx_lock);
1672 if (pi->pi_owner != NULL) {
1673 TAILQ_REMOVE(&pi->pi_owner->td_umtxq->uq_pi_contested,
1675 pi->pi_owner = NULL;
1677 KASSERT(TAILQ_EMPTY(&pi->pi_blocked),
1678 ("blocked queue not empty"));
1679 mtx_unlock_spin(&umtx_lock);
1680 TAILQ_REMOVE(&uc->uc_pi_list, pi, pi_hashlink);
1686 * Find a PI mutex in hash table.
1688 static struct umtx_pi *
1689 umtx_pi_lookup(struct umtx_key *key)
1691 struct umtxq_chain *uc;
1694 uc = umtxq_getchain(key);
1695 UMTXQ_LOCKED_ASSERT(uc);
1697 TAILQ_FOREACH(pi, &uc->uc_pi_list, pi_hashlink) {
1698 if (umtx_key_match(&pi->pi_key, key)) {
1706 * Insert a PI mutex into hash table.
1709 umtx_pi_insert(struct umtx_pi *pi)
1711 struct umtxq_chain *uc;
1713 uc = umtxq_getchain(&pi->pi_key);
1714 UMTXQ_LOCKED_ASSERT(uc);
1715 TAILQ_INSERT_TAIL(&uc->uc_pi_list, pi, pi_hashlink);
1722 _do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, int timo,
1726 struct umtx_pi *pi, *new_pi;
1727 uint32_t id, owner, old;
1733 if ((error = umtx_key_get(m, TYPE_PI_UMUTEX, GET_SHARE(flags),
1736 umtxq_lock(&uq->uq_key);
1737 pi = umtx_pi_lookup(&uq->uq_key);
1739 new_pi = umtx_pi_alloc(M_NOWAIT);
1740 if (new_pi == NULL) {
1741 umtxq_unlock(&uq->uq_key);
1742 new_pi = umtx_pi_alloc(M_WAITOK);
1743 umtxq_lock(&uq->uq_key);
1744 pi = umtx_pi_lookup(&uq->uq_key);
1746 umtx_pi_free(new_pi);
1750 if (new_pi != NULL) {
1751 new_pi->pi_key = uq->uq_key;
1752 umtx_pi_insert(new_pi);
1757 umtxq_unlock(&uq->uq_key);
1760 * Care must be exercised when dealing with umtx structure. It
1761 * can fault on any access.
1765 * Try the uncontested case. This should be done in userland.
1767 owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id);
1769 /* The acquire succeeded. */
1770 if (owner == UMUTEX_UNOWNED) {
1775 /* The address was invalid. */
1781 /* If no one owns it but it is contested try to acquire it. */
1782 if (owner == UMUTEX_CONTESTED) {
1783 owner = casuword32(&m->m_owner,
1784 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
1786 if (owner == UMUTEX_CONTESTED) {
1787 umtxq_lock(&uq->uq_key);
1788 umtxq_busy(&uq->uq_key);
1789 error = umtx_pi_claim(pi, td);
1790 umtxq_unbusy(&uq->uq_key);
1791 umtxq_unlock(&uq->uq_key);
1795 /* The address was invalid. */
1801 /* If this failed the lock has changed, restart. */
1805 if ((flags & UMUTEX_ERROR_CHECK) != 0 &&
1806 (owner & ~UMUTEX_CONTESTED) == id) {
1817 * If we caught a signal, we have retried and now
1823 umtxq_lock(&uq->uq_key);
1824 umtxq_busy(&uq->uq_key);
1825 umtxq_unlock(&uq->uq_key);
1828 * Set the contested bit so that a release in user space
1829 * knows to use the system call for unlock. If this fails
1830 * either some one else has acquired the lock or it has been
1833 old = casuword32(&m->m_owner, owner, owner | UMUTEX_CONTESTED);
1835 /* The address was invalid. */
1837 umtxq_lock(&uq->uq_key);
1838 umtxq_unbusy(&uq->uq_key);
1839 umtxq_unlock(&uq->uq_key);
1844 umtxq_lock(&uq->uq_key);
1846 * We set the contested bit, sleep. Otherwise the lock changed
1847 * and we need to retry or we lost a race to the thread
1848 * unlocking the umtx.
1851 error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED,
1854 umtxq_unbusy(&uq->uq_key);
1855 umtxq_unlock(&uq->uq_key);
1859 umtxq_lock(&uq->uq_key);
1861 umtxq_unlock(&uq->uq_key);
1863 umtx_key_release(&uq->uq_key);
1868 * Unlock a PI mutex.
1871 do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags)
1873 struct umtx_key key;
1874 struct umtx_q *uq_first, *uq_first2, *uq_me;
1875 struct umtx_pi *pi, *pi2;
1876 uint32_t owner, old, id;
1883 * Make sure we own this mtx.
1885 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
1889 if ((owner & ~UMUTEX_CONTESTED) != id)
1892 /* This should be done in userland */
1893 if ((owner & UMUTEX_CONTESTED) == 0) {
1894 old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED);
1902 /* We should only ever be in here for contested locks */
1903 if ((error = umtx_key_get(m, TYPE_PI_UMUTEX, GET_SHARE(flags),
1909 count = umtxq_count_pi(&key, &uq_first);
1910 if (uq_first != NULL) {
1911 mtx_lock_spin(&umtx_lock);
1912 pi = uq_first->uq_pi_blocked;
1913 KASSERT(pi != NULL, ("pi == NULL?"));
1914 if (pi->pi_owner != curthread) {
1915 mtx_unlock_spin(&umtx_lock);
1918 umtx_key_release(&key);
1919 /* userland messed the mutex */
1922 uq_me = curthread->td_umtxq;
1923 pi->pi_owner = NULL;
1924 TAILQ_REMOVE(&uq_me->uq_pi_contested, pi, pi_link);
1925 /* get highest priority thread which is still sleeping. */
1926 uq_first = TAILQ_FIRST(&pi->pi_blocked);
1927 while (uq_first != NULL &&
1928 (uq_first->uq_flags & UQF_UMTXQ) == 0) {
1929 uq_first = TAILQ_NEXT(uq_first, uq_lockq);
1932 TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) {
1933 uq_first2 = TAILQ_FIRST(&pi2->pi_blocked);
1934 if (uq_first2 != NULL) {
1935 if (pri > UPRI(uq_first2->uq_thread))
1936 pri = UPRI(uq_first2->uq_thread);
1939 thread_lock(curthread);
1940 sched_unlend_user_prio(curthread, pri);
1941 thread_unlock(curthread);
1942 mtx_unlock_spin(&umtx_lock);
1944 umtxq_signal_thread(uq_first);
1949 * When unlocking the umtx, it must be marked as unowned if
1950 * there is zero or one thread only waiting for it.
1951 * Otherwise, it must be marked as contested.
1953 old = casuword32(&m->m_owner, owner,
1954 count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
1959 umtx_key_release(&key);
1971 _do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, int timo,
1974 struct umtx_q *uq, *uq2;
1978 int error, pri, old_inherited_pri, su;
1982 if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags),
1985 su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0);
1987 old_inherited_pri = uq->uq_inherited_pri;
1988 umtxq_lock(&uq->uq_key);
1989 umtxq_busy(&uq->uq_key);
1990 umtxq_unlock(&uq->uq_key);
1992 ceiling = RTP_PRIO_MAX - fuword32(&m->m_ceilings[0]);
1993 if (ceiling > RTP_PRIO_MAX) {
1998 mtx_lock_spin(&umtx_lock);
1999 if (UPRI(td) < PRI_MIN_REALTIME + ceiling) {
2000 mtx_unlock_spin(&umtx_lock);
2004 if (su && PRI_MIN_REALTIME + ceiling < uq->uq_inherited_pri) {
2005 uq->uq_inherited_pri = PRI_MIN_REALTIME + ceiling;
2007 if (uq->uq_inherited_pri < UPRI(td))
2008 sched_lend_user_prio(td, uq->uq_inherited_pri);
2011 mtx_unlock_spin(&umtx_lock);
2013 owner = casuword32(&m->m_owner,
2014 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
2016 if (owner == UMUTEX_CONTESTED) {
2021 /* The address was invalid. */
2027 if ((flags & UMUTEX_ERROR_CHECK) != 0 &&
2028 (owner & ~UMUTEX_CONTESTED) == id) {
2039 * If we caught a signal, we have retried and now
2045 umtxq_lock(&uq->uq_key);
2047 umtxq_unbusy(&uq->uq_key);
2048 error = umtxq_sleep(uq, "umtxpp", timo);
2050 umtxq_unlock(&uq->uq_key);
2052 mtx_lock_spin(&umtx_lock);
2053 uq->uq_inherited_pri = old_inherited_pri;
2055 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2056 uq2 = TAILQ_FIRST(&pi->pi_blocked);
2058 if (pri > UPRI(uq2->uq_thread))
2059 pri = UPRI(uq2->uq_thread);
2062 if (pri > uq->uq_inherited_pri)
2063 pri = uq->uq_inherited_pri;
2065 sched_unlend_user_prio(td, pri);
2067 mtx_unlock_spin(&umtx_lock);
2071 mtx_lock_spin(&umtx_lock);
2072 uq->uq_inherited_pri = old_inherited_pri;
2074 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2075 uq2 = TAILQ_FIRST(&pi->pi_blocked);
2077 if (pri > UPRI(uq2->uq_thread))
2078 pri = UPRI(uq2->uq_thread);
2081 if (pri > uq->uq_inherited_pri)
2082 pri = uq->uq_inherited_pri;
2084 sched_unlend_user_prio(td, pri);
2086 mtx_unlock_spin(&umtx_lock);
2090 umtxq_lock(&uq->uq_key);
2091 umtxq_unbusy(&uq->uq_key);
2092 umtxq_unlock(&uq->uq_key);
2093 umtx_key_release(&uq->uq_key);
2098 * Unlock a PP mutex.
2101 do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags)
2103 struct umtx_key key;
2104 struct umtx_q *uq, *uq2;
2108 int error, pri, new_inherited_pri, su;
2112 su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0);
2115 * Make sure we own this mtx.
2117 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
2121 if ((owner & ~UMUTEX_CONTESTED) != id)
2124 error = copyin(&m->m_ceilings[1], &rceiling, sizeof(uint32_t));
2129 new_inherited_pri = PRI_MAX;
2131 rceiling = RTP_PRIO_MAX - rceiling;
2132 if (rceiling > RTP_PRIO_MAX)
2134 new_inherited_pri = PRI_MIN_REALTIME + rceiling;
2137 if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags),
2144 * For priority protected mutex, always set unlocked state
2145 * to UMUTEX_CONTESTED, so that userland always enters kernel
2146 * to lock the mutex, it is necessary because thread priority
2147 * has to be adjusted for such mutex.
2149 error = suword32(__DEVOLATILE(uint32_t *, &m->m_owner),
2154 umtxq_signal(&key, 1);
2161 mtx_lock_spin(&umtx_lock);
2163 uq->uq_inherited_pri = new_inherited_pri;
2165 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2166 uq2 = TAILQ_FIRST(&pi->pi_blocked);
2168 if (pri > UPRI(uq2->uq_thread))
2169 pri = UPRI(uq2->uq_thread);
2172 if (pri > uq->uq_inherited_pri)
2173 pri = uq->uq_inherited_pri;
2175 sched_unlend_user_prio(td, pri);
2177 mtx_unlock_spin(&umtx_lock);
2179 umtx_key_release(&key);
2184 do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling,
2185 uint32_t *old_ceiling)
2188 uint32_t save_ceiling;
2193 flags = fuword32(&m->m_flags);
2194 if ((flags & UMUTEX_PRIO_PROTECT) == 0)
2196 if (ceiling > RTP_PRIO_MAX)
2200 if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags),
2204 umtxq_lock(&uq->uq_key);
2205 umtxq_busy(&uq->uq_key);
2206 umtxq_unlock(&uq->uq_key);
2208 save_ceiling = fuword32(&m->m_ceilings[0]);
2210 owner = casuword32(&m->m_owner,
2211 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
2213 if (owner == UMUTEX_CONTESTED) {
2214 suword32(&m->m_ceilings[0], ceiling);
2215 suword32(__DEVOLATILE(uint32_t *, &m->m_owner),
2221 /* The address was invalid. */
2227 if ((owner & ~UMUTEX_CONTESTED) == id) {
2228 suword32(&m->m_ceilings[0], ceiling);
2234 * If we caught a signal, we have retried and now
2241 * We set the contested bit, sleep. Otherwise the lock changed
2242 * and we need to retry or we lost a race to the thread
2243 * unlocking the umtx.
2245 umtxq_lock(&uq->uq_key);
2247 umtxq_unbusy(&uq->uq_key);
2248 error = umtxq_sleep(uq, "umtxpp", 0);
2250 umtxq_unlock(&uq->uq_key);
2252 umtxq_lock(&uq->uq_key);
2254 umtxq_signal(&uq->uq_key, INT_MAX);
2255 umtxq_unbusy(&uq->uq_key);
2256 umtxq_unlock(&uq->uq_key);
2257 umtx_key_release(&uq->uq_key);
2258 if (error == 0 && old_ceiling != NULL)
2259 suword32(old_ceiling, save_ceiling);
2264 _do_lock_umutex(struct thread *td, struct umutex *m, int flags, int timo,
2267 switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) {
2269 return (_do_lock_normal(td, m, flags, timo, mode));
2270 case UMUTEX_PRIO_INHERIT:
2271 return (_do_lock_pi(td, m, flags, timo, mode));
2272 case UMUTEX_PRIO_PROTECT:
2273 return (_do_lock_pp(td, m, flags, timo, mode));
2279 * Lock a userland POSIX mutex.
2282 do_lock_umutex(struct thread *td, struct umutex *m,
2283 struct timespec *timeout, int mode)
2285 struct timespec ts, ts2, ts3;
2290 flags = fuword32(&m->m_flags);
2294 if (timeout == NULL) {
2295 error = _do_lock_umutex(td, m, flags, 0, mode);
2296 /* Mutex locking is restarted if it is interrupted. */
2297 if (error == EINTR && mode != _UMUTEX_WAIT)
2301 timespecadd(&ts, timeout);
2302 TIMESPEC_TO_TIMEVAL(&tv, timeout);
2304 error = _do_lock_umutex(td, m, flags, tvtohz(&tv), mode);
2305 if (error != ETIMEDOUT)
2307 getnanouptime(&ts2);
2308 if (timespeccmp(&ts2, &ts, >=)) {
2313 timespecsub(&ts3, &ts2);
2314 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
2316 /* Timed-locking is not restarted. */
2317 if (error == ERESTART)
2324 * Unlock a userland POSIX mutex.
2327 do_unlock_umutex(struct thread *td, struct umutex *m)
2331 flags = fuword32(&m->m_flags);
2335 switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) {
2337 return (do_unlock_normal(td, m, flags));
2338 case UMUTEX_PRIO_INHERIT:
2339 return (do_unlock_pi(td, m, flags));
2340 case UMUTEX_PRIO_PROTECT:
2341 return (do_unlock_pp(td, m, flags));
2348 do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m,
2349 struct timespec *timeout, u_long wflags)
2353 struct timespec cts, ets, tts;
2358 flags = fuword32(&cv->c_flags);
2359 error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key);
2362 umtxq_lock(&uq->uq_key);
2363 umtxq_busy(&uq->uq_key);
2365 umtxq_unlock(&uq->uq_key);
2368 * The magic thing is we should set c_has_waiters to 1 before
2369 * releasing user mutex.
2371 suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 1);
2373 umtxq_lock(&uq->uq_key);
2374 umtxq_unbusy(&uq->uq_key);
2375 umtxq_unlock(&uq->uq_key);
2377 error = do_unlock_umutex(td, m);
2379 umtxq_lock(&uq->uq_key);
2381 if ((wflags & UMTX_CHECK_UNPARKING) &&
2382 (td->td_pflags & TDP_WAKEUP)) {
2383 td->td_pflags &= ~TDP_WAKEUP;
2385 } else if (timeout == NULL) {
2386 error = umtxq_sleep(uq, "ucond", 0);
2388 getnanouptime(&ets);
2389 timespecadd(&ets, timeout);
2390 TIMESPEC_TO_TIMEVAL(&tv, timeout);
2392 error = umtxq_sleep(uq, "ucond", tvtohz(&tv));
2393 if (error != ETIMEDOUT)
2395 getnanouptime(&cts);
2396 if (timespeccmp(&cts, &ets, >=)) {
2401 timespecsub(&tts, &cts);
2402 TIMESPEC_TO_TIMEVAL(&tv, &tts);
2408 if ((uq->uq_flags & UQF_UMTXQ) == 0) {
2410 * If we concurrently got do_cv_signal()d
2411 * and we got an error or UNIX signals or a timeout,
2412 * then, perform another umtxq_signal to avoid
2413 * consuming the wakeup. This may cause supurious
2414 * wakeup for another thread which was just queued,
2415 * but SUSV3 explicitly allows supurious wakeup to
2416 * occur, and indeed a kernel based implementation
2419 if (!umtxq_signal(&uq->uq_key, 1))
2422 if (error == ERESTART)
2426 umtxq_unlock(&uq->uq_key);
2427 umtx_key_release(&uq->uq_key);
2432 * Signal a userland condition variable.
2435 do_cv_signal(struct thread *td, struct ucond *cv)
2437 struct umtx_key key;
2438 int error, cnt, nwake;
2441 flags = fuword32(&cv->c_flags);
2442 if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0)
2446 cnt = umtxq_count(&key);
2447 nwake = umtxq_signal(&key, 1);
2451 __DEVOLATILE(uint32_t *, &cv->c_has_waiters), 0);
2456 umtx_key_release(&key);
2461 do_cv_broadcast(struct thread *td, struct ucond *cv)
2463 struct umtx_key key;
2467 flags = fuword32(&cv->c_flags);
2468 if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0)
2473 umtxq_signal(&key, INT_MAX);
2476 error = suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 0);
2482 umtx_key_release(&key);
2487 do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag, int timo)
2490 uint32_t flags, wrflags;
2491 int32_t state, oldstate;
2492 int32_t blocked_readers;
2496 flags = fuword32(&rwlock->rw_flags);
2497 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
2501 wrflags = URWLOCK_WRITE_OWNER;
2502 if (!(fflag & URWLOCK_PREFER_READER) && !(flags & URWLOCK_PREFER_READER))
2503 wrflags |= URWLOCK_WRITE_WAITERS;
2506 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2507 /* try to lock it */
2508 while (!(state & wrflags)) {
2509 if (__predict_false(URWLOCK_READER_COUNT(state) == URWLOCK_MAX_READERS)) {
2510 umtx_key_release(&uq->uq_key);
2513 oldstate = casuword32(&rwlock->rw_state, state, state + 1);
2514 if (oldstate == state) {
2515 umtx_key_release(&uq->uq_key);
2524 /* grab monitor lock */
2525 umtxq_lock(&uq->uq_key);
2526 umtxq_busy(&uq->uq_key);
2527 umtxq_unlock(&uq->uq_key);
2530 * re-read the state, in case it changed between the try-lock above
2531 * and the check below
2533 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2535 /* set read contention bit */
2536 while ((state & wrflags) && !(state & URWLOCK_READ_WAITERS)) {
2537 oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_READ_WAITERS);
2538 if (oldstate == state)
2543 /* state is changed while setting flags, restart */
2544 if (!(state & wrflags)) {
2545 umtxq_lock(&uq->uq_key);
2546 umtxq_unbusy(&uq->uq_key);
2547 umtxq_unlock(&uq->uq_key);
2552 /* contention bit is set, before sleeping, increase read waiter count */
2553 blocked_readers = fuword32(&rwlock->rw_blocked_readers);
2554 suword32(&rwlock->rw_blocked_readers, blocked_readers+1);
2556 while (state & wrflags) {
2557 umtxq_lock(&uq->uq_key);
2559 umtxq_unbusy(&uq->uq_key);
2561 error = umtxq_sleep(uq, "urdlck", timo);
2563 umtxq_busy(&uq->uq_key);
2565 umtxq_unlock(&uq->uq_key);
2568 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2571 /* decrease read waiter count, and may clear read contention bit */
2572 blocked_readers = fuword32(&rwlock->rw_blocked_readers);
2573 suword32(&rwlock->rw_blocked_readers, blocked_readers-1);
2574 if (blocked_readers == 1) {
2575 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2577 oldstate = casuword32(&rwlock->rw_state, state,
2578 state & ~URWLOCK_READ_WAITERS);
2579 if (oldstate == state)
2585 umtxq_lock(&uq->uq_key);
2586 umtxq_unbusy(&uq->uq_key);
2587 umtxq_unlock(&uq->uq_key);
2589 umtx_key_release(&uq->uq_key);
2594 do_rw_rdlock2(struct thread *td, void *obj, long val, struct timespec *timeout)
2596 struct timespec ts, ts2, ts3;
2601 timespecadd(&ts, timeout);
2602 TIMESPEC_TO_TIMEVAL(&tv, timeout);
2604 error = do_rw_rdlock(td, obj, val, tvtohz(&tv));
2605 if (error != ETIMEDOUT)
2607 getnanouptime(&ts2);
2608 if (timespeccmp(&ts2, &ts, >=)) {
2613 timespecsub(&ts3, &ts2);
2614 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
2616 if (error == ERESTART)
2622 do_rw_wrlock(struct thread *td, struct urwlock *rwlock, int timo)
2626 int32_t state, oldstate;
2627 int32_t blocked_writers;
2628 int32_t blocked_readers;
2632 flags = fuword32(&rwlock->rw_flags);
2633 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
2637 blocked_readers = 0;
2639 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2640 while (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) {
2641 oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_WRITE_OWNER);
2642 if (oldstate == state) {
2643 umtx_key_release(&uq->uq_key);
2650 if (!(state & (URWLOCK_WRITE_OWNER|URWLOCK_WRITE_WAITERS)) &&
2651 blocked_readers != 0) {
2652 umtxq_lock(&uq->uq_key);
2653 umtxq_busy(&uq->uq_key);
2654 umtxq_signal_queue(&uq->uq_key, INT_MAX, UMTX_SHARED_QUEUE);
2655 umtxq_unbusy(&uq->uq_key);
2656 umtxq_unlock(&uq->uq_key);
2662 /* grab monitor lock */
2663 umtxq_lock(&uq->uq_key);
2664 umtxq_busy(&uq->uq_key);
2665 umtxq_unlock(&uq->uq_key);
2668 * re-read the state, in case it changed between the try-lock above
2669 * and the check below
2671 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2673 while (((state & URWLOCK_WRITE_OWNER) || URWLOCK_READER_COUNT(state) != 0) &&
2674 (state & URWLOCK_WRITE_WAITERS) == 0) {
2675 oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_WRITE_WAITERS);
2676 if (oldstate == state)
2681 if (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) {
2682 umtxq_lock(&uq->uq_key);
2683 umtxq_unbusy(&uq->uq_key);
2684 umtxq_unlock(&uq->uq_key);
2688 blocked_writers = fuword32(&rwlock->rw_blocked_writers);
2689 suword32(&rwlock->rw_blocked_writers, blocked_writers+1);
2691 while ((state & URWLOCK_WRITE_OWNER) || URWLOCK_READER_COUNT(state) != 0) {
2692 umtxq_lock(&uq->uq_key);
2693 umtxq_insert_queue(uq, UMTX_EXCLUSIVE_QUEUE);
2694 umtxq_unbusy(&uq->uq_key);
2696 error = umtxq_sleep(uq, "uwrlck", timo);
2698 umtxq_busy(&uq->uq_key);
2699 umtxq_remove_queue(uq, UMTX_EXCLUSIVE_QUEUE);
2700 umtxq_unlock(&uq->uq_key);
2703 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2706 blocked_writers = fuword32(&rwlock->rw_blocked_writers);
2707 suword32(&rwlock->rw_blocked_writers, blocked_writers-1);
2708 if (blocked_writers == 1) {
2709 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2711 oldstate = casuword32(&rwlock->rw_state, state,
2712 state & ~URWLOCK_WRITE_WAITERS);
2713 if (oldstate == state)
2717 blocked_readers = fuword32(&rwlock->rw_blocked_readers);
2719 blocked_readers = 0;
2721 umtxq_lock(&uq->uq_key);
2722 umtxq_unbusy(&uq->uq_key);
2723 umtxq_unlock(&uq->uq_key);
2726 umtx_key_release(&uq->uq_key);
2731 do_rw_wrlock2(struct thread *td, void *obj, struct timespec *timeout)
2733 struct timespec ts, ts2, ts3;
2738 timespecadd(&ts, timeout);
2739 TIMESPEC_TO_TIMEVAL(&tv, timeout);
2741 error = do_rw_wrlock(td, obj, tvtohz(&tv));
2742 if (error != ETIMEDOUT)
2744 getnanouptime(&ts2);
2745 if (timespeccmp(&ts2, &ts, >=)) {
2750 timespecsub(&ts3, &ts2);
2751 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
2753 if (error == ERESTART)
2759 do_rw_unlock(struct thread *td, struct urwlock *rwlock)
2763 int32_t state, oldstate;
2764 int error, q, count;
2767 flags = fuword32(&rwlock->rw_flags);
2768 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
2772 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2773 if (state & URWLOCK_WRITE_OWNER) {
2775 oldstate = casuword32(&rwlock->rw_state, state,
2776 state & ~URWLOCK_WRITE_OWNER);
2777 if (oldstate != state) {
2779 if (!(oldstate & URWLOCK_WRITE_OWNER)) {
2786 } else if (URWLOCK_READER_COUNT(state) != 0) {
2788 oldstate = casuword32(&rwlock->rw_state, state,
2790 if (oldstate != state) {
2792 if (URWLOCK_READER_COUNT(oldstate) == 0) {
2807 if (!(flags & URWLOCK_PREFER_READER)) {
2808 if (state & URWLOCK_WRITE_WAITERS) {
2810 q = UMTX_EXCLUSIVE_QUEUE;
2811 } else if (state & URWLOCK_READ_WAITERS) {
2813 q = UMTX_SHARED_QUEUE;
2816 if (state & URWLOCK_READ_WAITERS) {
2818 q = UMTX_SHARED_QUEUE;
2819 } else if (state & URWLOCK_WRITE_WAITERS) {
2821 q = UMTX_EXCLUSIVE_QUEUE;
2826 umtxq_lock(&uq->uq_key);
2827 umtxq_busy(&uq->uq_key);
2828 umtxq_signal_queue(&uq->uq_key, count, q);
2829 umtxq_unbusy(&uq->uq_key);
2830 umtxq_unlock(&uq->uq_key);
2833 umtx_key_release(&uq->uq_key);
2838 do_sem_wait(struct thread *td, struct _usem *sem, struct timespec *timeout)
2842 struct timespec cts, ets, tts;
2843 uint32_t flags, count;
2847 flags = fuword32(&sem->_flags);
2848 error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key);
2851 umtxq_lock(&uq->uq_key);
2852 umtxq_busy(&uq->uq_key);
2854 umtxq_unlock(&uq->uq_key);
2856 suword32(__DEVOLATILE(uint32_t *, &sem->_has_waiters), 1);
2858 count = fuword32(__DEVOLATILE(uint32_t *, &sem->_count));
2860 umtxq_lock(&uq->uq_key);
2861 umtxq_unbusy(&uq->uq_key);
2863 umtxq_unlock(&uq->uq_key);
2864 umtx_key_release(&uq->uq_key);
2868 umtxq_lock(&uq->uq_key);
2869 umtxq_unbusy(&uq->uq_key);
2870 umtxq_unlock(&uq->uq_key);
2872 umtxq_lock(&uq->uq_key);
2873 if (timeout == NULL) {
2874 error = umtxq_sleep(uq, "usem", 0);
2876 getnanouptime(&ets);
2877 timespecadd(&ets, timeout);
2878 TIMESPEC_TO_TIMEVAL(&tv, timeout);
2880 error = umtxq_sleep(uq, "usem", tvtohz(&tv));
2881 if (error != ETIMEDOUT)
2883 getnanouptime(&cts);
2884 if (timespeccmp(&cts, &ets, >=)) {
2889 timespecsub(&tts, &cts);
2890 TIMESPEC_TO_TIMEVAL(&tv, &tts);
2895 if ((uq->uq_flags & UQF_UMTXQ) == 0) {
2896 if (!umtxq_signal(&uq->uq_key, 1))
2899 if (error == ERESTART)
2903 umtxq_unlock(&uq->uq_key);
2904 umtx_key_release(&uq->uq_key);
2909 * Signal a userland condition variable.
2912 do_sem_wake(struct thread *td, struct _usem *sem)
2914 struct umtx_key key;
2915 int error, cnt, nwake;
2918 flags = fuword32(&sem->_flags);
2919 if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0)
2923 cnt = umtxq_count(&key);
2924 nwake = umtxq_signal(&key, 1);
2928 __DEVOLATILE(uint32_t *, &sem->_has_waiters), 0);
2933 umtx_key_release(&key);
2938 _umtx_lock(struct thread *td, struct _umtx_lock_args *uap)
2939 /* struct umtx *umtx */
2941 return _do_lock_umtx(td, uap->umtx, td->td_tid, 0);
2945 _umtx_unlock(struct thread *td, struct _umtx_unlock_args *uap)
2946 /* struct umtx *umtx */
2948 return do_unlock_umtx(td, uap->umtx, td->td_tid);
2952 __umtx_op_lock_umtx(struct thread *td, struct _umtx_op_args *uap)
2954 struct timespec *ts, timeout;
2957 /* Allow a null timespec (wait forever). */
2958 if (uap->uaddr2 == NULL)
2961 error = copyin(uap->uaddr2, &timeout, sizeof(timeout));
2964 if (timeout.tv_nsec >= 1000000000 ||
2965 timeout.tv_nsec < 0) {
2970 return (do_lock_umtx(td, uap->obj, uap->val, ts));
2974 __umtx_op_unlock_umtx(struct thread *td, struct _umtx_op_args *uap)
2976 return (do_unlock_umtx(td, uap->obj, uap->val));
2980 __umtx_op_wait(struct thread *td, struct _umtx_op_args *uap)
2982 struct timespec *ts, timeout;
2985 if (uap->uaddr2 == NULL)
2988 error = copyin(uap->uaddr2, &timeout, sizeof(timeout));
2991 if (timeout.tv_nsec >= 1000000000 ||
2992 timeout.tv_nsec < 0)
2996 return do_wait(td, uap->obj, uap->val, ts, 0, 0);
3000 __umtx_op_wait_uint(struct thread *td, struct _umtx_op_args *uap)
3002 struct timespec *ts, timeout;
3005 if (uap->uaddr2 == NULL)
3008 error = copyin(uap->uaddr2, &timeout, sizeof(timeout));
3011 if (timeout.tv_nsec >= 1000000000 ||
3012 timeout.tv_nsec < 0)
3016 return do_wait(td, uap->obj, uap->val, ts, 1, 0);
3020 __umtx_op_wait_uint_private(struct thread *td, struct _umtx_op_args *uap)
3022 struct timespec *ts, timeout;
3025 if (uap->uaddr2 == NULL)
3028 error = copyin(uap->uaddr2, &timeout, sizeof(timeout));
3031 if (timeout.tv_nsec >= 1000000000 ||
3032 timeout.tv_nsec < 0)
3036 return do_wait(td, uap->obj, uap->val, ts, 1, 1);
3040 __umtx_op_wake(struct thread *td, struct _umtx_op_args *uap)
3042 return (kern_umtx_wake(td, uap->obj, uap->val, 0));
3046 __umtx_op_wake_private(struct thread *td, struct _umtx_op_args *uap)
3048 return (kern_umtx_wake(td, uap->obj, uap->val, 1));
3052 __umtx_op_lock_umutex(struct thread *td, struct _umtx_op_args *uap)
3054 struct timespec *ts, timeout;
3057 /* Allow a null timespec (wait forever). */
3058 if (uap->uaddr2 == NULL)
3061 error = copyin(uap->uaddr2, &timeout,
3065 if (timeout.tv_nsec >= 1000000000 ||
3066 timeout.tv_nsec < 0) {
3071 return do_lock_umutex(td, uap->obj, ts, 0);
3075 __umtx_op_trylock_umutex(struct thread *td, struct _umtx_op_args *uap)
3077 return do_lock_umutex(td, uap->obj, NULL, _UMUTEX_TRY);
3081 __umtx_op_wait_umutex(struct thread *td, struct _umtx_op_args *uap)
3083 struct timespec *ts, timeout;
3086 /* Allow a null timespec (wait forever). */
3087 if (uap->uaddr2 == NULL)
3090 error = copyin(uap->uaddr2, &timeout,
3094 if (timeout.tv_nsec >= 1000000000 ||
3095 timeout.tv_nsec < 0) {
3100 return do_lock_umutex(td, uap->obj, ts, _UMUTEX_WAIT);
3104 __umtx_op_wake_umutex(struct thread *td, struct _umtx_op_args *uap)
3106 return do_wake_umutex(td, uap->obj);
3110 __umtx_op_unlock_umutex(struct thread *td, struct _umtx_op_args *uap)
3112 return do_unlock_umutex(td, uap->obj);
3116 __umtx_op_set_ceiling(struct thread *td, struct _umtx_op_args *uap)
3118 return do_set_ceiling(td, uap->obj, uap->val, uap->uaddr1);
3122 __umtx_op_cv_wait(struct thread *td, struct _umtx_op_args *uap)
3124 struct timespec *ts, timeout;
3127 /* Allow a null timespec (wait forever). */
3128 if (uap->uaddr2 == NULL)
3131 error = copyin(uap->uaddr2, &timeout,
3135 if (timeout.tv_nsec >= 1000000000 ||
3136 timeout.tv_nsec < 0) {
3141 return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val));
3145 __umtx_op_cv_signal(struct thread *td, struct _umtx_op_args *uap)
3147 return do_cv_signal(td, uap->obj);
3151 __umtx_op_cv_broadcast(struct thread *td, struct _umtx_op_args *uap)
3153 return do_cv_broadcast(td, uap->obj);
3157 __umtx_op_rw_rdlock(struct thread *td, struct _umtx_op_args *uap)
3159 struct timespec timeout;
3162 /* Allow a null timespec (wait forever). */
3163 if (uap->uaddr2 == NULL) {
3164 error = do_rw_rdlock(td, uap->obj, uap->val, 0);
3166 error = copyin(uap->uaddr2, &timeout,
3170 if (timeout.tv_nsec >= 1000000000 ||
3171 timeout.tv_nsec < 0) {
3174 error = do_rw_rdlock2(td, uap->obj, uap->val, &timeout);
3180 __umtx_op_rw_wrlock(struct thread *td, struct _umtx_op_args *uap)
3182 struct timespec timeout;
3185 /* Allow a null timespec (wait forever). */
3186 if (uap->uaddr2 == NULL) {
3187 error = do_rw_wrlock(td, uap->obj, 0);
3189 error = copyin(uap->uaddr2, &timeout,
3193 if (timeout.tv_nsec >= 1000000000 ||
3194 timeout.tv_nsec < 0) {
3198 error = do_rw_wrlock2(td, uap->obj, &timeout);
3204 __umtx_op_rw_unlock(struct thread *td, struct _umtx_op_args *uap)
3206 return do_rw_unlock(td, uap->obj);
3210 __umtx_op_sem_wait(struct thread *td, struct _umtx_op_args *uap)
3212 struct timespec *ts, timeout;
3215 /* Allow a null timespec (wait forever). */
3216 if (uap->uaddr2 == NULL)
3219 error = copyin(uap->uaddr2, &timeout,
3223 if (timeout.tv_nsec >= 1000000000 ||
3224 timeout.tv_nsec < 0) {
3229 return (do_sem_wait(td, uap->obj, ts));
3233 __umtx_op_sem_wake(struct thread *td, struct _umtx_op_args *uap)
3235 return do_sem_wake(td, uap->obj);
3238 typedef int (*_umtx_op_func)(struct thread *td, struct _umtx_op_args *uap);
3240 static _umtx_op_func op_table[] = {
3241 __umtx_op_lock_umtx, /* UMTX_OP_LOCK */
3242 __umtx_op_unlock_umtx, /* UMTX_OP_UNLOCK */
3243 __umtx_op_wait, /* UMTX_OP_WAIT */
3244 __umtx_op_wake, /* UMTX_OP_WAKE */
3245 __umtx_op_trylock_umutex, /* UMTX_OP_MUTEX_TRYLOCK */
3246 __umtx_op_lock_umutex, /* UMTX_OP_MUTEX_LOCK */
3247 __umtx_op_unlock_umutex, /* UMTX_OP_MUTEX_UNLOCK */
3248 __umtx_op_set_ceiling, /* UMTX_OP_SET_CEILING */
3249 __umtx_op_cv_wait, /* UMTX_OP_CV_WAIT*/
3250 __umtx_op_cv_signal, /* UMTX_OP_CV_SIGNAL */
3251 __umtx_op_cv_broadcast, /* UMTX_OP_CV_BROADCAST */
3252 __umtx_op_wait_uint, /* UMTX_OP_WAIT_UINT */
3253 __umtx_op_rw_rdlock, /* UMTX_OP_RW_RDLOCK */
3254 __umtx_op_rw_wrlock, /* UMTX_OP_RW_WRLOCK */
3255 __umtx_op_rw_unlock, /* UMTX_OP_RW_UNLOCK */
3256 __umtx_op_wait_uint_private, /* UMTX_OP_WAIT_UINT_PRIVATE */
3257 __umtx_op_wake_private, /* UMTX_OP_WAKE_PRIVATE */
3258 __umtx_op_wait_umutex, /* UMTX_OP_UMUTEX_WAIT */
3259 __umtx_op_wake_umutex, /* UMTX_OP_UMUTEX_WAKE */
3260 __umtx_op_sem_wait, /* UMTX_OP_SEM_WAIT */
3261 __umtx_op_sem_wake /* UMTX_OP_SEM_WAKE */
3265 _umtx_op(struct thread *td, struct _umtx_op_args *uap)
3267 if ((unsigned)uap->op < UMTX_OP_MAX)
3268 return (*op_table[uap->op])(td, uap);
3274 freebsd32_umtx_lock(struct thread *td, struct freebsd32_umtx_lock_args *uap)
3275 /* struct umtx *umtx */
3277 return (do_lock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid, NULL));
3281 freebsd32_umtx_unlock(struct thread *td, struct freebsd32_umtx_unlock_args *uap)
3282 /* struct umtx *umtx */
3284 return (do_unlock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid));
3293 copyin_timeout32(void *addr, struct timespec *tsp)
3295 struct timespec32 ts32;
3298 error = copyin(addr, &ts32, sizeof(struct timespec32));
3300 tsp->tv_sec = ts32.tv_sec;
3301 tsp->tv_nsec = ts32.tv_nsec;
3307 __umtx_op_lock_umtx_compat32(struct thread *td, struct _umtx_op_args *uap)
3309 struct timespec *ts, timeout;
3312 /* Allow a null timespec (wait forever). */
3313 if (uap->uaddr2 == NULL)
3316 error = copyin_timeout32(uap->uaddr2, &timeout);
3319 if (timeout.tv_nsec >= 1000000000 ||
3320 timeout.tv_nsec < 0) {
3325 return (do_lock_umtx32(td, uap->obj, uap->val, ts));
3329 __umtx_op_unlock_umtx_compat32(struct thread *td, struct _umtx_op_args *uap)
3331 return (do_unlock_umtx32(td, uap->obj, (uint32_t)uap->val));
3335 __umtx_op_wait_compat32(struct thread *td, struct _umtx_op_args *uap)
3337 struct timespec *ts, timeout;
3340 if (uap->uaddr2 == NULL)
3343 error = copyin_timeout32(uap->uaddr2, &timeout);
3346 if (timeout.tv_nsec >= 1000000000 ||
3347 timeout.tv_nsec < 0)
3351 return do_wait(td, uap->obj, uap->val, ts, 1, 0);
3355 __umtx_op_lock_umutex_compat32(struct thread *td, struct _umtx_op_args *uap)
3357 struct timespec *ts, timeout;
3360 /* Allow a null timespec (wait forever). */
3361 if (uap->uaddr2 == NULL)
3364 error = copyin_timeout32(uap->uaddr2, &timeout);
3367 if (timeout.tv_nsec >= 1000000000 ||
3368 timeout.tv_nsec < 0)
3372 return do_lock_umutex(td, uap->obj, ts, 0);
3376 __umtx_op_wait_umutex_compat32(struct thread *td, struct _umtx_op_args *uap)
3378 struct timespec *ts, timeout;
3381 /* Allow a null timespec (wait forever). */
3382 if (uap->uaddr2 == NULL)
3385 error = copyin_timeout32(uap->uaddr2, &timeout);
3388 if (timeout.tv_nsec >= 1000000000 ||
3389 timeout.tv_nsec < 0)
3393 return do_lock_umutex(td, uap->obj, ts, _UMUTEX_WAIT);
3397 __umtx_op_cv_wait_compat32(struct thread *td, struct _umtx_op_args *uap)
3399 struct timespec *ts, timeout;
3402 /* Allow a null timespec (wait forever). */
3403 if (uap->uaddr2 == NULL)
3406 error = copyin_timeout32(uap->uaddr2, &timeout);
3409 if (timeout.tv_nsec >= 1000000000 ||
3410 timeout.tv_nsec < 0)
3414 return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val));
3418 __umtx_op_rw_rdlock_compat32(struct thread *td, struct _umtx_op_args *uap)
3420 struct timespec timeout;
3423 /* Allow a null timespec (wait forever). */
3424 if (uap->uaddr2 == NULL) {
3425 error = do_rw_rdlock(td, uap->obj, uap->val, 0);
3427 error = copyin(uap->uaddr2, &timeout,
3431 if (timeout.tv_nsec >= 1000000000 ||
3432 timeout.tv_nsec < 0) {
3435 error = do_rw_rdlock2(td, uap->obj, uap->val, &timeout);
3441 __umtx_op_rw_wrlock_compat32(struct thread *td, struct _umtx_op_args *uap)
3443 struct timespec timeout;
3446 /* Allow a null timespec (wait forever). */
3447 if (uap->uaddr2 == NULL) {
3448 error = do_rw_wrlock(td, uap->obj, 0);
3450 error = copyin_timeout32(uap->uaddr2, &timeout);
3453 if (timeout.tv_nsec >= 1000000000 ||
3454 timeout.tv_nsec < 0) {
3458 error = do_rw_wrlock2(td, uap->obj, &timeout);
3464 __umtx_op_wait_uint_private_compat32(struct thread *td, struct _umtx_op_args *uap)
3466 struct timespec *ts, timeout;
3469 if (uap->uaddr2 == NULL)
3472 error = copyin_timeout32(uap->uaddr2, &timeout);
3475 if (timeout.tv_nsec >= 1000000000 ||
3476 timeout.tv_nsec < 0)
3480 return do_wait(td, uap->obj, uap->val, ts, 1, 1);
3484 __umtx_op_sem_wait_compat32(struct thread *td, struct _umtx_op_args *uap)
3486 struct timespec *ts, timeout;
3489 /* Allow a null timespec (wait forever). */
3490 if (uap->uaddr2 == NULL)
3493 error = copyin_timeout32(uap->uaddr2, &timeout);
3496 if (timeout.tv_nsec >= 1000000000 ||
3497 timeout.tv_nsec < 0)
3501 return (do_sem_wait(td, uap->obj, ts));
3504 static _umtx_op_func op_table_compat32[] = {
3505 __umtx_op_lock_umtx_compat32, /* UMTX_OP_LOCK */
3506 __umtx_op_unlock_umtx_compat32, /* UMTX_OP_UNLOCK */
3507 __umtx_op_wait_compat32, /* UMTX_OP_WAIT */
3508 __umtx_op_wake, /* UMTX_OP_WAKE */
3509 __umtx_op_trylock_umutex, /* UMTX_OP_MUTEX_LOCK */
3510 __umtx_op_lock_umutex_compat32, /* UMTX_OP_MUTEX_TRYLOCK */
3511 __umtx_op_unlock_umutex, /* UMTX_OP_MUTEX_UNLOCK */
3512 __umtx_op_set_ceiling, /* UMTX_OP_SET_CEILING */
3513 __umtx_op_cv_wait_compat32, /* UMTX_OP_CV_WAIT*/
3514 __umtx_op_cv_signal, /* UMTX_OP_CV_SIGNAL */
3515 __umtx_op_cv_broadcast, /* UMTX_OP_CV_BROADCAST */
3516 __umtx_op_wait_compat32, /* UMTX_OP_WAIT_UINT */
3517 __umtx_op_rw_rdlock_compat32, /* UMTX_OP_RW_RDLOCK */
3518 __umtx_op_rw_wrlock_compat32, /* UMTX_OP_RW_WRLOCK */
3519 __umtx_op_rw_unlock, /* UMTX_OP_RW_UNLOCK */
3520 __umtx_op_wait_uint_private_compat32, /* UMTX_OP_WAIT_UINT_PRIVATE */
3521 __umtx_op_wake_private, /* UMTX_OP_WAKE_PRIVATE */
3522 __umtx_op_wait_umutex_compat32, /* UMTX_OP_UMUTEX_WAIT */
3523 __umtx_op_wake_umutex, /* UMTX_OP_UMUTEX_WAKE */
3524 __umtx_op_sem_wait_compat32, /* UMTX_OP_SEM_WAIT */
3525 __umtx_op_sem_wake /* UMTX_OP_SEM_WAKE */
3529 freebsd32_umtx_op(struct thread *td, struct freebsd32_umtx_op_args *uap)
3531 if ((unsigned)uap->op < UMTX_OP_MAX)
3532 return (*op_table_compat32[uap->op])(td,
3533 (struct _umtx_op_args *)uap);
3539 umtx_thread_init(struct thread *td)
3541 td->td_umtxq = umtxq_alloc();
3542 td->td_umtxq->uq_thread = td;
3546 umtx_thread_fini(struct thread *td)
3548 umtxq_free(td->td_umtxq);
3552 * It will be called when new thread is created, e.g fork().
3555 umtx_thread_alloc(struct thread *td)
3560 uq->uq_inherited_pri = PRI_MAX;
3562 KASSERT(uq->uq_flags == 0, ("uq_flags != 0"));
3563 KASSERT(uq->uq_thread == td, ("uq_thread != td"));
3564 KASSERT(uq->uq_pi_blocked == NULL, ("uq_pi_blocked != NULL"));
3565 KASSERT(TAILQ_EMPTY(&uq->uq_pi_contested), ("uq_pi_contested is not empty"));
3572 umtx_exec_hook(void *arg __unused, struct proc *p __unused,
3573 struct image_params *imgp __unused)
3575 umtx_thread_cleanup(curthread);
3579 * thread_exit() hook.
3582 umtx_thread_exit(struct thread *td)
3584 umtx_thread_cleanup(td);
3588 * clean up umtx data.
3591 umtx_thread_cleanup(struct thread *td)
3596 if ((uq = td->td_umtxq) == NULL)
3599 mtx_lock_spin(&umtx_lock);
3600 uq->uq_inherited_pri = PRI_MAX;
3601 while ((pi = TAILQ_FIRST(&uq->uq_pi_contested)) != NULL) {
3602 pi->pi_owner = NULL;
3603 TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link);
3606 td->td_flags &= ~TDF_UBORROWING;
3608 mtx_unlock_spin(&umtx_lock);