2 * Copyright (c) 2004, David Xu <davidxu@freebsd.org>
3 * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org>
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice unmodified, this list of conditions, and the following
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
31 #include "opt_compat.h"
32 #include <sys/param.h>
33 #include <sys/kernel.h>
34 #include <sys/limits.h>
36 #include <sys/malloc.h>
37 #include <sys/mutex.h>
40 #include <sys/sched.h>
42 #include <sys/sysctl.h>
43 #include <sys/sysent.h>
44 #include <sys/systm.h>
45 #include <sys/sysproto.h>
46 #include <sys/eventhandler.h>
50 #include <vm/vm_param.h>
52 #include <vm/vm_map.h>
53 #include <vm/vm_object.h>
55 #include <machine/cpu.h>
57 #ifdef COMPAT_FREEBSD32
58 #include <compat/freebsd32/freebsd32_proto.h>
73 #define _UMUTEX_WAIT 2
75 /* Key to represent a unique userland synchronous object */
96 /* Priority inheritance mutex info. */
99 struct thread *pi_owner;
101 /* Reference count */
104 /* List entry to link umtx holding by thread */
105 TAILQ_ENTRY(umtx_pi) pi_link;
107 /* List entry in hash */
108 TAILQ_ENTRY(umtx_pi) pi_hashlink;
110 /* List for waiters */
111 TAILQ_HEAD(,umtx_q) pi_blocked;
113 /* Identify a userland lock object */
114 struct umtx_key pi_key;
117 /* A userland synchronous object user. */
119 /* Linked list for the hash. */
120 TAILQ_ENTRY(umtx_q) uq_link;
123 struct umtx_key uq_key;
127 #define UQF_UMTXQ 0x0001
129 /* The thread waits on. */
130 struct thread *uq_thread;
133 * Blocked on PI mutex. read can use chain lock
134 * or umtx_lock, write must have both chain lock and
135 * umtx_lock being hold.
137 struct umtx_pi *uq_pi_blocked;
139 /* On blocked list */
140 TAILQ_ENTRY(umtx_q) uq_lockq;
142 /* Thread contending with us */
143 TAILQ_HEAD(,umtx_pi) uq_pi_contested;
145 /* Inherited priority from PP mutex */
146 u_char uq_inherited_pri;
148 /* Spare queue ready to be reused */
149 struct umtxq_queue *uq_spare_queue;
151 /* The queue we on */
152 struct umtxq_queue *uq_cur_queue;
155 TAILQ_HEAD(umtxq_head, umtx_q);
157 /* Per-key wait-queue */
159 struct umtxq_head head;
161 LIST_ENTRY(umtxq_queue) link;
165 LIST_HEAD(umtxq_list, umtxq_queue);
167 /* Userland lock object's wait-queue chain */
169 /* Lock for this chain. */
172 /* List of sleep queues. */
173 struct umtxq_list uc_queue[2];
174 #define UMTX_SHARED_QUEUE 0
175 #define UMTX_EXCLUSIVE_QUEUE 1
177 LIST_HEAD(, umtxq_queue) uc_spare_queue;
182 /* Chain lock waiters */
185 /* All PI in the list */
186 TAILQ_HEAD(,umtx_pi) uc_pi_list;
190 #define UMTXQ_LOCKED_ASSERT(uc) mtx_assert(&(uc)->uc_lock, MA_OWNED)
191 #define UMTXQ_BUSY_ASSERT(uc) KASSERT(&(uc)->uc_busy, ("umtx chain is not busy"))
194 * Don't propagate time-sharing priority, there is a security reason,
195 * a user can simply introduce PI-mutex, let thread A lock the mutex,
196 * and let another thread B block on the mutex, because B is
197 * sleeping, its priority will be boosted, this causes A's priority to
198 * be boosted via priority propagating too and will never be lowered even
199 * if it is using 100%CPU, this is unfair to other processes.
202 #define UPRI(td) (((td)->td_user_pri >= PRI_MIN_TIMESHARE &&\
203 (td)->td_user_pri <= PRI_MAX_TIMESHARE) ?\
204 PRI_MAX_TIMESHARE : (td)->td_user_pri)
206 #define GOLDEN_RATIO_PRIME 2654404609U
207 #define UMTX_CHAINS 128
208 #define UMTX_SHIFTS (__WORD_BIT - 7)
210 #define THREAD_SHARE 0
211 #define PROCESS_SHARE 1
214 #define GET_SHARE(flags) \
215 (((flags) & USYNC_PROCESS_SHARED) == 0 ? THREAD_SHARE : PROCESS_SHARE)
217 #define BUSY_SPINS 200
219 static uma_zone_t umtx_pi_zone;
220 static struct umtxq_chain umtxq_chains[2][UMTX_CHAINS];
221 static MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory");
222 static int umtx_pi_allocated;
224 SYSCTL_NODE(_debug, OID_AUTO, umtx, CTLFLAG_RW, 0, "umtx debug");
225 SYSCTL_INT(_debug_umtx, OID_AUTO, umtx_pi_allocated, CTLFLAG_RD,
226 &umtx_pi_allocated, 0, "Allocated umtx_pi");
228 static void umtxq_sysinit(void *);
229 static void umtxq_hash(struct umtx_key *key);
230 static struct umtxq_chain *umtxq_getchain(struct umtx_key *key);
231 static void umtxq_lock(struct umtx_key *key);
232 static void umtxq_unlock(struct umtx_key *key);
233 static void umtxq_busy(struct umtx_key *key);
234 static void umtxq_unbusy(struct umtx_key *key);
235 static void umtxq_insert_queue(struct umtx_q *uq, int q);
236 static void umtxq_remove_queue(struct umtx_q *uq, int q);
237 static int umtxq_sleep(struct umtx_q *uq, const char *wmesg, int timo);
238 static int umtxq_count(struct umtx_key *key);
239 static int umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2);
240 static int umtx_key_get(void *addr, int type, int share,
241 struct umtx_key *key);
242 static void umtx_key_release(struct umtx_key *key);
243 static struct umtx_pi *umtx_pi_alloc(int);
244 static void umtx_pi_free(struct umtx_pi *pi);
245 static void umtx_pi_adjust_locked(struct thread *td, u_char oldpri);
246 static int do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags);
247 static void umtx_thread_cleanup(struct thread *td);
248 static void umtx_exec_hook(void *arg __unused, struct proc *p __unused,
249 struct image_params *imgp __unused);
250 SYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_sysinit, NULL);
252 #define umtxq_signal(key, nwake) umtxq_signal_queue((key), (nwake), UMTX_SHARED_QUEUE)
253 #define umtxq_insert(uq) umtxq_insert_queue((uq), UMTX_SHARED_QUEUE)
254 #define umtxq_remove(uq) umtxq_remove_queue((uq), UMTX_SHARED_QUEUE)
256 static struct mtx umtx_lock;
259 umtxq_sysinit(void *arg __unused)
263 umtx_pi_zone = uma_zcreate("umtx pi", sizeof(struct umtx_pi),
264 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
265 for (i = 0; i < 2; ++i) {
266 for (j = 0; j < UMTX_CHAINS; ++j) {
267 mtx_init(&umtxq_chains[i][j].uc_lock, "umtxql", NULL,
268 MTX_DEF | MTX_DUPOK);
269 LIST_INIT(&umtxq_chains[i][j].uc_queue[0]);
270 LIST_INIT(&umtxq_chains[i][j].uc_queue[1]);
271 LIST_INIT(&umtxq_chains[i][j].uc_spare_queue);
272 TAILQ_INIT(&umtxq_chains[i][j].uc_pi_list);
273 umtxq_chains[i][j].uc_busy = 0;
274 umtxq_chains[i][j].uc_waiters = 0;
277 mtx_init(&umtx_lock, "umtx lock", NULL, MTX_SPIN);
278 EVENTHANDLER_REGISTER(process_exec, umtx_exec_hook, NULL,
279 EVENTHANDLER_PRI_ANY);
287 uq = malloc(sizeof(struct umtx_q), M_UMTX, M_WAITOK | M_ZERO);
288 uq->uq_spare_queue = malloc(sizeof(struct umtxq_queue), M_UMTX, M_WAITOK | M_ZERO);
289 TAILQ_INIT(&uq->uq_spare_queue->head);
290 TAILQ_INIT(&uq->uq_pi_contested);
291 uq->uq_inherited_pri = PRI_MAX;
296 umtxq_free(struct umtx_q *uq)
298 MPASS(uq->uq_spare_queue != NULL);
299 free(uq->uq_spare_queue, M_UMTX);
304 umtxq_hash(struct umtx_key *key)
306 unsigned n = (uintptr_t)key->info.both.a + key->info.both.b;
307 key->hash = ((n * GOLDEN_RATIO_PRIME) >> UMTX_SHIFTS) % UMTX_CHAINS;
311 umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2)
313 return (k1->type == k2->type &&
314 k1->info.both.a == k2->info.both.a &&
315 k1->info.both.b == k2->info.both.b);
318 static inline struct umtxq_chain *
319 umtxq_getchain(struct umtx_key *key)
321 if (key->type <= TYPE_SEM)
322 return (&umtxq_chains[1][key->hash]);
323 return (&umtxq_chains[0][key->hash]);
330 umtxq_lock(struct umtx_key *key)
332 struct umtxq_chain *uc;
334 uc = umtxq_getchain(key);
335 mtx_lock(&uc->uc_lock);
342 umtxq_unlock(struct umtx_key *key)
344 struct umtxq_chain *uc;
346 uc = umtxq_getchain(key);
347 mtx_unlock(&uc->uc_lock);
351 * Set chain to busy state when following operation
352 * may be blocked (kernel mutex can not be used).
355 umtxq_busy(struct umtx_key *key)
357 struct umtxq_chain *uc;
359 uc = umtxq_getchain(key);
360 mtx_assert(&uc->uc_lock, MA_OWNED);
364 int count = BUSY_SPINS;
367 while (uc->uc_busy && --count > 0)
373 while (uc->uc_busy) {
375 msleep(uc, &uc->uc_lock, 0, "umtxqb", 0);
386 umtxq_unbusy(struct umtx_key *key)
388 struct umtxq_chain *uc;
390 uc = umtxq_getchain(key);
391 mtx_assert(&uc->uc_lock, MA_OWNED);
392 KASSERT(uc->uc_busy != 0, ("not busy"));
398 static struct umtxq_queue *
399 umtxq_queue_lookup(struct umtx_key *key, int q)
401 struct umtxq_queue *uh;
402 struct umtxq_chain *uc;
404 uc = umtxq_getchain(key);
405 UMTXQ_LOCKED_ASSERT(uc);
406 LIST_FOREACH(uh, &uc->uc_queue[q], link) {
407 if (umtx_key_match(&uh->key, key))
415 umtxq_insert_queue(struct umtx_q *uq, int q)
417 struct umtxq_queue *uh;
418 struct umtxq_chain *uc;
420 uc = umtxq_getchain(&uq->uq_key);
421 UMTXQ_LOCKED_ASSERT(uc);
422 KASSERT((uq->uq_flags & UQF_UMTXQ) == 0, ("umtx_q is already on queue"));
423 uh = umtxq_queue_lookup(&uq->uq_key, q);
425 LIST_INSERT_HEAD(&uc->uc_spare_queue, uq->uq_spare_queue, link);
427 uh = uq->uq_spare_queue;
428 uh->key = uq->uq_key;
429 LIST_INSERT_HEAD(&uc->uc_queue[q], uh, link);
431 uq->uq_spare_queue = NULL;
433 TAILQ_INSERT_TAIL(&uh->head, uq, uq_link);
435 uq->uq_flags |= UQF_UMTXQ;
436 uq->uq_cur_queue = uh;
441 umtxq_remove_queue(struct umtx_q *uq, int q)
443 struct umtxq_chain *uc;
444 struct umtxq_queue *uh;
446 uc = umtxq_getchain(&uq->uq_key);
447 UMTXQ_LOCKED_ASSERT(uc);
448 if (uq->uq_flags & UQF_UMTXQ) {
449 uh = uq->uq_cur_queue;
450 TAILQ_REMOVE(&uh->head, uq, uq_link);
452 uq->uq_flags &= ~UQF_UMTXQ;
453 if (TAILQ_EMPTY(&uh->head)) {
454 KASSERT(uh->length == 0,
455 ("inconsistent umtxq_queue length"));
456 LIST_REMOVE(uh, link);
458 uh = LIST_FIRST(&uc->uc_spare_queue);
459 KASSERT(uh != NULL, ("uc_spare_queue is empty"));
460 LIST_REMOVE(uh, link);
462 uq->uq_spare_queue = uh;
463 uq->uq_cur_queue = NULL;
468 * Check if there are multiple waiters
471 umtxq_count(struct umtx_key *key)
473 struct umtxq_chain *uc;
474 struct umtxq_queue *uh;
476 uc = umtxq_getchain(key);
477 UMTXQ_LOCKED_ASSERT(uc);
478 uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE);
485 * Check if there are multiple PI waiters and returns first
489 umtxq_count_pi(struct umtx_key *key, struct umtx_q **first)
491 struct umtxq_chain *uc;
492 struct umtxq_queue *uh;
495 uc = umtxq_getchain(key);
496 UMTXQ_LOCKED_ASSERT(uc);
497 uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE);
499 *first = TAILQ_FIRST(&uh->head);
506 * Wake up threads waiting on an userland object.
510 umtxq_signal_queue(struct umtx_key *key, int n_wake, int q)
512 struct umtxq_chain *uc;
513 struct umtxq_queue *uh;
518 uc = umtxq_getchain(key);
519 UMTXQ_LOCKED_ASSERT(uc);
520 uh = umtxq_queue_lookup(key, q);
522 while ((uq = TAILQ_FIRST(&uh->head)) != NULL) {
523 umtxq_remove_queue(uq, q);
534 * Wake up specified thread.
537 umtxq_signal_thread(struct umtx_q *uq)
539 struct umtxq_chain *uc;
541 uc = umtxq_getchain(&uq->uq_key);
542 UMTXQ_LOCKED_ASSERT(uc);
548 * Put thread into sleep state, before sleeping, check if
549 * thread was removed from umtx queue.
552 umtxq_sleep(struct umtx_q *uq, const char *wmesg, int timo)
554 struct umtxq_chain *uc;
557 uc = umtxq_getchain(&uq->uq_key);
558 UMTXQ_LOCKED_ASSERT(uc);
559 if (!(uq->uq_flags & UQF_UMTXQ))
561 error = msleep(uq, &uc->uc_lock, PCATCH, wmesg, timo);
562 if (error == EWOULDBLOCK)
568 * Convert userspace address into unique logical address.
571 umtx_key_get(void *addr, int type, int share, struct umtx_key *key)
573 struct thread *td = curthread;
575 vm_map_entry_t entry;
581 if (share == THREAD_SHARE) {
583 key->info.private.vs = td->td_proc->p_vmspace;
584 key->info.private.addr = (uintptr_t)addr;
586 MPASS(share == PROCESS_SHARE || share == AUTO_SHARE);
587 map = &td->td_proc->p_vmspace->vm_map;
588 if (vm_map_lookup(&map, (vm_offset_t)addr, VM_PROT_WRITE,
589 &entry, &key->info.shared.object, &pindex, &prot,
590 &wired) != KERN_SUCCESS) {
594 if ((share == PROCESS_SHARE) ||
595 (share == AUTO_SHARE &&
596 VM_INHERIT_SHARE == entry->inheritance)) {
598 key->info.shared.offset = entry->offset + entry->start -
600 vm_object_reference(key->info.shared.object);
603 key->info.private.vs = td->td_proc->p_vmspace;
604 key->info.private.addr = (uintptr_t)addr;
606 vm_map_lookup_done(map, entry);
617 umtx_key_release(struct umtx_key *key)
620 vm_object_deallocate(key->info.shared.object);
624 * Lock a umtx object.
627 _do_lock_umtx(struct thread *td, struct umtx *umtx, u_long id, int timo)
637 * Care must be exercised when dealing with umtx structure. It
638 * can fault on any access.
642 * Try the uncontested case. This should be done in userland.
644 owner = casuword(&umtx->u_owner, UMTX_UNOWNED, id);
646 /* The acquire succeeded. */
647 if (owner == UMTX_UNOWNED)
650 /* The address was invalid. */
654 /* If no one owns it but it is contested try to acquire it. */
655 if (owner == UMTX_CONTESTED) {
656 owner = casuword(&umtx->u_owner,
657 UMTX_CONTESTED, id | UMTX_CONTESTED);
659 if (owner == UMTX_CONTESTED)
662 /* The address was invalid. */
666 /* If this failed the lock has changed, restart. */
671 * If we caught a signal, we have retried and now
677 if ((error = umtx_key_get(umtx, TYPE_SIMPLE_LOCK,
678 AUTO_SHARE, &uq->uq_key)) != 0)
681 umtxq_lock(&uq->uq_key);
682 umtxq_busy(&uq->uq_key);
684 umtxq_unbusy(&uq->uq_key);
685 umtxq_unlock(&uq->uq_key);
688 * Set the contested bit so that a release in user space
689 * knows to use the system call for unlock. If this fails
690 * either some one else has acquired the lock or it has been
693 old = casuword(&umtx->u_owner, owner, owner | UMTX_CONTESTED);
695 /* The address was invalid. */
697 umtxq_lock(&uq->uq_key);
699 umtxq_unlock(&uq->uq_key);
700 umtx_key_release(&uq->uq_key);
705 * We set the contested bit, sleep. Otherwise the lock changed
706 * and we need to retry or we lost a race to the thread
707 * unlocking the umtx.
709 umtxq_lock(&uq->uq_key);
711 error = umtxq_sleep(uq, "umtx", timo);
713 umtxq_unlock(&uq->uq_key);
714 umtx_key_release(&uq->uq_key);
721 * Lock a umtx object.
724 do_lock_umtx(struct thread *td, struct umtx *umtx, u_long id,
725 struct timespec *timeout)
727 struct timespec ts, ts2, ts3;
731 if (timeout == NULL) {
732 error = _do_lock_umtx(td, umtx, id, 0);
733 /* Mutex locking is restarted if it is interrupted. */
738 timespecadd(&ts, timeout);
739 TIMESPEC_TO_TIMEVAL(&tv, timeout);
741 error = _do_lock_umtx(td, umtx, id, tvtohz(&tv));
742 if (error != ETIMEDOUT)
745 if (timespeccmp(&ts2, &ts, >=)) {
750 timespecsub(&ts3, &ts2);
751 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
753 /* Timed-locking is not restarted. */
754 if (error == ERESTART)
761 * Unlock a umtx object.
764 do_unlock_umtx(struct thread *td, struct umtx *umtx, u_long id)
773 * Make sure we own this mtx.
775 owner = fuword(__DEVOLATILE(u_long *, &umtx->u_owner));
779 if ((owner & ~UMTX_CONTESTED) != id)
782 /* This should be done in userland */
783 if ((owner & UMTX_CONTESTED) == 0) {
784 old = casuword(&umtx->u_owner, owner, UMTX_UNOWNED);
792 /* We should only ever be in here for contested locks */
793 if ((error = umtx_key_get(umtx, TYPE_SIMPLE_LOCK, AUTO_SHARE,
799 count = umtxq_count(&key);
803 * When unlocking the umtx, it must be marked as unowned if
804 * there is zero or one thread only waiting for it.
805 * Otherwise, it must be marked as contested.
807 old = casuword(&umtx->u_owner, owner,
808 count <= 1 ? UMTX_UNOWNED : UMTX_CONTESTED);
810 umtxq_signal(&key,1);
813 umtx_key_release(&key);
821 #ifdef COMPAT_FREEBSD32
824 * Lock a umtx object.
827 _do_lock_umtx32(struct thread *td, uint32_t *m, uint32_t id, int timo)
837 * Care must be exercised when dealing with umtx structure. It
838 * can fault on any access.
842 * Try the uncontested case. This should be done in userland.
844 owner = casuword32(m, UMUTEX_UNOWNED, id);
846 /* The acquire succeeded. */
847 if (owner == UMUTEX_UNOWNED)
850 /* The address was invalid. */
854 /* If no one owns it but it is contested try to acquire it. */
855 if (owner == UMUTEX_CONTESTED) {
856 owner = casuword32(m,
857 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
858 if (owner == UMUTEX_CONTESTED)
861 /* The address was invalid. */
865 /* If this failed the lock has changed, restart. */
870 * If we caught a signal, we have retried and now
876 if ((error = umtx_key_get(m, TYPE_SIMPLE_LOCK,
877 AUTO_SHARE, &uq->uq_key)) != 0)
880 umtxq_lock(&uq->uq_key);
881 umtxq_busy(&uq->uq_key);
883 umtxq_unbusy(&uq->uq_key);
884 umtxq_unlock(&uq->uq_key);
887 * Set the contested bit so that a release in user space
888 * knows to use the system call for unlock. If this fails
889 * either some one else has acquired the lock or it has been
892 old = casuword32(m, owner, owner | UMUTEX_CONTESTED);
894 /* The address was invalid. */
896 umtxq_lock(&uq->uq_key);
898 umtxq_unlock(&uq->uq_key);
899 umtx_key_release(&uq->uq_key);
904 * We set the contested bit, sleep. Otherwise the lock changed
905 * and we need to retry or we lost a race to the thread
906 * unlocking the umtx.
908 umtxq_lock(&uq->uq_key);
910 error = umtxq_sleep(uq, "umtx", timo);
912 umtxq_unlock(&uq->uq_key);
913 umtx_key_release(&uq->uq_key);
920 * Lock a umtx object.
923 do_lock_umtx32(struct thread *td, void *m, uint32_t id,
924 struct timespec *timeout)
926 struct timespec ts, ts2, ts3;
930 if (timeout == NULL) {
931 error = _do_lock_umtx32(td, m, id, 0);
932 /* Mutex locking is restarted if it is interrupted. */
937 timespecadd(&ts, timeout);
938 TIMESPEC_TO_TIMEVAL(&tv, timeout);
940 error = _do_lock_umtx32(td, m, id, tvtohz(&tv));
941 if (error != ETIMEDOUT)
944 if (timespeccmp(&ts2, &ts, >=)) {
949 timespecsub(&ts3, &ts2);
950 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
952 /* Timed-locking is not restarted. */
953 if (error == ERESTART)
960 * Unlock a umtx object.
963 do_unlock_umtx32(struct thread *td, uint32_t *m, uint32_t id)
972 * Make sure we own this mtx.
978 if ((owner & ~UMUTEX_CONTESTED) != id)
981 /* This should be done in userland */
982 if ((owner & UMUTEX_CONTESTED) == 0) {
983 old = casuword32(m, owner, UMUTEX_UNOWNED);
991 /* We should only ever be in here for contested locks */
992 if ((error = umtx_key_get(m, TYPE_SIMPLE_LOCK, AUTO_SHARE,
998 count = umtxq_count(&key);
1002 * When unlocking the umtx, it must be marked as unowned if
1003 * there is zero or one thread only waiting for it.
1004 * Otherwise, it must be marked as contested.
1006 old = casuword32(m, owner,
1007 count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
1009 umtxq_signal(&key,1);
1012 umtx_key_release(&key);
1022 * Fetch and compare value, sleep on the address if value is not changed.
1025 do_wait(struct thread *td, void *addr, u_long id,
1026 struct timespec *timeout, int compat32, int is_private)
1029 struct timespec ts, ts2, ts3;
1035 if ((error = umtx_key_get(addr, TYPE_SIMPLE_WAIT,
1036 is_private ? THREAD_SHARE : AUTO_SHARE, &uq->uq_key)) != 0)
1039 umtxq_lock(&uq->uq_key);
1041 umtxq_unlock(&uq->uq_key);
1045 tmp = (unsigned int)fuword32(addr);
1047 umtxq_lock(&uq->uq_key);
1049 umtxq_unlock(&uq->uq_key);
1050 } else if (timeout == NULL) {
1051 umtxq_lock(&uq->uq_key);
1052 error = umtxq_sleep(uq, "uwait", 0);
1054 umtxq_unlock(&uq->uq_key);
1057 timespecadd(&ts, timeout);
1058 TIMESPEC_TO_TIMEVAL(&tv, timeout);
1059 umtxq_lock(&uq->uq_key);
1061 error = umtxq_sleep(uq, "uwait", tvtohz(&tv));
1062 if (!(uq->uq_flags & UQF_UMTXQ)) {
1066 if (error != ETIMEDOUT)
1068 umtxq_unlock(&uq->uq_key);
1069 getnanouptime(&ts2);
1070 if (timespeccmp(&ts2, &ts, >=)) {
1072 umtxq_lock(&uq->uq_key);
1076 timespecsub(&ts3, &ts2);
1077 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
1078 umtxq_lock(&uq->uq_key);
1081 umtxq_unlock(&uq->uq_key);
1083 umtx_key_release(&uq->uq_key);
1084 if (error == ERESTART)
1090 * Wake up threads sleeping on the specified address.
1093 kern_umtx_wake(struct thread *td, void *uaddr, int n_wake, int is_private)
1095 struct umtx_key key;
1098 if ((ret = umtx_key_get(uaddr, TYPE_SIMPLE_WAIT,
1099 is_private ? THREAD_SHARE : AUTO_SHARE, &key)) != 0)
1102 ret = umtxq_signal(&key, n_wake);
1104 umtx_key_release(&key);
1109 * Lock PTHREAD_PRIO_NONE protocol POSIX mutex.
1112 _do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, int timo,
1116 uint32_t owner, old, id;
1123 * Care must be exercised when dealing with umtx structure. It
1124 * can fault on any access.
1127 owner = fuword32(__DEVOLATILE(void *, &m->m_owner));
1128 if (mode == _UMUTEX_WAIT) {
1129 if (owner == UMUTEX_UNOWNED || owner == UMUTEX_CONTESTED)
1133 * Try the uncontested case. This should be done in userland.
1135 owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id);
1137 /* The acquire succeeded. */
1138 if (owner == UMUTEX_UNOWNED)
1141 /* The address was invalid. */
1145 /* If no one owns it but it is contested try to acquire it. */
1146 if (owner == UMUTEX_CONTESTED) {
1147 owner = casuword32(&m->m_owner,
1148 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
1150 if (owner == UMUTEX_CONTESTED)
1153 /* The address was invalid. */
1157 /* If this failed the lock has changed, restart. */
1162 if ((flags & UMUTEX_ERROR_CHECK) != 0 &&
1163 (owner & ~UMUTEX_CONTESTED) == id)
1166 if (mode == _UMUTEX_TRY)
1170 * If we caught a signal, we have retried and now
1176 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX,
1177 GET_SHARE(flags), &uq->uq_key)) != 0)
1180 umtxq_lock(&uq->uq_key);
1181 umtxq_busy(&uq->uq_key);
1183 umtxq_unlock(&uq->uq_key);
1186 * Set the contested bit so that a release in user space
1187 * knows to use the system call for unlock. If this fails
1188 * either some one else has acquired the lock or it has been
1191 old = casuword32(&m->m_owner, owner, owner | UMUTEX_CONTESTED);
1193 /* The address was invalid. */
1195 umtxq_lock(&uq->uq_key);
1197 umtxq_unbusy(&uq->uq_key);
1198 umtxq_unlock(&uq->uq_key);
1199 umtx_key_release(&uq->uq_key);
1204 * We set the contested bit, sleep. Otherwise the lock changed
1205 * and we need to retry or we lost a race to the thread
1206 * unlocking the umtx.
1208 umtxq_lock(&uq->uq_key);
1209 umtxq_unbusy(&uq->uq_key);
1211 error = umtxq_sleep(uq, "umtxn", timo);
1213 umtxq_unlock(&uq->uq_key);
1214 umtx_key_release(&uq->uq_key);
1221 * Lock PTHREAD_PRIO_NONE protocol POSIX mutex.
1224 * Unlock PTHREAD_PRIO_NONE protocol POSIX mutex.
1227 do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags)
1229 struct umtx_key key;
1230 uint32_t owner, old, id;
1236 * Make sure we own this mtx.
1238 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
1242 if ((owner & ~UMUTEX_CONTESTED) != id)
1245 if ((owner & UMUTEX_CONTESTED) == 0) {
1246 old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED);
1254 /* We should only ever be in here for contested locks */
1255 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags),
1261 count = umtxq_count(&key);
1265 * When unlocking the umtx, it must be marked as unowned if
1266 * there is zero or one thread only waiting for it.
1267 * Otherwise, it must be marked as contested.
1269 old = casuword32(&m->m_owner, owner,
1270 count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
1272 umtxq_signal(&key,1);
1275 umtx_key_release(&key);
1284 * Check if the mutex is available and wake up a waiter,
1285 * only for simple mutex.
1288 do_wake_umutex(struct thread *td, struct umutex *m)
1290 struct umtx_key key;
1296 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
1300 if ((owner & ~UMUTEX_CONTESTED) != 0)
1303 flags = fuword32(&m->m_flags);
1305 /* We should only ever be in here for contested locks */
1306 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags),
1312 count = umtxq_count(&key);
1316 owner = casuword32(&m->m_owner, UMUTEX_CONTESTED, UMUTEX_UNOWNED);
1319 if (count != 0 && (owner & ~UMUTEX_CONTESTED) == 0)
1320 umtxq_signal(&key, 1);
1323 umtx_key_release(&key);
1327 static inline struct umtx_pi *
1328 umtx_pi_alloc(int flags)
1332 pi = uma_zalloc(umtx_pi_zone, M_ZERO | flags);
1333 TAILQ_INIT(&pi->pi_blocked);
1334 atomic_add_int(&umtx_pi_allocated, 1);
1339 umtx_pi_free(struct umtx_pi *pi)
1341 uma_zfree(umtx_pi_zone, pi);
1342 atomic_add_int(&umtx_pi_allocated, -1);
1346 * Adjust the thread's position on a pi_state after its priority has been
1350 umtx_pi_adjust_thread(struct umtx_pi *pi, struct thread *td)
1352 struct umtx_q *uq, *uq1, *uq2;
1355 mtx_assert(&umtx_lock, MA_OWNED);
1362 * Check if the thread needs to be moved on the blocked chain.
1363 * It needs to be moved if either its priority is lower than
1364 * the previous thread or higher than the next thread.
1366 uq1 = TAILQ_PREV(uq, umtxq_head, uq_lockq);
1367 uq2 = TAILQ_NEXT(uq, uq_lockq);
1368 if ((uq1 != NULL && UPRI(td) < UPRI(uq1->uq_thread)) ||
1369 (uq2 != NULL && UPRI(td) > UPRI(uq2->uq_thread))) {
1371 * Remove thread from blocked chain and determine where
1372 * it should be moved to.
1374 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
1375 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) {
1376 td1 = uq1->uq_thread;
1377 MPASS(td1->td_proc->p_magic == P_MAGIC);
1378 if (UPRI(td1) > UPRI(td))
1383 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq);
1385 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq);
1391 * Propagate priority when a thread is blocked on POSIX
1395 umtx_propagate_priority(struct thread *td)
1401 mtx_assert(&umtx_lock, MA_OWNED);
1404 pi = uq->uq_pi_blocked;
1413 MPASS(td->td_proc != NULL);
1414 MPASS(td->td_proc->p_magic == P_MAGIC);
1416 if (UPRI(td) <= pri)
1420 sched_lend_user_prio(td, pri);
1424 * Pick up the lock that td is blocked on.
1427 pi = uq->uq_pi_blocked;
1428 /* Resort td on the list if needed. */
1429 if (!umtx_pi_adjust_thread(pi, td))
1435 * Unpropagate priority for a PI mutex when a thread blocked on
1436 * it is interrupted by signal or resumed by others.
1439 umtx_unpropagate_priority(struct umtx_pi *pi)
1441 struct umtx_q *uq, *uq_owner;
1442 struct umtx_pi *pi2;
1445 mtx_assert(&umtx_lock, MA_OWNED);
1447 while (pi != NULL && pi->pi_owner != NULL) {
1449 uq_owner = pi->pi_owner->td_umtxq;
1451 TAILQ_FOREACH(pi2, &uq_owner->uq_pi_contested, pi_link) {
1452 uq = TAILQ_FIRST(&pi2->pi_blocked);
1454 if (pri > UPRI(uq->uq_thread))
1455 pri = UPRI(uq->uq_thread);
1459 if (pri > uq_owner->uq_inherited_pri)
1460 pri = uq_owner->uq_inherited_pri;
1461 thread_lock(pi->pi_owner);
1462 oldpri = pi->pi_owner->td_user_pri;
1463 sched_unlend_user_prio(pi->pi_owner, pri);
1464 thread_unlock(pi->pi_owner);
1465 if (uq_owner->uq_pi_blocked != NULL)
1466 umtx_pi_adjust_locked(pi->pi_owner, oldpri);
1467 pi = uq_owner->uq_pi_blocked;
1472 * Insert a PI mutex into owned list.
1475 umtx_pi_setowner(struct umtx_pi *pi, struct thread *owner)
1477 struct umtx_q *uq_owner;
1479 uq_owner = owner->td_umtxq;
1480 mtx_assert(&umtx_lock, MA_OWNED);
1481 if (pi->pi_owner != NULL)
1482 panic("pi_ower != NULL");
1483 pi->pi_owner = owner;
1484 TAILQ_INSERT_TAIL(&uq_owner->uq_pi_contested, pi, pi_link);
1488 * Claim ownership of a PI mutex.
1491 umtx_pi_claim(struct umtx_pi *pi, struct thread *owner)
1493 struct umtx_q *uq, *uq_owner;
1495 uq_owner = owner->td_umtxq;
1496 mtx_lock_spin(&umtx_lock);
1497 if (pi->pi_owner == owner) {
1498 mtx_unlock_spin(&umtx_lock);
1502 if (pi->pi_owner != NULL) {
1504 * userland may have already messed the mutex, sigh.
1506 mtx_unlock_spin(&umtx_lock);
1509 umtx_pi_setowner(pi, owner);
1510 uq = TAILQ_FIRST(&pi->pi_blocked);
1514 pri = UPRI(uq->uq_thread);
1516 if (pri < UPRI(owner))
1517 sched_lend_user_prio(owner, pri);
1518 thread_unlock(owner);
1520 mtx_unlock_spin(&umtx_lock);
1525 umtx_pi_adjust_locked(struct thread *td, u_char oldpri)
1532 * Pick up the lock that td is blocked on.
1534 pi = uq->uq_pi_blocked;
1537 /* Resort the turnstile on the list. */
1538 if (!umtx_pi_adjust_thread(pi, td))
1542 * If our priority was lowered and we are at the head of the
1543 * turnstile, then propagate our new priority up the chain.
1545 if (uq == TAILQ_FIRST(&pi->pi_blocked) && UPRI(td) < oldpri)
1546 umtx_propagate_priority(td);
1550 * Adjust a thread's order position in its blocked PI mutex,
1551 * this may result new priority propagating process.
1554 umtx_pi_adjust(struct thread *td, u_char oldpri)
1560 mtx_lock_spin(&umtx_lock);
1562 * Pick up the lock that td is blocked on.
1564 pi = uq->uq_pi_blocked;
1566 umtx_pi_adjust_locked(td, oldpri);
1567 mtx_unlock_spin(&umtx_lock);
1571 * Sleep on a PI mutex.
1574 umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi,
1575 uint32_t owner, const char *wmesg, int timo)
1577 struct umtxq_chain *uc;
1578 struct thread *td, *td1;
1584 KASSERT(td == curthread, ("inconsistent uq_thread"));
1585 uc = umtxq_getchain(&uq->uq_key);
1586 UMTXQ_LOCKED_ASSERT(uc);
1587 UMTXQ_BUSY_ASSERT(uc);
1589 mtx_lock_spin(&umtx_lock);
1590 if (pi->pi_owner == NULL) {
1591 mtx_unlock_spin(&umtx_lock);
1592 /* XXX Only look up thread in current process. */
1593 td1 = tdfind(owner, curproc->p_pid);
1594 mtx_lock_spin(&umtx_lock);
1596 if (pi->pi_owner == NULL)
1597 umtx_pi_setowner(pi, td1);
1598 PROC_UNLOCK(td1->td_proc);
1602 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) {
1603 pri = UPRI(uq1->uq_thread);
1609 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq);
1611 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq);
1613 uq->uq_pi_blocked = pi;
1615 td->td_flags |= TDF_UPIBLOCKED;
1617 umtx_propagate_priority(td);
1618 mtx_unlock_spin(&umtx_lock);
1619 umtxq_unbusy(&uq->uq_key);
1621 if (uq->uq_flags & UQF_UMTXQ) {
1622 error = msleep(uq, &uc->uc_lock, PCATCH, wmesg, timo);
1623 if (error == EWOULDBLOCK)
1625 if (uq->uq_flags & UQF_UMTXQ) {
1629 mtx_lock_spin(&umtx_lock);
1630 uq->uq_pi_blocked = NULL;
1632 td->td_flags &= ~TDF_UPIBLOCKED;
1634 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
1635 umtx_unpropagate_priority(pi);
1636 mtx_unlock_spin(&umtx_lock);
1637 umtxq_unlock(&uq->uq_key);
1643 * Add reference count for a PI mutex.
1646 umtx_pi_ref(struct umtx_pi *pi)
1648 struct umtxq_chain *uc;
1650 uc = umtxq_getchain(&pi->pi_key);
1651 UMTXQ_LOCKED_ASSERT(uc);
1656 * Decrease reference count for a PI mutex, if the counter
1657 * is decreased to zero, its memory space is freed.
1660 umtx_pi_unref(struct umtx_pi *pi)
1662 struct umtxq_chain *uc;
1664 uc = umtxq_getchain(&pi->pi_key);
1665 UMTXQ_LOCKED_ASSERT(uc);
1666 KASSERT(pi->pi_refcount > 0, ("invalid reference count"));
1667 if (--pi->pi_refcount == 0) {
1668 mtx_lock_spin(&umtx_lock);
1669 if (pi->pi_owner != NULL) {
1670 TAILQ_REMOVE(&pi->pi_owner->td_umtxq->uq_pi_contested,
1672 pi->pi_owner = NULL;
1674 KASSERT(TAILQ_EMPTY(&pi->pi_blocked),
1675 ("blocked queue not empty"));
1676 mtx_unlock_spin(&umtx_lock);
1677 TAILQ_REMOVE(&uc->uc_pi_list, pi, pi_hashlink);
1683 * Find a PI mutex in hash table.
1685 static struct umtx_pi *
1686 umtx_pi_lookup(struct umtx_key *key)
1688 struct umtxq_chain *uc;
1691 uc = umtxq_getchain(key);
1692 UMTXQ_LOCKED_ASSERT(uc);
1694 TAILQ_FOREACH(pi, &uc->uc_pi_list, pi_hashlink) {
1695 if (umtx_key_match(&pi->pi_key, key)) {
1703 * Insert a PI mutex into hash table.
1706 umtx_pi_insert(struct umtx_pi *pi)
1708 struct umtxq_chain *uc;
1710 uc = umtxq_getchain(&pi->pi_key);
1711 UMTXQ_LOCKED_ASSERT(uc);
1712 TAILQ_INSERT_TAIL(&uc->uc_pi_list, pi, pi_hashlink);
1719 _do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, int timo,
1723 struct umtx_pi *pi, *new_pi;
1724 uint32_t id, owner, old;
1730 if ((error = umtx_key_get(m, TYPE_PI_UMUTEX, GET_SHARE(flags),
1733 umtxq_lock(&uq->uq_key);
1734 pi = umtx_pi_lookup(&uq->uq_key);
1736 new_pi = umtx_pi_alloc(M_NOWAIT);
1737 if (new_pi == NULL) {
1738 umtxq_unlock(&uq->uq_key);
1739 new_pi = umtx_pi_alloc(M_WAITOK);
1740 umtxq_lock(&uq->uq_key);
1741 pi = umtx_pi_lookup(&uq->uq_key);
1743 umtx_pi_free(new_pi);
1747 if (new_pi != NULL) {
1748 new_pi->pi_key = uq->uq_key;
1749 umtx_pi_insert(new_pi);
1754 umtxq_unlock(&uq->uq_key);
1757 * Care must be exercised when dealing with umtx structure. It
1758 * can fault on any access.
1762 * Try the uncontested case. This should be done in userland.
1764 owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id);
1766 /* The acquire succeeded. */
1767 if (owner == UMUTEX_UNOWNED) {
1772 /* The address was invalid. */
1778 /* If no one owns it but it is contested try to acquire it. */
1779 if (owner == UMUTEX_CONTESTED) {
1780 owner = casuword32(&m->m_owner,
1781 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
1783 if (owner == UMUTEX_CONTESTED) {
1784 umtxq_lock(&uq->uq_key);
1785 umtxq_busy(&uq->uq_key);
1786 error = umtx_pi_claim(pi, td);
1787 umtxq_unbusy(&uq->uq_key);
1788 umtxq_unlock(&uq->uq_key);
1792 /* The address was invalid. */
1798 /* If this failed the lock has changed, restart. */
1802 if ((flags & UMUTEX_ERROR_CHECK) != 0 &&
1803 (owner & ~UMUTEX_CONTESTED) == id) {
1814 * If we caught a signal, we have retried and now
1820 umtxq_lock(&uq->uq_key);
1821 umtxq_busy(&uq->uq_key);
1822 umtxq_unlock(&uq->uq_key);
1825 * Set the contested bit so that a release in user space
1826 * knows to use the system call for unlock. If this fails
1827 * either some one else has acquired the lock or it has been
1830 old = casuword32(&m->m_owner, owner, owner | UMUTEX_CONTESTED);
1832 /* The address was invalid. */
1834 umtxq_lock(&uq->uq_key);
1835 umtxq_unbusy(&uq->uq_key);
1836 umtxq_unlock(&uq->uq_key);
1841 umtxq_lock(&uq->uq_key);
1843 * We set the contested bit, sleep. Otherwise the lock changed
1844 * and we need to retry or we lost a race to the thread
1845 * unlocking the umtx.
1848 error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED,
1851 umtxq_unbusy(&uq->uq_key);
1852 umtxq_unlock(&uq->uq_key);
1856 umtxq_lock(&uq->uq_key);
1858 umtxq_unlock(&uq->uq_key);
1860 umtx_key_release(&uq->uq_key);
1865 * Unlock a PI mutex.
1868 do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags)
1870 struct umtx_key key;
1871 struct umtx_q *uq_first, *uq_first2, *uq_me;
1872 struct umtx_pi *pi, *pi2;
1873 uint32_t owner, old, id;
1880 * Make sure we own this mtx.
1882 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
1886 if ((owner & ~UMUTEX_CONTESTED) != id)
1889 /* This should be done in userland */
1890 if ((owner & UMUTEX_CONTESTED) == 0) {
1891 old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED);
1899 /* We should only ever be in here for contested locks */
1900 if ((error = umtx_key_get(m, TYPE_PI_UMUTEX, GET_SHARE(flags),
1906 count = umtxq_count_pi(&key, &uq_first);
1907 if (uq_first != NULL) {
1908 mtx_lock_spin(&umtx_lock);
1909 pi = uq_first->uq_pi_blocked;
1910 KASSERT(pi != NULL, ("pi == NULL?"));
1911 if (pi->pi_owner != curthread) {
1912 mtx_unlock_spin(&umtx_lock);
1915 umtx_key_release(&key);
1916 /* userland messed the mutex */
1919 uq_me = curthread->td_umtxq;
1920 pi->pi_owner = NULL;
1921 TAILQ_REMOVE(&uq_me->uq_pi_contested, pi, pi_link);
1922 /* get highest priority thread which is still sleeping. */
1923 uq_first = TAILQ_FIRST(&pi->pi_blocked);
1924 while (uq_first != NULL &&
1925 (uq_first->uq_flags & UQF_UMTXQ) == 0) {
1926 uq_first = TAILQ_NEXT(uq_first, uq_lockq);
1929 TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) {
1930 uq_first2 = TAILQ_FIRST(&pi2->pi_blocked);
1931 if (uq_first2 != NULL) {
1932 if (pri > UPRI(uq_first2->uq_thread))
1933 pri = UPRI(uq_first2->uq_thread);
1936 thread_lock(curthread);
1937 sched_unlend_user_prio(curthread, pri);
1938 thread_unlock(curthread);
1939 mtx_unlock_spin(&umtx_lock);
1941 umtxq_signal_thread(uq_first);
1946 * When unlocking the umtx, it must be marked as unowned if
1947 * there is zero or one thread only waiting for it.
1948 * Otherwise, it must be marked as contested.
1950 old = casuword32(&m->m_owner, owner,
1951 count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
1956 umtx_key_release(&key);
1968 _do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, int timo,
1971 struct umtx_q *uq, *uq2;
1975 int error, pri, old_inherited_pri, su;
1979 if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags),
1982 su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0);
1984 old_inherited_pri = uq->uq_inherited_pri;
1985 umtxq_lock(&uq->uq_key);
1986 umtxq_busy(&uq->uq_key);
1987 umtxq_unlock(&uq->uq_key);
1989 ceiling = RTP_PRIO_MAX - fuword32(&m->m_ceilings[0]);
1990 if (ceiling > RTP_PRIO_MAX) {
1995 mtx_lock_spin(&umtx_lock);
1996 if (UPRI(td) < PRI_MIN_REALTIME + ceiling) {
1997 mtx_unlock_spin(&umtx_lock);
2001 if (su && PRI_MIN_REALTIME + ceiling < uq->uq_inherited_pri) {
2002 uq->uq_inherited_pri = PRI_MIN_REALTIME + ceiling;
2004 if (uq->uq_inherited_pri < UPRI(td))
2005 sched_lend_user_prio(td, uq->uq_inherited_pri);
2008 mtx_unlock_spin(&umtx_lock);
2010 owner = casuword32(&m->m_owner,
2011 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
2013 if (owner == UMUTEX_CONTESTED) {
2018 /* The address was invalid. */
2024 if ((flags & UMUTEX_ERROR_CHECK) != 0 &&
2025 (owner & ~UMUTEX_CONTESTED) == id) {
2036 * If we caught a signal, we have retried and now
2042 umtxq_lock(&uq->uq_key);
2044 umtxq_unbusy(&uq->uq_key);
2045 error = umtxq_sleep(uq, "umtxpp", timo);
2047 umtxq_unlock(&uq->uq_key);
2049 mtx_lock_spin(&umtx_lock);
2050 uq->uq_inherited_pri = old_inherited_pri;
2052 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2053 uq2 = TAILQ_FIRST(&pi->pi_blocked);
2055 if (pri > UPRI(uq2->uq_thread))
2056 pri = UPRI(uq2->uq_thread);
2059 if (pri > uq->uq_inherited_pri)
2060 pri = uq->uq_inherited_pri;
2062 sched_unlend_user_prio(td, pri);
2064 mtx_unlock_spin(&umtx_lock);
2068 mtx_lock_spin(&umtx_lock);
2069 uq->uq_inherited_pri = old_inherited_pri;
2071 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2072 uq2 = TAILQ_FIRST(&pi->pi_blocked);
2074 if (pri > UPRI(uq2->uq_thread))
2075 pri = UPRI(uq2->uq_thread);
2078 if (pri > uq->uq_inherited_pri)
2079 pri = uq->uq_inherited_pri;
2081 sched_unlend_user_prio(td, pri);
2083 mtx_unlock_spin(&umtx_lock);
2087 umtxq_lock(&uq->uq_key);
2088 umtxq_unbusy(&uq->uq_key);
2089 umtxq_unlock(&uq->uq_key);
2090 umtx_key_release(&uq->uq_key);
2095 * Unlock a PP mutex.
2098 do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags)
2100 struct umtx_key key;
2101 struct umtx_q *uq, *uq2;
2105 int error, pri, new_inherited_pri, su;
2109 su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0);
2112 * Make sure we own this mtx.
2114 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
2118 if ((owner & ~UMUTEX_CONTESTED) != id)
2121 error = copyin(&m->m_ceilings[1], &rceiling, sizeof(uint32_t));
2126 new_inherited_pri = PRI_MAX;
2128 rceiling = RTP_PRIO_MAX - rceiling;
2129 if (rceiling > RTP_PRIO_MAX)
2131 new_inherited_pri = PRI_MIN_REALTIME + rceiling;
2134 if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags),
2141 * For priority protected mutex, always set unlocked state
2142 * to UMUTEX_CONTESTED, so that userland always enters kernel
2143 * to lock the mutex, it is necessary because thread priority
2144 * has to be adjusted for such mutex.
2146 error = suword32(__DEVOLATILE(uint32_t *, &m->m_owner),
2151 umtxq_signal(&key, 1);
2158 mtx_lock_spin(&umtx_lock);
2160 uq->uq_inherited_pri = new_inherited_pri;
2162 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2163 uq2 = TAILQ_FIRST(&pi->pi_blocked);
2165 if (pri > UPRI(uq2->uq_thread))
2166 pri = UPRI(uq2->uq_thread);
2169 if (pri > uq->uq_inherited_pri)
2170 pri = uq->uq_inherited_pri;
2172 sched_unlend_user_prio(td, pri);
2174 mtx_unlock_spin(&umtx_lock);
2176 umtx_key_release(&key);
2181 do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling,
2182 uint32_t *old_ceiling)
2185 uint32_t save_ceiling;
2190 flags = fuword32(&m->m_flags);
2191 if ((flags & UMUTEX_PRIO_PROTECT) == 0)
2193 if (ceiling > RTP_PRIO_MAX)
2197 if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags),
2201 umtxq_lock(&uq->uq_key);
2202 umtxq_busy(&uq->uq_key);
2203 umtxq_unlock(&uq->uq_key);
2205 save_ceiling = fuword32(&m->m_ceilings[0]);
2207 owner = casuword32(&m->m_owner,
2208 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
2210 if (owner == UMUTEX_CONTESTED) {
2211 suword32(&m->m_ceilings[0], ceiling);
2212 suword32(__DEVOLATILE(uint32_t *, &m->m_owner),
2218 /* The address was invalid. */
2224 if ((owner & ~UMUTEX_CONTESTED) == id) {
2225 suword32(&m->m_ceilings[0], ceiling);
2231 * If we caught a signal, we have retried and now
2238 * We set the contested bit, sleep. Otherwise the lock changed
2239 * and we need to retry or we lost a race to the thread
2240 * unlocking the umtx.
2242 umtxq_lock(&uq->uq_key);
2244 umtxq_unbusy(&uq->uq_key);
2245 error = umtxq_sleep(uq, "umtxpp", 0);
2247 umtxq_unlock(&uq->uq_key);
2249 umtxq_lock(&uq->uq_key);
2251 umtxq_signal(&uq->uq_key, INT_MAX);
2252 umtxq_unbusy(&uq->uq_key);
2253 umtxq_unlock(&uq->uq_key);
2254 umtx_key_release(&uq->uq_key);
2255 if (error == 0 && old_ceiling != NULL)
2256 suword32(old_ceiling, save_ceiling);
2261 _do_lock_umutex(struct thread *td, struct umutex *m, int flags, int timo,
2264 switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) {
2266 return (_do_lock_normal(td, m, flags, timo, mode));
2267 case UMUTEX_PRIO_INHERIT:
2268 return (_do_lock_pi(td, m, flags, timo, mode));
2269 case UMUTEX_PRIO_PROTECT:
2270 return (_do_lock_pp(td, m, flags, timo, mode));
2276 * Lock a userland POSIX mutex.
2279 do_lock_umutex(struct thread *td, struct umutex *m,
2280 struct timespec *timeout, int mode)
2282 struct timespec ts, ts2, ts3;
2287 flags = fuword32(&m->m_flags);
2291 if (timeout == NULL) {
2292 error = _do_lock_umutex(td, m, flags, 0, mode);
2293 /* Mutex locking is restarted if it is interrupted. */
2294 if (error == EINTR && mode != _UMUTEX_WAIT)
2298 timespecadd(&ts, timeout);
2299 TIMESPEC_TO_TIMEVAL(&tv, timeout);
2301 error = _do_lock_umutex(td, m, flags, tvtohz(&tv), mode);
2302 if (error != ETIMEDOUT)
2304 getnanouptime(&ts2);
2305 if (timespeccmp(&ts2, &ts, >=)) {
2310 timespecsub(&ts3, &ts2);
2311 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
2313 /* Timed-locking is not restarted. */
2314 if (error == ERESTART)
2321 * Unlock a userland POSIX mutex.
2324 do_unlock_umutex(struct thread *td, struct umutex *m)
2328 flags = fuword32(&m->m_flags);
2332 switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) {
2334 return (do_unlock_normal(td, m, flags));
2335 case UMUTEX_PRIO_INHERIT:
2336 return (do_unlock_pi(td, m, flags));
2337 case UMUTEX_PRIO_PROTECT:
2338 return (do_unlock_pp(td, m, flags));
2345 do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m,
2346 struct timespec *timeout, u_long wflags)
2350 struct timespec cts, ets, tts;
2355 flags = fuword32(&cv->c_flags);
2356 error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key);
2359 umtxq_lock(&uq->uq_key);
2360 umtxq_busy(&uq->uq_key);
2362 umtxq_unlock(&uq->uq_key);
2365 * The magic thing is we should set c_has_waiters to 1 before
2366 * releasing user mutex.
2368 suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 1);
2370 umtxq_lock(&uq->uq_key);
2371 umtxq_unbusy(&uq->uq_key);
2372 umtxq_unlock(&uq->uq_key);
2374 error = do_unlock_umutex(td, m);
2376 umtxq_lock(&uq->uq_key);
2378 if ((wflags & UMTX_CHECK_UNPARKING) &&
2379 (td->td_pflags & TDP_WAKEUP)) {
2380 td->td_pflags &= ~TDP_WAKEUP;
2382 } else if (timeout == NULL) {
2383 error = umtxq_sleep(uq, "ucond", 0);
2385 getnanouptime(&ets);
2386 timespecadd(&ets, timeout);
2387 TIMESPEC_TO_TIMEVAL(&tv, timeout);
2389 error = umtxq_sleep(uq, "ucond", tvtohz(&tv));
2390 if (error != ETIMEDOUT)
2392 getnanouptime(&cts);
2393 if (timespeccmp(&cts, &ets, >=)) {
2398 timespecsub(&tts, &cts);
2399 TIMESPEC_TO_TIMEVAL(&tv, &tts);
2404 if ((uq->uq_flags & UQF_UMTXQ) == 0)
2408 if (error == ERESTART)
2412 umtxq_unlock(&uq->uq_key);
2413 umtx_key_release(&uq->uq_key);
2418 * Signal a userland condition variable.
2421 do_cv_signal(struct thread *td, struct ucond *cv)
2423 struct umtx_key key;
2424 int error, cnt, nwake;
2427 flags = fuword32(&cv->c_flags);
2428 if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0)
2432 cnt = umtxq_count(&key);
2433 nwake = umtxq_signal(&key, 1);
2437 __DEVOLATILE(uint32_t *, &cv->c_has_waiters), 0);
2442 umtx_key_release(&key);
2447 do_cv_broadcast(struct thread *td, struct ucond *cv)
2449 struct umtx_key key;
2453 flags = fuword32(&cv->c_flags);
2454 if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0)
2459 umtxq_signal(&key, INT_MAX);
2462 error = suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 0);
2468 umtx_key_release(&key);
2473 do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag, int timo)
2476 uint32_t flags, wrflags;
2477 int32_t state, oldstate;
2478 int32_t blocked_readers;
2482 flags = fuword32(&rwlock->rw_flags);
2483 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
2487 wrflags = URWLOCK_WRITE_OWNER;
2488 if (!(fflag & URWLOCK_PREFER_READER) && !(flags & URWLOCK_PREFER_READER))
2489 wrflags |= URWLOCK_WRITE_WAITERS;
2492 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2493 /* try to lock it */
2494 while (!(state & wrflags)) {
2495 if (__predict_false(URWLOCK_READER_COUNT(state) == URWLOCK_MAX_READERS)) {
2496 umtx_key_release(&uq->uq_key);
2499 oldstate = casuword32(&rwlock->rw_state, state, state + 1);
2500 if (oldstate == state) {
2501 umtx_key_release(&uq->uq_key);
2510 /* grab monitor lock */
2511 umtxq_lock(&uq->uq_key);
2512 umtxq_busy(&uq->uq_key);
2513 umtxq_unlock(&uq->uq_key);
2516 * re-read the state, in case it changed between the try-lock above
2517 * and the check below
2519 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2521 /* set read contention bit */
2522 while ((state & wrflags) && !(state & URWLOCK_READ_WAITERS)) {
2523 oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_READ_WAITERS);
2524 if (oldstate == state)
2529 /* state is changed while setting flags, restart */
2530 if (!(state & wrflags)) {
2531 umtxq_lock(&uq->uq_key);
2532 umtxq_unbusy(&uq->uq_key);
2533 umtxq_unlock(&uq->uq_key);
2538 /* contention bit is set, before sleeping, increase read waiter count */
2539 blocked_readers = fuword32(&rwlock->rw_blocked_readers);
2540 suword32(&rwlock->rw_blocked_readers, blocked_readers+1);
2542 while (state & wrflags) {
2543 umtxq_lock(&uq->uq_key);
2545 umtxq_unbusy(&uq->uq_key);
2547 error = umtxq_sleep(uq, "urdlck", timo);
2549 umtxq_busy(&uq->uq_key);
2551 umtxq_unlock(&uq->uq_key);
2554 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2557 /* decrease read waiter count, and may clear read contention bit */
2558 blocked_readers = fuword32(&rwlock->rw_blocked_readers);
2559 suword32(&rwlock->rw_blocked_readers, blocked_readers-1);
2560 if (blocked_readers == 1) {
2561 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2563 oldstate = casuword32(&rwlock->rw_state, state,
2564 state & ~URWLOCK_READ_WAITERS);
2565 if (oldstate == state)
2571 umtxq_lock(&uq->uq_key);
2572 umtxq_unbusy(&uq->uq_key);
2573 umtxq_unlock(&uq->uq_key);
2575 umtx_key_release(&uq->uq_key);
2580 do_rw_rdlock2(struct thread *td, void *obj, long val, struct timespec *timeout)
2582 struct timespec ts, ts2, ts3;
2587 timespecadd(&ts, timeout);
2588 TIMESPEC_TO_TIMEVAL(&tv, timeout);
2590 error = do_rw_rdlock(td, obj, val, tvtohz(&tv));
2591 if (error != ETIMEDOUT)
2593 getnanouptime(&ts2);
2594 if (timespeccmp(&ts2, &ts, >=)) {
2599 timespecsub(&ts3, &ts2);
2600 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
2602 if (error == ERESTART)
2608 do_rw_wrlock(struct thread *td, struct urwlock *rwlock, int timo)
2612 int32_t state, oldstate;
2613 int32_t blocked_writers;
2614 int32_t blocked_readers;
2618 flags = fuword32(&rwlock->rw_flags);
2619 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
2623 blocked_readers = 0;
2625 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2626 while (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) {
2627 oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_WRITE_OWNER);
2628 if (oldstate == state) {
2629 umtx_key_release(&uq->uq_key);
2636 if (!(state & (URWLOCK_WRITE_OWNER|URWLOCK_WRITE_WAITERS)) &&
2637 blocked_readers != 0) {
2638 umtxq_lock(&uq->uq_key);
2639 umtxq_busy(&uq->uq_key);
2640 umtxq_signal_queue(&uq->uq_key, INT_MAX, UMTX_SHARED_QUEUE);
2641 umtxq_unbusy(&uq->uq_key);
2642 umtxq_unlock(&uq->uq_key);
2648 /* grab monitor lock */
2649 umtxq_lock(&uq->uq_key);
2650 umtxq_busy(&uq->uq_key);
2651 umtxq_unlock(&uq->uq_key);
2654 * re-read the state, in case it changed between the try-lock above
2655 * and the check below
2657 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2659 while (((state & URWLOCK_WRITE_OWNER) || URWLOCK_READER_COUNT(state) != 0) &&
2660 (state & URWLOCK_WRITE_WAITERS) == 0) {
2661 oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_WRITE_WAITERS);
2662 if (oldstate == state)
2667 if (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) {
2668 umtxq_lock(&uq->uq_key);
2669 umtxq_unbusy(&uq->uq_key);
2670 umtxq_unlock(&uq->uq_key);
2674 blocked_writers = fuword32(&rwlock->rw_blocked_writers);
2675 suword32(&rwlock->rw_blocked_writers, blocked_writers+1);
2677 while ((state & URWLOCK_WRITE_OWNER) || URWLOCK_READER_COUNT(state) != 0) {
2678 umtxq_lock(&uq->uq_key);
2679 umtxq_insert_queue(uq, UMTX_EXCLUSIVE_QUEUE);
2680 umtxq_unbusy(&uq->uq_key);
2682 error = umtxq_sleep(uq, "uwrlck", timo);
2684 umtxq_busy(&uq->uq_key);
2685 umtxq_remove_queue(uq, UMTX_EXCLUSIVE_QUEUE);
2686 umtxq_unlock(&uq->uq_key);
2689 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2692 blocked_writers = fuword32(&rwlock->rw_blocked_writers);
2693 suword32(&rwlock->rw_blocked_writers, blocked_writers-1);
2694 if (blocked_writers == 1) {
2695 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2697 oldstate = casuword32(&rwlock->rw_state, state,
2698 state & ~URWLOCK_WRITE_WAITERS);
2699 if (oldstate == state)
2703 blocked_readers = fuword32(&rwlock->rw_blocked_readers);
2705 blocked_readers = 0;
2707 umtxq_lock(&uq->uq_key);
2708 umtxq_unbusy(&uq->uq_key);
2709 umtxq_unlock(&uq->uq_key);
2712 umtx_key_release(&uq->uq_key);
2717 do_rw_wrlock2(struct thread *td, void *obj, struct timespec *timeout)
2719 struct timespec ts, ts2, ts3;
2724 timespecadd(&ts, timeout);
2725 TIMESPEC_TO_TIMEVAL(&tv, timeout);
2727 error = do_rw_wrlock(td, obj, tvtohz(&tv));
2728 if (error != ETIMEDOUT)
2730 getnanouptime(&ts2);
2731 if (timespeccmp(&ts2, &ts, >=)) {
2736 timespecsub(&ts3, &ts2);
2737 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
2739 if (error == ERESTART)
2745 do_rw_unlock(struct thread *td, struct urwlock *rwlock)
2749 int32_t state, oldstate;
2750 int error, q, count;
2753 flags = fuword32(&rwlock->rw_flags);
2754 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
2758 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2759 if (state & URWLOCK_WRITE_OWNER) {
2761 oldstate = casuword32(&rwlock->rw_state, state,
2762 state & ~URWLOCK_WRITE_OWNER);
2763 if (oldstate != state) {
2765 if (!(oldstate & URWLOCK_WRITE_OWNER)) {
2772 } else if (URWLOCK_READER_COUNT(state) != 0) {
2774 oldstate = casuword32(&rwlock->rw_state, state,
2776 if (oldstate != state) {
2778 if (URWLOCK_READER_COUNT(oldstate) == 0) {
2793 if (!(flags & URWLOCK_PREFER_READER)) {
2794 if (state & URWLOCK_WRITE_WAITERS) {
2796 q = UMTX_EXCLUSIVE_QUEUE;
2797 } else if (state & URWLOCK_READ_WAITERS) {
2799 q = UMTX_SHARED_QUEUE;
2802 if (state & URWLOCK_READ_WAITERS) {
2804 q = UMTX_SHARED_QUEUE;
2805 } else if (state & URWLOCK_WRITE_WAITERS) {
2807 q = UMTX_EXCLUSIVE_QUEUE;
2812 umtxq_lock(&uq->uq_key);
2813 umtxq_busy(&uq->uq_key);
2814 umtxq_signal_queue(&uq->uq_key, count, q);
2815 umtxq_unbusy(&uq->uq_key);
2816 umtxq_unlock(&uq->uq_key);
2819 umtx_key_release(&uq->uq_key);
2824 do_sem_wait(struct thread *td, struct _usem *sem, struct timespec *timeout)
2828 struct timespec cts, ets, tts;
2829 uint32_t flags, count;
2833 flags = fuword32(&sem->_flags);
2834 error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key);
2837 umtxq_lock(&uq->uq_key);
2838 umtxq_busy(&uq->uq_key);
2840 umtxq_unlock(&uq->uq_key);
2842 if (fuword32(__DEVOLATILE(uint32_t *, &sem->_has_waiters)) == 0)
2843 casuword32(__DEVOLATILE(uint32_t *, &sem->_has_waiters), 0, 1);
2845 count = fuword32(__DEVOLATILE(uint32_t *, &sem->_count));
2847 umtxq_lock(&uq->uq_key);
2848 umtxq_unbusy(&uq->uq_key);
2850 umtxq_unlock(&uq->uq_key);
2851 umtx_key_release(&uq->uq_key);
2855 umtxq_lock(&uq->uq_key);
2856 umtxq_unbusy(&uq->uq_key);
2857 umtxq_unlock(&uq->uq_key);
2859 umtxq_lock(&uq->uq_key);
2860 if (timeout == NULL) {
2861 error = umtxq_sleep(uq, "usem", 0);
2863 getnanouptime(&ets);
2864 timespecadd(&ets, timeout);
2865 TIMESPEC_TO_TIMEVAL(&tv, timeout);
2867 error = umtxq_sleep(uq, "usem", tvtohz(&tv));
2868 if (error != ETIMEDOUT)
2870 getnanouptime(&cts);
2871 if (timespeccmp(&cts, &ets, >=)) {
2876 timespecsub(&tts, &cts);
2877 TIMESPEC_TO_TIMEVAL(&tv, &tts);
2881 if ((uq->uq_flags & UQF_UMTXQ) == 0)
2885 if (error == ERESTART)
2888 umtxq_unlock(&uq->uq_key);
2889 umtx_key_release(&uq->uq_key);
2894 * Signal a userland condition variable.
2897 do_sem_wake(struct thread *td, struct _usem *sem)
2899 struct umtx_key key;
2900 int error, cnt, nwake;
2903 flags = fuword32(&sem->_flags);
2904 if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0)
2908 cnt = umtxq_count(&key);
2909 nwake = umtxq_signal(&key, 1);
2913 __DEVOLATILE(uint32_t *, &sem->_has_waiters), 0);
2918 umtx_key_release(&key);
2923 _umtx_lock(struct thread *td, struct _umtx_lock_args *uap)
2924 /* struct umtx *umtx */
2926 return _do_lock_umtx(td, uap->umtx, td->td_tid, 0);
2930 _umtx_unlock(struct thread *td, struct _umtx_unlock_args *uap)
2931 /* struct umtx *umtx */
2933 return do_unlock_umtx(td, uap->umtx, td->td_tid);
2937 __umtx_op_lock_umtx(struct thread *td, struct _umtx_op_args *uap)
2939 struct timespec *ts, timeout;
2942 /* Allow a null timespec (wait forever). */
2943 if (uap->uaddr2 == NULL)
2946 error = copyin(uap->uaddr2, &timeout, sizeof(timeout));
2949 if (timeout.tv_nsec >= 1000000000 ||
2950 timeout.tv_nsec < 0) {
2955 return (do_lock_umtx(td, uap->obj, uap->val, ts));
2959 __umtx_op_unlock_umtx(struct thread *td, struct _umtx_op_args *uap)
2961 return (do_unlock_umtx(td, uap->obj, uap->val));
2965 __umtx_op_wait(struct thread *td, struct _umtx_op_args *uap)
2967 struct timespec *ts, timeout;
2970 if (uap->uaddr2 == NULL)
2973 error = copyin(uap->uaddr2, &timeout, sizeof(timeout));
2976 if (timeout.tv_nsec >= 1000000000 ||
2977 timeout.tv_nsec < 0)
2981 return do_wait(td, uap->obj, uap->val, ts, 0, 0);
2985 __umtx_op_wait_uint(struct thread *td, struct _umtx_op_args *uap)
2987 struct timespec *ts, timeout;
2990 if (uap->uaddr2 == NULL)
2993 error = copyin(uap->uaddr2, &timeout, sizeof(timeout));
2996 if (timeout.tv_nsec >= 1000000000 ||
2997 timeout.tv_nsec < 0)
3001 return do_wait(td, uap->obj, uap->val, ts, 1, 0);
3005 __umtx_op_wait_uint_private(struct thread *td, struct _umtx_op_args *uap)
3007 struct timespec *ts, timeout;
3010 if (uap->uaddr2 == NULL)
3013 error = copyin(uap->uaddr2, &timeout, sizeof(timeout));
3016 if (timeout.tv_nsec >= 1000000000 ||
3017 timeout.tv_nsec < 0)
3021 return do_wait(td, uap->obj, uap->val, ts, 1, 1);
3025 __umtx_op_wake(struct thread *td, struct _umtx_op_args *uap)
3027 return (kern_umtx_wake(td, uap->obj, uap->val, 0));
3031 __umtx_op_wake_private(struct thread *td, struct _umtx_op_args *uap)
3033 return (kern_umtx_wake(td, uap->obj, uap->val, 1));
3037 __umtx_op_lock_umutex(struct thread *td, struct _umtx_op_args *uap)
3039 struct timespec *ts, timeout;
3042 /* Allow a null timespec (wait forever). */
3043 if (uap->uaddr2 == NULL)
3046 error = copyin(uap->uaddr2, &timeout,
3050 if (timeout.tv_nsec >= 1000000000 ||
3051 timeout.tv_nsec < 0) {
3056 return do_lock_umutex(td, uap->obj, ts, 0);
3060 __umtx_op_trylock_umutex(struct thread *td, struct _umtx_op_args *uap)
3062 return do_lock_umutex(td, uap->obj, NULL, _UMUTEX_TRY);
3066 __umtx_op_wait_umutex(struct thread *td, struct _umtx_op_args *uap)
3068 struct timespec *ts, timeout;
3071 /* Allow a null timespec (wait forever). */
3072 if (uap->uaddr2 == NULL)
3075 error = copyin(uap->uaddr2, &timeout,
3079 if (timeout.tv_nsec >= 1000000000 ||
3080 timeout.tv_nsec < 0) {
3085 return do_lock_umutex(td, uap->obj, ts, _UMUTEX_WAIT);
3089 __umtx_op_wake_umutex(struct thread *td, struct _umtx_op_args *uap)
3091 return do_wake_umutex(td, uap->obj);
3095 __umtx_op_unlock_umutex(struct thread *td, struct _umtx_op_args *uap)
3097 return do_unlock_umutex(td, uap->obj);
3101 __umtx_op_set_ceiling(struct thread *td, struct _umtx_op_args *uap)
3103 return do_set_ceiling(td, uap->obj, uap->val, uap->uaddr1);
3107 __umtx_op_cv_wait(struct thread *td, struct _umtx_op_args *uap)
3109 struct timespec *ts, timeout;
3112 /* Allow a null timespec (wait forever). */
3113 if (uap->uaddr2 == NULL)
3116 error = copyin(uap->uaddr2, &timeout,
3120 if (timeout.tv_nsec >= 1000000000 ||
3121 timeout.tv_nsec < 0) {
3126 return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val));
3130 __umtx_op_cv_signal(struct thread *td, struct _umtx_op_args *uap)
3132 return do_cv_signal(td, uap->obj);
3136 __umtx_op_cv_broadcast(struct thread *td, struct _umtx_op_args *uap)
3138 return do_cv_broadcast(td, uap->obj);
3142 __umtx_op_rw_rdlock(struct thread *td, struct _umtx_op_args *uap)
3144 struct timespec timeout;
3147 /* Allow a null timespec (wait forever). */
3148 if (uap->uaddr2 == NULL) {
3149 error = do_rw_rdlock(td, uap->obj, uap->val, 0);
3151 error = copyin(uap->uaddr2, &timeout,
3155 if (timeout.tv_nsec >= 1000000000 ||
3156 timeout.tv_nsec < 0) {
3159 error = do_rw_rdlock2(td, uap->obj, uap->val, &timeout);
3165 __umtx_op_rw_wrlock(struct thread *td, struct _umtx_op_args *uap)
3167 struct timespec timeout;
3170 /* Allow a null timespec (wait forever). */
3171 if (uap->uaddr2 == NULL) {
3172 error = do_rw_wrlock(td, uap->obj, 0);
3174 error = copyin(uap->uaddr2, &timeout,
3178 if (timeout.tv_nsec >= 1000000000 ||
3179 timeout.tv_nsec < 0) {
3183 error = do_rw_wrlock2(td, uap->obj, &timeout);
3189 __umtx_op_rw_unlock(struct thread *td, struct _umtx_op_args *uap)
3191 return do_rw_unlock(td, uap->obj);
3195 __umtx_op_sem_wait(struct thread *td, struct _umtx_op_args *uap)
3197 struct timespec *ts, timeout;
3200 /* Allow a null timespec (wait forever). */
3201 if (uap->uaddr2 == NULL)
3204 error = copyin(uap->uaddr2, &timeout,
3208 if (timeout.tv_nsec >= 1000000000 ||
3209 timeout.tv_nsec < 0) {
3214 return (do_sem_wait(td, uap->obj, ts));
3218 __umtx_op_sem_wake(struct thread *td, struct _umtx_op_args *uap)
3220 return do_sem_wake(td, uap->obj);
3223 typedef int (*_umtx_op_func)(struct thread *td, struct _umtx_op_args *uap);
3225 static _umtx_op_func op_table[] = {
3226 __umtx_op_lock_umtx, /* UMTX_OP_LOCK */
3227 __umtx_op_unlock_umtx, /* UMTX_OP_UNLOCK */
3228 __umtx_op_wait, /* UMTX_OP_WAIT */
3229 __umtx_op_wake, /* UMTX_OP_WAKE */
3230 __umtx_op_trylock_umutex, /* UMTX_OP_MUTEX_TRYLOCK */
3231 __umtx_op_lock_umutex, /* UMTX_OP_MUTEX_LOCK */
3232 __umtx_op_unlock_umutex, /* UMTX_OP_MUTEX_UNLOCK */
3233 __umtx_op_set_ceiling, /* UMTX_OP_SET_CEILING */
3234 __umtx_op_cv_wait, /* UMTX_OP_CV_WAIT*/
3235 __umtx_op_cv_signal, /* UMTX_OP_CV_SIGNAL */
3236 __umtx_op_cv_broadcast, /* UMTX_OP_CV_BROADCAST */
3237 __umtx_op_wait_uint, /* UMTX_OP_WAIT_UINT */
3238 __umtx_op_rw_rdlock, /* UMTX_OP_RW_RDLOCK */
3239 __umtx_op_rw_wrlock, /* UMTX_OP_RW_WRLOCK */
3240 __umtx_op_rw_unlock, /* UMTX_OP_RW_UNLOCK */
3241 __umtx_op_wait_uint_private, /* UMTX_OP_WAIT_UINT_PRIVATE */
3242 __umtx_op_wake_private, /* UMTX_OP_WAKE_PRIVATE */
3243 __umtx_op_wait_umutex, /* UMTX_OP_UMUTEX_WAIT */
3244 __umtx_op_wake_umutex, /* UMTX_OP_UMUTEX_WAKE */
3245 __umtx_op_sem_wait, /* UMTX_OP_SEM_WAIT */
3246 __umtx_op_sem_wake /* UMTX_OP_SEM_WAKE */
3250 _umtx_op(struct thread *td, struct _umtx_op_args *uap)
3252 if ((unsigned)uap->op < UMTX_OP_MAX)
3253 return (*op_table[uap->op])(td, uap);
3257 #ifdef COMPAT_FREEBSD32
3259 freebsd32_umtx_lock(struct thread *td, struct freebsd32_umtx_lock_args *uap)
3260 /* struct umtx *umtx */
3262 return (do_lock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid, NULL));
3266 freebsd32_umtx_unlock(struct thread *td, struct freebsd32_umtx_unlock_args *uap)
3267 /* struct umtx *umtx */
3269 return (do_unlock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid));
3278 copyin_timeout32(void *addr, struct timespec *tsp)
3280 struct timespec32 ts32;
3283 error = copyin(addr, &ts32, sizeof(struct timespec32));
3285 tsp->tv_sec = ts32.tv_sec;
3286 tsp->tv_nsec = ts32.tv_nsec;
3292 __umtx_op_lock_umtx_compat32(struct thread *td, struct _umtx_op_args *uap)
3294 struct timespec *ts, timeout;
3297 /* Allow a null timespec (wait forever). */
3298 if (uap->uaddr2 == NULL)
3301 error = copyin_timeout32(uap->uaddr2, &timeout);
3304 if (timeout.tv_nsec >= 1000000000 ||
3305 timeout.tv_nsec < 0) {
3310 return (do_lock_umtx32(td, uap->obj, uap->val, ts));
3314 __umtx_op_unlock_umtx_compat32(struct thread *td, struct _umtx_op_args *uap)
3316 return (do_unlock_umtx32(td, uap->obj, (uint32_t)uap->val));
3320 __umtx_op_wait_compat32(struct thread *td, struct _umtx_op_args *uap)
3322 struct timespec *ts, timeout;
3325 if (uap->uaddr2 == NULL)
3328 error = copyin_timeout32(uap->uaddr2, &timeout);
3331 if (timeout.tv_nsec >= 1000000000 ||
3332 timeout.tv_nsec < 0)
3336 return do_wait(td, uap->obj, uap->val, ts, 1, 0);
3340 __umtx_op_lock_umutex_compat32(struct thread *td, struct _umtx_op_args *uap)
3342 struct timespec *ts, timeout;
3345 /* Allow a null timespec (wait forever). */
3346 if (uap->uaddr2 == NULL)
3349 error = copyin_timeout32(uap->uaddr2, &timeout);
3352 if (timeout.tv_nsec >= 1000000000 ||
3353 timeout.tv_nsec < 0)
3357 return do_lock_umutex(td, uap->obj, ts, 0);
3361 __umtx_op_wait_umutex_compat32(struct thread *td, struct _umtx_op_args *uap)
3363 struct timespec *ts, timeout;
3366 /* Allow a null timespec (wait forever). */
3367 if (uap->uaddr2 == NULL)
3370 error = copyin_timeout32(uap->uaddr2, &timeout);
3373 if (timeout.tv_nsec >= 1000000000 ||
3374 timeout.tv_nsec < 0)
3378 return do_lock_umutex(td, uap->obj, ts, _UMUTEX_WAIT);
3382 __umtx_op_cv_wait_compat32(struct thread *td, struct _umtx_op_args *uap)
3384 struct timespec *ts, timeout;
3387 /* Allow a null timespec (wait forever). */
3388 if (uap->uaddr2 == NULL)
3391 error = copyin_timeout32(uap->uaddr2, &timeout);
3394 if (timeout.tv_nsec >= 1000000000 ||
3395 timeout.tv_nsec < 0)
3399 return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val));
3403 __umtx_op_rw_rdlock_compat32(struct thread *td, struct _umtx_op_args *uap)
3405 struct timespec timeout;
3408 /* Allow a null timespec (wait forever). */
3409 if (uap->uaddr2 == NULL) {
3410 error = do_rw_rdlock(td, uap->obj, uap->val, 0);
3412 error = copyin(uap->uaddr2, &timeout,
3416 if (timeout.tv_nsec >= 1000000000 ||
3417 timeout.tv_nsec < 0) {
3420 error = do_rw_rdlock2(td, uap->obj, uap->val, &timeout);
3426 __umtx_op_rw_wrlock_compat32(struct thread *td, struct _umtx_op_args *uap)
3428 struct timespec timeout;
3431 /* Allow a null timespec (wait forever). */
3432 if (uap->uaddr2 == NULL) {
3433 error = do_rw_wrlock(td, uap->obj, 0);
3435 error = copyin_timeout32(uap->uaddr2, &timeout);
3438 if (timeout.tv_nsec >= 1000000000 ||
3439 timeout.tv_nsec < 0) {
3443 error = do_rw_wrlock2(td, uap->obj, &timeout);
3449 __umtx_op_wait_uint_private_compat32(struct thread *td, struct _umtx_op_args *uap)
3451 struct timespec *ts, timeout;
3454 if (uap->uaddr2 == NULL)
3457 error = copyin_timeout32(uap->uaddr2, &timeout);
3460 if (timeout.tv_nsec >= 1000000000 ||
3461 timeout.tv_nsec < 0)
3465 return do_wait(td, uap->obj, uap->val, ts, 1, 1);
3469 __umtx_op_sem_wait_compat32(struct thread *td, struct _umtx_op_args *uap)
3471 struct timespec *ts, timeout;
3474 /* Allow a null timespec (wait forever). */
3475 if (uap->uaddr2 == NULL)
3478 error = copyin_timeout32(uap->uaddr2, &timeout);
3481 if (timeout.tv_nsec >= 1000000000 ||
3482 timeout.tv_nsec < 0)
3486 return (do_sem_wait(td, uap->obj, ts));
3489 static _umtx_op_func op_table_compat32[] = {
3490 __umtx_op_lock_umtx_compat32, /* UMTX_OP_LOCK */
3491 __umtx_op_unlock_umtx_compat32, /* UMTX_OP_UNLOCK */
3492 __umtx_op_wait_compat32, /* UMTX_OP_WAIT */
3493 __umtx_op_wake, /* UMTX_OP_WAKE */
3494 __umtx_op_trylock_umutex, /* UMTX_OP_MUTEX_LOCK */
3495 __umtx_op_lock_umutex_compat32, /* UMTX_OP_MUTEX_TRYLOCK */
3496 __umtx_op_unlock_umutex, /* UMTX_OP_MUTEX_UNLOCK */
3497 __umtx_op_set_ceiling, /* UMTX_OP_SET_CEILING */
3498 __umtx_op_cv_wait_compat32, /* UMTX_OP_CV_WAIT*/
3499 __umtx_op_cv_signal, /* UMTX_OP_CV_SIGNAL */
3500 __umtx_op_cv_broadcast, /* UMTX_OP_CV_BROADCAST */
3501 __umtx_op_wait_compat32, /* UMTX_OP_WAIT_UINT */
3502 __umtx_op_rw_rdlock_compat32, /* UMTX_OP_RW_RDLOCK */
3503 __umtx_op_rw_wrlock_compat32, /* UMTX_OP_RW_WRLOCK */
3504 __umtx_op_rw_unlock, /* UMTX_OP_RW_UNLOCK */
3505 __umtx_op_wait_uint_private_compat32, /* UMTX_OP_WAIT_UINT_PRIVATE */
3506 __umtx_op_wake_private, /* UMTX_OP_WAKE_PRIVATE */
3507 __umtx_op_wait_umutex_compat32, /* UMTX_OP_UMUTEX_WAIT */
3508 __umtx_op_wake_umutex, /* UMTX_OP_UMUTEX_WAKE */
3509 __umtx_op_sem_wait_compat32, /* UMTX_OP_SEM_WAIT */
3510 __umtx_op_sem_wake /* UMTX_OP_SEM_WAKE */
3514 freebsd32_umtx_op(struct thread *td, struct freebsd32_umtx_op_args *uap)
3516 if ((unsigned)uap->op < UMTX_OP_MAX)
3517 return (*op_table_compat32[uap->op])(td,
3518 (struct _umtx_op_args *)uap);
3524 umtx_thread_init(struct thread *td)
3526 td->td_umtxq = umtxq_alloc();
3527 td->td_umtxq->uq_thread = td;
3531 umtx_thread_fini(struct thread *td)
3533 umtxq_free(td->td_umtxq);
3537 * It will be called when new thread is created, e.g fork().
3540 umtx_thread_alloc(struct thread *td)
3545 uq->uq_inherited_pri = PRI_MAX;
3547 KASSERT(uq->uq_flags == 0, ("uq_flags != 0"));
3548 KASSERT(uq->uq_thread == td, ("uq_thread != td"));
3549 KASSERT(uq->uq_pi_blocked == NULL, ("uq_pi_blocked != NULL"));
3550 KASSERT(TAILQ_EMPTY(&uq->uq_pi_contested), ("uq_pi_contested is not empty"));
3557 umtx_exec_hook(void *arg __unused, struct proc *p __unused,
3558 struct image_params *imgp __unused)
3560 umtx_thread_cleanup(curthread);
3564 * thread_exit() hook.
3567 umtx_thread_exit(struct thread *td)
3569 umtx_thread_cleanup(td);
3573 * clean up umtx data.
3576 umtx_thread_cleanup(struct thread *td)
3581 if ((uq = td->td_umtxq) == NULL)
3584 mtx_lock_spin(&umtx_lock);
3585 uq->uq_inherited_pri = PRI_MAX;
3586 while ((pi = TAILQ_FIRST(&uq->uq_pi_contested)) != NULL) {
3587 pi->pi_owner = NULL;
3588 TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link);
3591 td->td_flags &= ~TDF_UBORROWING;
3593 mtx_unlock_spin(&umtx_lock);