2 * Copyright (c) 2004, David Xu <davidxu@freebsd.org>
3 * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org>
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice unmodified, this list of conditions, and the following
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
31 #include "opt_compat.h"
32 #include <sys/param.h>
33 #include <sys/kernel.h>
34 #include <sys/limits.h>
36 #include <sys/malloc.h>
37 #include <sys/mutex.h>
40 #include <sys/sched.h>
42 #include <sys/sysctl.h>
43 #include <sys/sysent.h>
44 #include <sys/systm.h>
45 #include <sys/sysproto.h>
46 #include <sys/eventhandler.h>
50 #include <vm/vm_param.h>
52 #include <vm/vm_map.h>
53 #include <vm/vm_object.h>
55 #include <machine/cpu.h>
58 #include <compat/freebsd32/freebsd32_proto.h>
61 #define TYPE_SIMPLE_WAIT 0
63 #define TYPE_SIMPLE_LOCK 2
64 #define TYPE_NORMAL_UMUTEX 3
65 #define TYPE_PI_UMUTEX 4
66 #define TYPE_PP_UMUTEX 5
70 #define _UMUTEX_WAIT 2
72 /* Key to represent a unique userland synchronous object */
93 /* Priority inheritance mutex info. */
96 struct thread *pi_owner;
101 /* List entry to link umtx holding by thread */
102 TAILQ_ENTRY(umtx_pi) pi_link;
104 /* List entry in hash */
105 TAILQ_ENTRY(umtx_pi) pi_hashlink;
107 /* List for waiters */
108 TAILQ_HEAD(,umtx_q) pi_blocked;
110 /* Identify a userland lock object */
111 struct umtx_key pi_key;
114 /* A userland synchronous object user. */
116 /* Linked list for the hash. */
117 TAILQ_ENTRY(umtx_q) uq_link;
120 struct umtx_key uq_key;
124 #define UQF_UMTXQ 0x0001
126 /* The thread waits on. */
127 struct thread *uq_thread;
130 * Blocked on PI mutex. read can use chain lock
131 * or umtx_lock, write must have both chain lock and
132 * umtx_lock being hold.
134 struct umtx_pi *uq_pi_blocked;
136 /* On blocked list */
137 TAILQ_ENTRY(umtx_q) uq_lockq;
139 /* Thread contending with us */
140 TAILQ_HEAD(,umtx_pi) uq_pi_contested;
142 /* Inherited priority from PP mutex */
143 u_char uq_inherited_pri;
146 TAILQ_HEAD(umtxq_head, umtx_q);
148 /* Userland lock object's wait-queue chain */
150 /* Lock for this chain. */
153 /* List of sleep queues. */
154 struct umtxq_head uc_queue[2];
155 #define UMTX_SHARED_QUEUE 0
156 #define UMTX_EXCLUSIVE_QUEUE 1
161 /* Chain lock waiters */
164 /* All PI in the list */
165 TAILQ_HEAD(,umtx_pi) uc_pi_list;
168 #define UMTXQ_LOCKED_ASSERT(uc) mtx_assert(&(uc)->uc_lock, MA_OWNED)
169 #define UMTXQ_BUSY_ASSERT(uc) KASSERT(&(uc)->uc_busy, ("umtx chain is not busy"))
172 * Don't propagate time-sharing priority, there is a security reason,
173 * a user can simply introduce PI-mutex, let thread A lock the mutex,
174 * and let another thread B block on the mutex, because B is
175 * sleeping, its priority will be boosted, this causes A's priority to
176 * be boosted via priority propagating too and will never be lowered even
177 * if it is using 100%CPU, this is unfair to other processes.
180 #define UPRI(td) (((td)->td_user_pri >= PRI_MIN_TIMESHARE &&\
181 (td)->td_user_pri <= PRI_MAX_TIMESHARE) ?\
182 PRI_MAX_TIMESHARE : (td)->td_user_pri)
184 #define GOLDEN_RATIO_PRIME 2654404609U
185 #define UMTX_CHAINS 128
186 #define UMTX_SHIFTS (__WORD_BIT - 7)
188 #define THREAD_SHARE 0
189 #define PROCESS_SHARE 1
192 #define GET_SHARE(flags) \
193 (((flags) & USYNC_PROCESS_SHARED) == 0 ? THREAD_SHARE : PROCESS_SHARE)
195 #define BUSY_SPINS 200
197 static uma_zone_t umtx_pi_zone;
198 static struct umtxq_chain umtxq_chains[2][UMTX_CHAINS];
199 static MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory");
200 static int umtx_pi_allocated;
202 SYSCTL_NODE(_debug, OID_AUTO, umtx, CTLFLAG_RW, 0, "umtx debug");
203 SYSCTL_INT(_debug_umtx, OID_AUTO, umtx_pi_allocated, CTLFLAG_RD,
204 &umtx_pi_allocated, 0, "Allocated umtx_pi");
206 static void umtxq_sysinit(void *);
207 static void umtxq_hash(struct umtx_key *key);
208 static struct umtxq_chain *umtxq_getchain(struct umtx_key *key);
209 static void umtxq_lock(struct umtx_key *key);
210 static void umtxq_unlock(struct umtx_key *key);
211 static void umtxq_busy(struct umtx_key *key);
212 static void umtxq_unbusy(struct umtx_key *key);
213 static void umtxq_insert_queue(struct umtx_q *uq, int q);
214 static void umtxq_remove_queue(struct umtx_q *uq, int q);
215 static int umtxq_sleep(struct umtx_q *uq, const char *wmesg, int timo);
216 static int umtxq_count(struct umtx_key *key);
217 static int umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2);
218 static int umtx_key_get(void *addr, int type, int share,
219 struct umtx_key *key);
220 static void umtx_key_release(struct umtx_key *key);
221 static struct umtx_pi *umtx_pi_alloc(int);
222 static void umtx_pi_free(struct umtx_pi *pi);
223 static void umtx_pi_adjust_locked(struct thread *td, u_char oldpri);
224 static int do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags);
225 static void umtx_thread_cleanup(struct thread *td);
226 static void umtx_exec_hook(void *arg __unused, struct proc *p __unused,
227 struct image_params *imgp __unused);
228 SYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_sysinit, NULL);
230 #define umtxq_signal(key, nwake) umtxq_signal_queue((key), (nwake), UMTX_SHARED_QUEUE)
231 #define umtxq_insert(uq) umtxq_insert_queue((uq), UMTX_SHARED_QUEUE)
232 #define umtxq_remove(uq) umtxq_remove_queue((uq), UMTX_SHARED_QUEUE)
234 static struct mtx umtx_lock;
237 umtxq_sysinit(void *arg __unused)
241 umtx_pi_zone = uma_zcreate("umtx pi", sizeof(struct umtx_pi),
242 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
243 for (i = 0; i < 2; ++i) {
244 for (j = 0; j < UMTX_CHAINS; ++j) {
245 mtx_init(&umtxq_chains[i][j].uc_lock, "umtxql", NULL,
246 MTX_DEF | MTX_DUPOK);
247 TAILQ_INIT(&umtxq_chains[i][j].uc_queue[0]);
248 TAILQ_INIT(&umtxq_chains[i][j].uc_queue[1]);
249 TAILQ_INIT(&umtxq_chains[i][j].uc_pi_list);
250 umtxq_chains[i][j].uc_busy = 0;
251 umtxq_chains[i][j].uc_waiters = 0;
254 mtx_init(&umtx_lock, "umtx lock", NULL, MTX_SPIN);
255 EVENTHANDLER_REGISTER(process_exec, umtx_exec_hook, NULL,
256 EVENTHANDLER_PRI_ANY);
264 uq = malloc(sizeof(struct umtx_q), M_UMTX, M_WAITOK | M_ZERO);
265 TAILQ_INIT(&uq->uq_pi_contested);
266 uq->uq_inherited_pri = PRI_MAX;
271 umtxq_free(struct umtx_q *uq)
277 umtxq_hash(struct umtx_key *key)
279 unsigned n = (uintptr_t)key->info.both.a + key->info.both.b;
280 key->hash = ((n * GOLDEN_RATIO_PRIME) >> UMTX_SHIFTS) % UMTX_CHAINS;
284 umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2)
286 return (k1->type == k2->type &&
287 k1->info.both.a == k2->info.both.a &&
288 k1->info.both.b == k2->info.both.b);
291 static inline struct umtxq_chain *
292 umtxq_getchain(struct umtx_key *key)
294 if (key->type <= TYPE_CV)
295 return (&umtxq_chains[1][key->hash]);
296 return (&umtxq_chains[0][key->hash]);
303 umtxq_lock(struct umtx_key *key)
305 struct umtxq_chain *uc;
307 uc = umtxq_getchain(key);
308 mtx_lock(&uc->uc_lock);
315 umtxq_unlock(struct umtx_key *key)
317 struct umtxq_chain *uc;
319 uc = umtxq_getchain(key);
320 mtx_unlock(&uc->uc_lock);
324 * Set chain to busy state when following operation
325 * may be blocked (kernel mutex can not be used).
328 umtxq_busy(struct umtx_key *key)
330 struct umtxq_chain *uc;
332 uc = umtxq_getchain(key);
333 mtx_assert(&uc->uc_lock, MA_OWNED);
337 int count = BUSY_SPINS;
340 while (uc->uc_busy && --count > 0)
346 while (uc->uc_busy) {
348 msleep(uc, &uc->uc_lock, 0, "umtxqb", 0);
359 umtxq_unbusy(struct umtx_key *key)
361 struct umtxq_chain *uc;
363 uc = umtxq_getchain(key);
364 mtx_assert(&uc->uc_lock, MA_OWNED);
365 KASSERT(uc->uc_busy != 0, ("not busy"));
372 umtxq_insert_queue(struct umtx_q *uq, int q)
374 struct umtxq_chain *uc;
376 uc = umtxq_getchain(&uq->uq_key);
377 UMTXQ_LOCKED_ASSERT(uc);
378 TAILQ_INSERT_TAIL(&uc->uc_queue[q], uq, uq_link);
379 uq->uq_flags |= UQF_UMTXQ;
383 umtxq_remove_queue(struct umtx_q *uq, int q)
385 struct umtxq_chain *uc;
387 uc = umtxq_getchain(&uq->uq_key);
388 UMTXQ_LOCKED_ASSERT(uc);
389 if (uq->uq_flags & UQF_UMTXQ) {
390 TAILQ_REMOVE(&uc->uc_queue[q], uq, uq_link);
391 uq->uq_flags &= ~UQF_UMTXQ;
396 * Check if there are multiple waiters
399 umtxq_count(struct umtx_key *key)
401 struct umtxq_chain *uc;
405 uc = umtxq_getchain(key);
406 UMTXQ_LOCKED_ASSERT(uc);
407 TAILQ_FOREACH(uq, &uc->uc_queue[UMTX_SHARED_QUEUE], uq_link) {
408 if (umtx_key_match(&uq->uq_key, key)) {
417 * Check if there are multiple PI waiters and returns first
421 umtxq_count_pi(struct umtx_key *key, struct umtx_q **first)
423 struct umtxq_chain *uc;
428 uc = umtxq_getchain(key);
429 UMTXQ_LOCKED_ASSERT(uc);
430 TAILQ_FOREACH(uq, &uc->uc_queue[UMTX_SHARED_QUEUE], uq_link) {
431 if (umtx_key_match(&uq->uq_key, key)) {
441 * Wake up threads waiting on an userland object.
445 umtxq_signal_queue(struct umtx_key *key, int n_wake, int q)
447 struct umtxq_chain *uc;
448 struct umtx_q *uq, *next;
452 uc = umtxq_getchain(key);
453 UMTXQ_LOCKED_ASSERT(uc);
454 TAILQ_FOREACH_SAFE(uq, &uc->uc_queue[q], uq_link, next) {
455 if (umtx_key_match(&uq->uq_key, key)) {
456 umtxq_remove_queue(uq, q);
467 * Wake up specified thread.
470 umtxq_signal_thread(struct umtx_q *uq)
472 struct umtxq_chain *uc;
474 uc = umtxq_getchain(&uq->uq_key);
475 UMTXQ_LOCKED_ASSERT(uc);
481 * Put thread into sleep state, before sleeping, check if
482 * thread was removed from umtx queue.
485 umtxq_sleep(struct umtx_q *uq, const char *wmesg, int timo)
487 struct umtxq_chain *uc;
490 uc = umtxq_getchain(&uq->uq_key);
491 UMTXQ_LOCKED_ASSERT(uc);
492 if (!(uq->uq_flags & UQF_UMTXQ))
494 error = msleep(uq, &uc->uc_lock, PCATCH, wmesg, timo);
495 if (error == EWOULDBLOCK)
501 * Convert userspace address into unique logical address.
504 umtx_key_get(void *addr, int type, int share, struct umtx_key *key)
506 struct thread *td = curthread;
508 vm_map_entry_t entry;
514 if (share == THREAD_SHARE) {
516 key->info.private.vs = td->td_proc->p_vmspace;
517 key->info.private.addr = (uintptr_t)addr;
519 MPASS(share == PROCESS_SHARE || share == AUTO_SHARE);
520 map = &td->td_proc->p_vmspace->vm_map;
521 if (vm_map_lookup(&map, (vm_offset_t)addr, VM_PROT_WRITE,
522 &entry, &key->info.shared.object, &pindex, &prot,
523 &wired) != KERN_SUCCESS) {
527 if ((share == PROCESS_SHARE) ||
528 (share == AUTO_SHARE &&
529 VM_INHERIT_SHARE == entry->inheritance)) {
531 key->info.shared.offset = entry->offset + entry->start -
533 vm_object_reference(key->info.shared.object);
536 key->info.private.vs = td->td_proc->p_vmspace;
537 key->info.private.addr = (uintptr_t)addr;
539 vm_map_lookup_done(map, entry);
550 umtx_key_release(struct umtx_key *key)
553 vm_object_deallocate(key->info.shared.object);
557 * Lock a umtx object.
560 _do_lock_umtx(struct thread *td, struct umtx *umtx, u_long id, int timo)
570 * Care must be exercised when dealing with umtx structure. It
571 * can fault on any access.
575 * Try the uncontested case. This should be done in userland.
577 owner = casuword(&umtx->u_owner, UMTX_UNOWNED, id);
579 /* The acquire succeeded. */
580 if (owner == UMTX_UNOWNED)
583 /* The address was invalid. */
587 /* If no one owns it but it is contested try to acquire it. */
588 if (owner == UMTX_CONTESTED) {
589 owner = casuword(&umtx->u_owner,
590 UMTX_CONTESTED, id | UMTX_CONTESTED);
592 if (owner == UMTX_CONTESTED)
595 /* The address was invalid. */
599 /* If this failed the lock has changed, restart. */
604 * If we caught a signal, we have retried and now
610 if ((error = umtx_key_get(umtx, TYPE_SIMPLE_LOCK,
611 AUTO_SHARE, &uq->uq_key)) != 0)
614 umtxq_lock(&uq->uq_key);
615 umtxq_busy(&uq->uq_key);
617 umtxq_unbusy(&uq->uq_key);
618 umtxq_unlock(&uq->uq_key);
621 * Set the contested bit so that a release in user space
622 * knows to use the system call for unlock. If this fails
623 * either some one else has acquired the lock or it has been
626 old = casuword(&umtx->u_owner, owner, owner | UMTX_CONTESTED);
628 /* The address was invalid. */
630 umtxq_lock(&uq->uq_key);
632 umtxq_unlock(&uq->uq_key);
633 umtx_key_release(&uq->uq_key);
638 * We set the contested bit, sleep. Otherwise the lock changed
639 * and we need to retry or we lost a race to the thread
640 * unlocking the umtx.
642 umtxq_lock(&uq->uq_key);
644 error = umtxq_sleep(uq, "umtx", timo);
646 umtxq_unlock(&uq->uq_key);
647 umtx_key_release(&uq->uq_key);
654 * Lock a umtx object.
657 do_lock_umtx(struct thread *td, struct umtx *umtx, u_long id,
658 struct timespec *timeout)
660 struct timespec ts, ts2, ts3;
664 if (timeout == NULL) {
665 error = _do_lock_umtx(td, umtx, id, 0);
666 /* Mutex locking is restarted if it is interrupted. */
671 timespecadd(&ts, timeout);
672 TIMESPEC_TO_TIMEVAL(&tv, timeout);
674 error = _do_lock_umtx(td, umtx, id, tvtohz(&tv));
675 if (error != ETIMEDOUT)
678 if (timespeccmp(&ts2, &ts, >=)) {
683 timespecsub(&ts3, &ts2);
684 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
686 /* Timed-locking is not restarted. */
687 if (error == ERESTART)
694 * Unlock a umtx object.
697 do_unlock_umtx(struct thread *td, struct umtx *umtx, u_long id)
706 * Make sure we own this mtx.
708 owner = fuword(__DEVOLATILE(u_long *, &umtx->u_owner));
712 if ((owner & ~UMTX_CONTESTED) != id)
715 /* This should be done in userland */
716 if ((owner & UMTX_CONTESTED) == 0) {
717 old = casuword(&umtx->u_owner, owner, UMTX_UNOWNED);
725 /* We should only ever be in here for contested locks */
726 if ((error = umtx_key_get(umtx, TYPE_SIMPLE_LOCK, AUTO_SHARE,
732 count = umtxq_count(&key);
736 * When unlocking the umtx, it must be marked as unowned if
737 * there is zero or one thread only waiting for it.
738 * Otherwise, it must be marked as contested.
740 old = casuword(&umtx->u_owner, owner,
741 count <= 1 ? UMTX_UNOWNED : UMTX_CONTESTED);
743 umtxq_signal(&key,1);
746 umtx_key_release(&key);
757 * Lock a umtx object.
760 _do_lock_umtx32(struct thread *td, uint32_t *m, uint32_t id, int timo)
770 * Care must be exercised when dealing with umtx structure. It
771 * can fault on any access.
775 * Try the uncontested case. This should be done in userland.
777 owner = casuword32(m, UMUTEX_UNOWNED, id);
779 /* The acquire succeeded. */
780 if (owner == UMUTEX_UNOWNED)
783 /* The address was invalid. */
787 /* If no one owns it but it is contested try to acquire it. */
788 if (owner == UMUTEX_CONTESTED) {
789 owner = casuword32(m,
790 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
791 if (owner == UMUTEX_CONTESTED)
794 /* The address was invalid. */
798 /* If this failed the lock has changed, restart. */
803 * If we caught a signal, we have retried and now
809 if ((error = umtx_key_get(m, TYPE_SIMPLE_LOCK,
810 AUTO_SHARE, &uq->uq_key)) != 0)
813 umtxq_lock(&uq->uq_key);
814 umtxq_busy(&uq->uq_key);
816 umtxq_unbusy(&uq->uq_key);
817 umtxq_unlock(&uq->uq_key);
820 * Set the contested bit so that a release in user space
821 * knows to use the system call for unlock. If this fails
822 * either some one else has acquired the lock or it has been
825 old = casuword32(m, owner, owner | UMUTEX_CONTESTED);
827 /* The address was invalid. */
829 umtxq_lock(&uq->uq_key);
831 umtxq_unlock(&uq->uq_key);
832 umtx_key_release(&uq->uq_key);
837 * We set the contested bit, sleep. Otherwise the lock changed
838 * and we need to retry or we lost a race to the thread
839 * unlocking the umtx.
841 umtxq_lock(&uq->uq_key);
843 error = umtxq_sleep(uq, "umtx", timo);
845 umtxq_unlock(&uq->uq_key);
846 umtx_key_release(&uq->uq_key);
853 * Lock a umtx object.
856 do_lock_umtx32(struct thread *td, void *m, uint32_t id,
857 struct timespec *timeout)
859 struct timespec ts, ts2, ts3;
863 if (timeout == NULL) {
864 error = _do_lock_umtx32(td, m, id, 0);
865 /* Mutex locking is restarted if it is interrupted. */
870 timespecadd(&ts, timeout);
871 TIMESPEC_TO_TIMEVAL(&tv, timeout);
873 error = _do_lock_umtx32(td, m, id, tvtohz(&tv));
874 if (error != ETIMEDOUT)
877 if (timespeccmp(&ts2, &ts, >=)) {
882 timespecsub(&ts3, &ts2);
883 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
885 /* Timed-locking is not restarted. */
886 if (error == ERESTART)
893 * Unlock a umtx object.
896 do_unlock_umtx32(struct thread *td, uint32_t *m, uint32_t id)
905 * Make sure we own this mtx.
911 if ((owner & ~UMUTEX_CONTESTED) != id)
914 /* This should be done in userland */
915 if ((owner & UMUTEX_CONTESTED) == 0) {
916 old = casuword32(m, owner, UMUTEX_UNOWNED);
924 /* We should only ever be in here for contested locks */
925 if ((error = umtx_key_get(m, TYPE_SIMPLE_LOCK, AUTO_SHARE,
931 count = umtxq_count(&key);
935 * When unlocking the umtx, it must be marked as unowned if
936 * there is zero or one thread only waiting for it.
937 * Otherwise, it must be marked as contested.
939 old = casuword32(m, owner,
940 count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
942 umtxq_signal(&key,1);
945 umtx_key_release(&key);
955 * Fetch and compare value, sleep on the address if value is not changed.
958 do_wait(struct thread *td, void *addr, u_long id,
959 struct timespec *timeout, int compat32, int is_private)
962 struct timespec ts, ts2, ts3;
968 if ((error = umtx_key_get(addr, TYPE_SIMPLE_WAIT,
969 is_private ? THREAD_SHARE : AUTO_SHARE, &uq->uq_key)) != 0)
972 umtxq_lock(&uq->uq_key);
974 umtxq_unlock(&uq->uq_key);
978 tmp = (unsigned int)fuword32(addr);
980 umtxq_lock(&uq->uq_key);
982 umtxq_unlock(&uq->uq_key);
983 } else if (timeout == NULL) {
984 umtxq_lock(&uq->uq_key);
985 error = umtxq_sleep(uq, "uwait", 0);
987 umtxq_unlock(&uq->uq_key);
990 timespecadd(&ts, timeout);
991 TIMESPEC_TO_TIMEVAL(&tv, timeout);
992 umtxq_lock(&uq->uq_key);
994 error = umtxq_sleep(uq, "uwait", tvtohz(&tv));
995 if (!(uq->uq_flags & UQF_UMTXQ))
997 if (error != ETIMEDOUT)
999 umtxq_unlock(&uq->uq_key);
1000 getnanouptime(&ts2);
1001 if (timespeccmp(&ts2, &ts, >=)) {
1003 umtxq_lock(&uq->uq_key);
1007 timespecsub(&ts3, &ts2);
1008 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
1009 umtxq_lock(&uq->uq_key);
1012 umtxq_unlock(&uq->uq_key);
1014 umtx_key_release(&uq->uq_key);
1015 if (error == ERESTART)
1021 * Wake up threads sleeping on the specified address.
1024 kern_umtx_wake(struct thread *td, void *uaddr, int n_wake, int is_private)
1026 struct umtx_key key;
1029 if ((ret = umtx_key_get(uaddr, TYPE_SIMPLE_WAIT,
1030 is_private ? THREAD_SHARE : AUTO_SHARE, &key)) != 0)
1033 ret = umtxq_signal(&key, n_wake);
1035 umtx_key_release(&key);
1040 * Lock PTHREAD_PRIO_NONE protocol POSIX mutex.
1043 _do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, int timo,
1047 uint32_t owner, old, id;
1054 * Care must be exercised when dealing with umtx structure. It
1055 * can fault on any access.
1058 owner = fuword32(__DEVOLATILE(void *, &m->m_owner));
1059 if (mode == _UMUTEX_WAIT) {
1060 if (owner == UMUTEX_UNOWNED || owner == UMUTEX_CONTESTED)
1064 * Try the uncontested case. This should be done in userland.
1066 owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id);
1068 /* The acquire succeeded. */
1069 if (owner == UMUTEX_UNOWNED)
1072 /* The address was invalid. */
1076 /* If no one owns it but it is contested try to acquire it. */
1077 if (owner == UMUTEX_CONTESTED) {
1078 owner = casuword32(&m->m_owner,
1079 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
1081 if (owner == UMUTEX_CONTESTED)
1084 /* The address was invalid. */
1088 /* If this failed the lock has changed, restart. */
1093 if ((flags & UMUTEX_ERROR_CHECK) != 0 &&
1094 (owner & ~UMUTEX_CONTESTED) == id)
1097 if (mode == _UMUTEX_TRY)
1101 * If we caught a signal, we have retried and now
1107 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX,
1108 GET_SHARE(flags), &uq->uq_key)) != 0)
1111 umtxq_lock(&uq->uq_key);
1112 umtxq_busy(&uq->uq_key);
1114 umtxq_unlock(&uq->uq_key);
1117 * Set the contested bit so that a release in user space
1118 * knows to use the system call for unlock. If this fails
1119 * either some one else has acquired the lock or it has been
1122 old = casuword32(&m->m_owner, owner, owner | UMUTEX_CONTESTED);
1124 /* The address was invalid. */
1126 umtxq_lock(&uq->uq_key);
1128 umtxq_unbusy(&uq->uq_key);
1129 umtxq_unlock(&uq->uq_key);
1130 umtx_key_release(&uq->uq_key);
1135 * We set the contested bit, sleep. Otherwise the lock changed
1136 * and we need to retry or we lost a race to the thread
1137 * unlocking the umtx.
1139 umtxq_lock(&uq->uq_key);
1140 umtxq_unbusy(&uq->uq_key);
1142 error = umtxq_sleep(uq, "umtxn", timo);
1144 umtxq_unlock(&uq->uq_key);
1145 umtx_key_release(&uq->uq_key);
1152 * Lock PTHREAD_PRIO_NONE protocol POSIX mutex.
1155 * Unlock PTHREAD_PRIO_NONE protocol POSIX mutex.
1158 do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags)
1160 struct umtx_key key;
1161 uint32_t owner, old, id;
1167 * Make sure we own this mtx.
1169 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
1173 if ((owner & ~UMUTEX_CONTESTED) != id)
1176 if ((owner & UMUTEX_CONTESTED) == 0) {
1177 old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED);
1185 /* We should only ever be in here for contested locks */
1186 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags),
1192 count = umtxq_count(&key);
1196 * When unlocking the umtx, it must be marked as unowned if
1197 * there is zero or one thread only waiting for it.
1198 * Otherwise, it must be marked as contested.
1200 old = casuword32(&m->m_owner, owner,
1201 count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
1203 umtxq_signal(&key,1);
1206 umtx_key_release(&key);
1215 * Check if the mutex is available and wake up a waiter,
1216 * only for simple mutex.
1219 do_wake_umutex(struct thread *td, struct umutex *m)
1221 struct umtx_key key;
1227 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
1231 if ((owner & ~UMUTEX_CONTESTED) != 0)
1234 flags = fuword32(&m->m_flags);
1236 /* We should only ever be in here for contested locks */
1237 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags),
1243 count = umtxq_count(&key);
1247 owner = casuword32(&m->m_owner, UMUTEX_CONTESTED, UMUTEX_UNOWNED);
1250 if (count != 0 && (owner & ~UMUTEX_CONTESTED) == 0)
1251 umtxq_signal(&key, 1);
1254 umtx_key_release(&key);
1258 static inline struct umtx_pi *
1259 umtx_pi_alloc(int flags)
1263 pi = uma_zalloc(umtx_pi_zone, M_ZERO | flags);
1264 TAILQ_INIT(&pi->pi_blocked);
1265 atomic_add_int(&umtx_pi_allocated, 1);
1270 umtx_pi_free(struct umtx_pi *pi)
1272 uma_zfree(umtx_pi_zone, pi);
1273 atomic_add_int(&umtx_pi_allocated, -1);
1277 * Adjust the thread's position on a pi_state after its priority has been
1281 umtx_pi_adjust_thread(struct umtx_pi *pi, struct thread *td)
1283 struct umtx_q *uq, *uq1, *uq2;
1286 mtx_assert(&umtx_lock, MA_OWNED);
1293 * Check if the thread needs to be moved on the blocked chain.
1294 * It needs to be moved if either its priority is lower than
1295 * the previous thread or higher than the next thread.
1297 uq1 = TAILQ_PREV(uq, umtxq_head, uq_lockq);
1298 uq2 = TAILQ_NEXT(uq, uq_lockq);
1299 if ((uq1 != NULL && UPRI(td) < UPRI(uq1->uq_thread)) ||
1300 (uq2 != NULL && UPRI(td) > UPRI(uq2->uq_thread))) {
1302 * Remove thread from blocked chain and determine where
1303 * it should be moved to.
1305 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
1306 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) {
1307 td1 = uq1->uq_thread;
1308 MPASS(td1->td_proc->p_magic == P_MAGIC);
1309 if (UPRI(td1) > UPRI(td))
1314 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq);
1316 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq);
1322 * Propagate priority when a thread is blocked on POSIX
1326 umtx_propagate_priority(struct thread *td)
1332 mtx_assert(&umtx_lock, MA_OWNED);
1335 pi = uq->uq_pi_blocked;
1344 MPASS(td->td_proc != NULL);
1345 MPASS(td->td_proc->p_magic == P_MAGIC);
1347 if (UPRI(td) <= pri)
1351 sched_lend_user_prio(td, pri);
1355 * Pick up the lock that td is blocked on.
1358 pi = uq->uq_pi_blocked;
1359 /* Resort td on the list if needed. */
1360 if (!umtx_pi_adjust_thread(pi, td))
1366 * Unpropagate priority for a PI mutex when a thread blocked on
1367 * it is interrupted by signal or resumed by others.
1370 umtx_unpropagate_priority(struct umtx_pi *pi)
1372 struct umtx_q *uq, *uq_owner;
1373 struct umtx_pi *pi2;
1376 mtx_assert(&umtx_lock, MA_OWNED);
1378 while (pi != NULL && pi->pi_owner != NULL) {
1380 uq_owner = pi->pi_owner->td_umtxq;
1382 TAILQ_FOREACH(pi2, &uq_owner->uq_pi_contested, pi_link) {
1383 uq = TAILQ_FIRST(&pi2->pi_blocked);
1385 if (pri > UPRI(uq->uq_thread))
1386 pri = UPRI(uq->uq_thread);
1390 if (pri > uq_owner->uq_inherited_pri)
1391 pri = uq_owner->uq_inherited_pri;
1392 thread_lock(pi->pi_owner);
1393 oldpri = pi->pi_owner->td_user_pri;
1394 sched_unlend_user_prio(pi->pi_owner, pri);
1395 thread_unlock(pi->pi_owner);
1396 if (uq_owner->uq_pi_blocked != NULL)
1397 umtx_pi_adjust_locked(pi->pi_owner, oldpri);
1398 pi = uq_owner->uq_pi_blocked;
1403 * Insert a PI mutex into owned list.
1406 umtx_pi_setowner(struct umtx_pi *pi, struct thread *owner)
1408 struct umtx_q *uq_owner;
1410 uq_owner = owner->td_umtxq;
1411 mtx_assert(&umtx_lock, MA_OWNED);
1412 if (pi->pi_owner != NULL)
1413 panic("pi_ower != NULL");
1414 pi->pi_owner = owner;
1415 TAILQ_INSERT_TAIL(&uq_owner->uq_pi_contested, pi, pi_link);
1419 * Claim ownership of a PI mutex.
1422 umtx_pi_claim(struct umtx_pi *pi, struct thread *owner)
1424 struct umtx_q *uq, *uq_owner;
1426 uq_owner = owner->td_umtxq;
1427 mtx_lock_spin(&umtx_lock);
1428 if (pi->pi_owner == owner) {
1429 mtx_unlock_spin(&umtx_lock);
1433 if (pi->pi_owner != NULL) {
1435 * userland may have already messed the mutex, sigh.
1437 mtx_unlock_spin(&umtx_lock);
1440 umtx_pi_setowner(pi, owner);
1441 uq = TAILQ_FIRST(&pi->pi_blocked);
1445 pri = UPRI(uq->uq_thread);
1447 if (pri < UPRI(owner))
1448 sched_lend_user_prio(owner, pri);
1449 thread_unlock(owner);
1451 mtx_unlock_spin(&umtx_lock);
1456 umtx_pi_adjust_locked(struct thread *td, u_char oldpri)
1463 * Pick up the lock that td is blocked on.
1465 pi = uq->uq_pi_blocked;
1468 /* Resort the turnstile on the list. */
1469 if (!umtx_pi_adjust_thread(pi, td))
1473 * If our priority was lowered and we are at the head of the
1474 * turnstile, then propagate our new priority up the chain.
1476 if (uq == TAILQ_FIRST(&pi->pi_blocked) && UPRI(td) < oldpri)
1477 umtx_propagate_priority(td);
1481 * Adjust a thread's order position in its blocked PI mutex,
1482 * this may result new priority propagating process.
1485 umtx_pi_adjust(struct thread *td, u_char oldpri)
1491 mtx_lock_spin(&umtx_lock);
1493 * Pick up the lock that td is blocked on.
1495 pi = uq->uq_pi_blocked;
1497 umtx_pi_adjust_locked(td, oldpri);
1498 mtx_unlock_spin(&umtx_lock);
1502 * Sleep on a PI mutex.
1505 umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi,
1506 uint32_t owner, const char *wmesg, int timo)
1508 struct umtxq_chain *uc;
1509 struct thread *td, *td1;
1515 KASSERT(td == curthread, ("inconsistent uq_thread"));
1516 uc = umtxq_getchain(&uq->uq_key);
1517 UMTXQ_LOCKED_ASSERT(uc);
1518 UMTXQ_BUSY_ASSERT(uc);
1520 mtx_lock_spin(&umtx_lock);
1521 if (pi->pi_owner == NULL) {
1523 * Current, We only support process private PI-mutex,
1524 * non-contended PI-mutexes are locked in userland.
1525 * Process shared PI-mutex should always be initialized
1526 * by kernel and be registered in kernel, locking should
1527 * always be done by kernel to avoid security problems.
1528 * For process private PI-mutex, we can find owner
1529 * thread and boost its priority safely.
1531 mtx_unlock_spin(&umtx_lock);
1533 td1 = thread_find(curproc, owner);
1534 mtx_lock_spin(&umtx_lock);
1535 if (td1 != NULL && pi->pi_owner == NULL) {
1536 uq1 = td1->td_umtxq;
1537 umtx_pi_setowner(pi, td1);
1539 PROC_UNLOCK(curproc);
1542 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) {
1543 pri = UPRI(uq1->uq_thread);
1549 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq);
1551 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq);
1553 uq->uq_pi_blocked = pi;
1555 td->td_flags |= TDF_UPIBLOCKED;
1557 umtx_propagate_priority(td);
1558 mtx_unlock_spin(&umtx_lock);
1559 umtxq_unbusy(&uq->uq_key);
1561 if (uq->uq_flags & UQF_UMTXQ) {
1562 error = msleep(uq, &uc->uc_lock, PCATCH, wmesg, timo);
1563 if (error == EWOULDBLOCK)
1565 if (uq->uq_flags & UQF_UMTXQ) {
1569 mtx_lock_spin(&umtx_lock);
1570 uq->uq_pi_blocked = NULL;
1572 td->td_flags &= ~TDF_UPIBLOCKED;
1574 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
1575 umtx_unpropagate_priority(pi);
1576 mtx_unlock_spin(&umtx_lock);
1577 umtxq_unlock(&uq->uq_key);
1583 * Add reference count for a PI mutex.
1586 umtx_pi_ref(struct umtx_pi *pi)
1588 struct umtxq_chain *uc;
1590 uc = umtxq_getchain(&pi->pi_key);
1591 UMTXQ_LOCKED_ASSERT(uc);
1596 * Decrease reference count for a PI mutex, if the counter
1597 * is decreased to zero, its memory space is freed.
1600 umtx_pi_unref(struct umtx_pi *pi)
1602 struct umtxq_chain *uc;
1604 uc = umtxq_getchain(&pi->pi_key);
1605 UMTXQ_LOCKED_ASSERT(uc);
1606 KASSERT(pi->pi_refcount > 0, ("invalid reference count"));
1607 if (--pi->pi_refcount == 0) {
1608 mtx_lock_spin(&umtx_lock);
1609 if (pi->pi_owner != NULL) {
1610 TAILQ_REMOVE(&pi->pi_owner->td_umtxq->uq_pi_contested,
1612 pi->pi_owner = NULL;
1614 KASSERT(TAILQ_EMPTY(&pi->pi_blocked),
1615 ("blocked queue not empty"));
1616 mtx_unlock_spin(&umtx_lock);
1617 TAILQ_REMOVE(&uc->uc_pi_list, pi, pi_hashlink);
1623 * Find a PI mutex in hash table.
1625 static struct umtx_pi *
1626 umtx_pi_lookup(struct umtx_key *key)
1628 struct umtxq_chain *uc;
1631 uc = umtxq_getchain(key);
1632 UMTXQ_LOCKED_ASSERT(uc);
1634 TAILQ_FOREACH(pi, &uc->uc_pi_list, pi_hashlink) {
1635 if (umtx_key_match(&pi->pi_key, key)) {
1643 * Insert a PI mutex into hash table.
1646 umtx_pi_insert(struct umtx_pi *pi)
1648 struct umtxq_chain *uc;
1650 uc = umtxq_getchain(&pi->pi_key);
1651 UMTXQ_LOCKED_ASSERT(uc);
1652 TAILQ_INSERT_TAIL(&uc->uc_pi_list, pi, pi_hashlink);
1659 _do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, int timo,
1663 struct umtx_pi *pi, *new_pi;
1664 uint32_t id, owner, old;
1670 if ((error = umtx_key_get(m, TYPE_PI_UMUTEX, GET_SHARE(flags),
1673 umtxq_lock(&uq->uq_key);
1674 pi = umtx_pi_lookup(&uq->uq_key);
1676 new_pi = umtx_pi_alloc(M_NOWAIT);
1677 if (new_pi == NULL) {
1678 umtxq_unlock(&uq->uq_key);
1679 new_pi = umtx_pi_alloc(M_WAITOK);
1680 umtxq_lock(&uq->uq_key);
1681 pi = umtx_pi_lookup(&uq->uq_key);
1683 umtx_pi_free(new_pi);
1687 if (new_pi != NULL) {
1688 new_pi->pi_key = uq->uq_key;
1689 umtx_pi_insert(new_pi);
1694 umtxq_unlock(&uq->uq_key);
1697 * Care must be exercised when dealing with umtx structure. It
1698 * can fault on any access.
1702 * Try the uncontested case. This should be done in userland.
1704 owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id);
1706 /* The acquire succeeded. */
1707 if (owner == UMUTEX_UNOWNED) {
1712 /* The address was invalid. */
1718 /* If no one owns it but it is contested try to acquire it. */
1719 if (owner == UMUTEX_CONTESTED) {
1720 owner = casuword32(&m->m_owner,
1721 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
1723 if (owner == UMUTEX_CONTESTED) {
1724 umtxq_lock(&uq->uq_key);
1725 umtxq_busy(&uq->uq_key);
1726 error = umtx_pi_claim(pi, td);
1727 umtxq_unbusy(&uq->uq_key);
1728 umtxq_unlock(&uq->uq_key);
1732 /* The address was invalid. */
1738 /* If this failed the lock has changed, restart. */
1742 if ((flags & UMUTEX_ERROR_CHECK) != 0 &&
1743 (owner & ~UMUTEX_CONTESTED) == id) {
1754 * If we caught a signal, we have retried and now
1760 umtxq_lock(&uq->uq_key);
1761 umtxq_busy(&uq->uq_key);
1762 umtxq_unlock(&uq->uq_key);
1765 * Set the contested bit so that a release in user space
1766 * knows to use the system call for unlock. If this fails
1767 * either some one else has acquired the lock or it has been
1770 old = casuword32(&m->m_owner, owner, owner | UMUTEX_CONTESTED);
1772 /* The address was invalid. */
1774 umtxq_lock(&uq->uq_key);
1775 umtxq_unbusy(&uq->uq_key);
1776 umtxq_unlock(&uq->uq_key);
1781 umtxq_lock(&uq->uq_key);
1783 * We set the contested bit, sleep. Otherwise the lock changed
1784 * and we need to retry or we lost a race to the thread
1785 * unlocking the umtx.
1788 error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED,
1791 umtxq_unbusy(&uq->uq_key);
1792 umtxq_unlock(&uq->uq_key);
1796 umtxq_lock(&uq->uq_key);
1798 umtxq_unlock(&uq->uq_key);
1800 umtx_key_release(&uq->uq_key);
1805 * Unlock a PI mutex.
1808 do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags)
1810 struct umtx_key key;
1811 struct umtx_q *uq_first, *uq_first2, *uq_me;
1812 struct umtx_pi *pi, *pi2;
1813 uint32_t owner, old, id;
1820 * Make sure we own this mtx.
1822 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
1826 if ((owner & ~UMUTEX_CONTESTED) != id)
1829 /* This should be done in userland */
1830 if ((owner & UMUTEX_CONTESTED) == 0) {
1831 old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED);
1839 /* We should only ever be in here for contested locks */
1840 if ((error = umtx_key_get(m, TYPE_PI_UMUTEX, GET_SHARE(flags),
1846 count = umtxq_count_pi(&key, &uq_first);
1847 if (uq_first != NULL) {
1848 mtx_lock_spin(&umtx_lock);
1849 pi = uq_first->uq_pi_blocked;
1850 KASSERT(pi != NULL, ("pi == NULL?"));
1851 if (pi->pi_owner != curthread) {
1852 mtx_unlock_spin(&umtx_lock);
1855 umtx_key_release(&key);
1856 /* userland messed the mutex */
1859 uq_me = curthread->td_umtxq;
1860 pi->pi_owner = NULL;
1861 TAILQ_REMOVE(&uq_me->uq_pi_contested, pi, pi_link);
1862 /* get highest priority thread which is still sleeping. */
1863 uq_first = TAILQ_FIRST(&pi->pi_blocked);
1864 while (uq_first != NULL &&
1865 (uq_first->uq_flags & UQF_UMTXQ) == 0) {
1866 uq_first = TAILQ_NEXT(uq_first, uq_lockq);
1869 TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) {
1870 uq_first2 = TAILQ_FIRST(&pi2->pi_blocked);
1871 if (uq_first2 != NULL) {
1872 if (pri > UPRI(uq_first2->uq_thread))
1873 pri = UPRI(uq_first2->uq_thread);
1876 thread_lock(curthread);
1877 sched_unlend_user_prio(curthread, pri);
1878 thread_unlock(curthread);
1879 mtx_unlock_spin(&umtx_lock);
1881 umtxq_signal_thread(uq_first);
1886 * When unlocking the umtx, it must be marked as unowned if
1887 * there is zero or one thread only waiting for it.
1888 * Otherwise, it must be marked as contested.
1890 old = casuword32(&m->m_owner, owner,
1891 count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
1896 umtx_key_release(&key);
1908 _do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, int timo,
1911 struct umtx_q *uq, *uq2;
1915 int error, pri, old_inherited_pri, su;
1919 if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags),
1922 su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0);
1924 old_inherited_pri = uq->uq_inherited_pri;
1925 umtxq_lock(&uq->uq_key);
1926 umtxq_busy(&uq->uq_key);
1927 umtxq_unlock(&uq->uq_key);
1929 ceiling = RTP_PRIO_MAX - fuword32(&m->m_ceilings[0]);
1930 if (ceiling > RTP_PRIO_MAX) {
1935 mtx_lock_spin(&umtx_lock);
1936 if (UPRI(td) < PRI_MIN_REALTIME + ceiling) {
1937 mtx_unlock_spin(&umtx_lock);
1941 if (su && PRI_MIN_REALTIME + ceiling < uq->uq_inherited_pri) {
1942 uq->uq_inherited_pri = PRI_MIN_REALTIME + ceiling;
1944 if (uq->uq_inherited_pri < UPRI(td))
1945 sched_lend_user_prio(td, uq->uq_inherited_pri);
1948 mtx_unlock_spin(&umtx_lock);
1950 owner = casuword32(&m->m_owner,
1951 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
1953 if (owner == UMUTEX_CONTESTED) {
1958 /* The address was invalid. */
1964 if ((flags & UMUTEX_ERROR_CHECK) != 0 &&
1965 (owner & ~UMUTEX_CONTESTED) == id) {
1976 * If we caught a signal, we have retried and now
1982 umtxq_lock(&uq->uq_key);
1984 umtxq_unbusy(&uq->uq_key);
1985 error = umtxq_sleep(uq, "umtxpp", timo);
1987 umtxq_unlock(&uq->uq_key);
1989 mtx_lock_spin(&umtx_lock);
1990 uq->uq_inherited_pri = old_inherited_pri;
1992 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
1993 uq2 = TAILQ_FIRST(&pi->pi_blocked);
1995 if (pri > UPRI(uq2->uq_thread))
1996 pri = UPRI(uq2->uq_thread);
1999 if (pri > uq->uq_inherited_pri)
2000 pri = uq->uq_inherited_pri;
2002 sched_unlend_user_prio(td, pri);
2004 mtx_unlock_spin(&umtx_lock);
2008 mtx_lock_spin(&umtx_lock);
2009 uq->uq_inherited_pri = old_inherited_pri;
2011 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2012 uq2 = TAILQ_FIRST(&pi->pi_blocked);
2014 if (pri > UPRI(uq2->uq_thread))
2015 pri = UPRI(uq2->uq_thread);
2018 if (pri > uq->uq_inherited_pri)
2019 pri = uq->uq_inherited_pri;
2021 sched_unlend_user_prio(td, pri);
2023 mtx_unlock_spin(&umtx_lock);
2027 umtxq_lock(&uq->uq_key);
2028 umtxq_unbusy(&uq->uq_key);
2029 umtxq_unlock(&uq->uq_key);
2030 umtx_key_release(&uq->uq_key);
2035 * Unlock a PP mutex.
2038 do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags)
2040 struct umtx_key key;
2041 struct umtx_q *uq, *uq2;
2045 int error, pri, new_inherited_pri, su;
2049 su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0);
2052 * Make sure we own this mtx.
2054 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
2058 if ((owner & ~UMUTEX_CONTESTED) != id)
2061 error = copyin(&m->m_ceilings[1], &rceiling, sizeof(uint32_t));
2066 new_inherited_pri = PRI_MAX;
2068 rceiling = RTP_PRIO_MAX - rceiling;
2069 if (rceiling > RTP_PRIO_MAX)
2071 new_inherited_pri = PRI_MIN_REALTIME + rceiling;
2074 if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags),
2081 * For priority protected mutex, always set unlocked state
2082 * to UMUTEX_CONTESTED, so that userland always enters kernel
2083 * to lock the mutex, it is necessary because thread priority
2084 * has to be adjusted for such mutex.
2086 error = suword32(__DEVOLATILE(uint32_t *, &m->m_owner),
2091 umtxq_signal(&key, 1);
2098 mtx_lock_spin(&umtx_lock);
2100 uq->uq_inherited_pri = new_inherited_pri;
2102 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2103 uq2 = TAILQ_FIRST(&pi->pi_blocked);
2105 if (pri > UPRI(uq2->uq_thread))
2106 pri = UPRI(uq2->uq_thread);
2109 if (pri > uq->uq_inherited_pri)
2110 pri = uq->uq_inherited_pri;
2112 sched_unlend_user_prio(td, pri);
2114 mtx_unlock_spin(&umtx_lock);
2116 umtx_key_release(&key);
2121 do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling,
2122 uint32_t *old_ceiling)
2125 uint32_t save_ceiling;
2130 flags = fuword32(&m->m_flags);
2131 if ((flags & UMUTEX_PRIO_PROTECT) == 0)
2133 if (ceiling > RTP_PRIO_MAX)
2137 if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags),
2141 umtxq_lock(&uq->uq_key);
2142 umtxq_busy(&uq->uq_key);
2143 umtxq_unlock(&uq->uq_key);
2145 save_ceiling = fuword32(&m->m_ceilings[0]);
2147 owner = casuword32(&m->m_owner,
2148 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
2150 if (owner == UMUTEX_CONTESTED) {
2151 suword32(&m->m_ceilings[0], ceiling);
2152 suword32(__DEVOLATILE(uint32_t *, &m->m_owner),
2158 /* The address was invalid. */
2164 if ((owner & ~UMUTEX_CONTESTED) == id) {
2165 suword32(&m->m_ceilings[0], ceiling);
2171 * If we caught a signal, we have retried and now
2178 * We set the contested bit, sleep. Otherwise the lock changed
2179 * and we need to retry or we lost a race to the thread
2180 * unlocking the umtx.
2182 umtxq_lock(&uq->uq_key);
2184 umtxq_unbusy(&uq->uq_key);
2185 error = umtxq_sleep(uq, "umtxpp", 0);
2187 umtxq_unlock(&uq->uq_key);
2189 umtxq_lock(&uq->uq_key);
2191 umtxq_signal(&uq->uq_key, INT_MAX);
2192 umtxq_unbusy(&uq->uq_key);
2193 umtxq_unlock(&uq->uq_key);
2194 umtx_key_release(&uq->uq_key);
2195 if (error == 0 && old_ceiling != NULL)
2196 suword32(old_ceiling, save_ceiling);
2201 _do_lock_umutex(struct thread *td, struct umutex *m, int flags, int timo,
2204 switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) {
2206 return (_do_lock_normal(td, m, flags, timo, mode));
2207 case UMUTEX_PRIO_INHERIT:
2208 return (_do_lock_pi(td, m, flags, timo, mode));
2209 case UMUTEX_PRIO_PROTECT:
2210 return (_do_lock_pp(td, m, flags, timo, mode));
2216 * Lock a userland POSIX mutex.
2219 do_lock_umutex(struct thread *td, struct umutex *m,
2220 struct timespec *timeout, int mode)
2222 struct timespec ts, ts2, ts3;
2227 flags = fuword32(&m->m_flags);
2231 if (timeout == NULL) {
2232 error = _do_lock_umutex(td, m, flags, 0, mode);
2233 /* Mutex locking is restarted if it is interrupted. */
2234 if (error == EINTR && mode != _UMUTEX_WAIT)
2238 timespecadd(&ts, timeout);
2239 TIMESPEC_TO_TIMEVAL(&tv, timeout);
2241 error = _do_lock_umutex(td, m, flags, tvtohz(&tv), mode);
2242 if (error != ETIMEDOUT)
2244 getnanouptime(&ts2);
2245 if (timespeccmp(&ts2, &ts, >=)) {
2250 timespecsub(&ts3, &ts2);
2251 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
2253 /* Timed-locking is not restarted. */
2254 if (error == ERESTART)
2261 * Unlock a userland POSIX mutex.
2264 do_unlock_umutex(struct thread *td, struct umutex *m)
2268 flags = fuword32(&m->m_flags);
2272 switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) {
2274 return (do_unlock_normal(td, m, flags));
2275 case UMUTEX_PRIO_INHERIT:
2276 return (do_unlock_pi(td, m, flags));
2277 case UMUTEX_PRIO_PROTECT:
2278 return (do_unlock_pp(td, m, flags));
2285 do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m,
2286 struct timespec *timeout, u_long wflags)
2290 struct timespec cts, ets, tts;
2295 flags = fuword32(&cv->c_flags);
2296 error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key);
2299 umtxq_lock(&uq->uq_key);
2300 umtxq_busy(&uq->uq_key);
2302 umtxq_unlock(&uq->uq_key);
2305 * The magic thing is we should set c_has_waiters to 1 before
2306 * releasing user mutex.
2308 suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 1);
2310 umtxq_lock(&uq->uq_key);
2311 umtxq_unbusy(&uq->uq_key);
2312 umtxq_unlock(&uq->uq_key);
2314 error = do_unlock_umutex(td, m);
2316 umtxq_lock(&uq->uq_key);
2318 if ((wflags & UMTX_CHECK_UNPARKING) &&
2319 (td->td_pflags & TDP_WAKEUP)) {
2320 td->td_pflags &= ~TDP_WAKEUP;
2322 } else if (timeout == NULL) {
2323 error = umtxq_sleep(uq, "ucond", 0);
2325 getnanouptime(&ets);
2326 timespecadd(&ets, timeout);
2327 TIMESPEC_TO_TIMEVAL(&tv, timeout);
2329 error = umtxq_sleep(uq, "ucond", tvtohz(&tv));
2330 if (error != ETIMEDOUT)
2332 getnanouptime(&cts);
2333 if (timespeccmp(&cts, &ets, >=)) {
2338 timespecsub(&tts, &cts);
2339 TIMESPEC_TO_TIMEVAL(&tv, &tts);
2345 if ((uq->uq_flags & UQF_UMTXQ) == 0) {
2347 * If we concurrently got do_cv_signal()d
2348 * and we got an error or UNIX signals or a timeout,
2349 * then, perform another umtxq_signal to avoid
2350 * consuming the wakeup. This may cause supurious
2351 * wakeup for another thread which was just queued,
2352 * but SUSV3 explicitly allows supurious wakeup to
2353 * occur, and indeed a kernel based implementation
2356 if (!umtxq_signal(&uq->uq_key, 1))
2359 if (error == ERESTART)
2363 umtxq_unlock(&uq->uq_key);
2364 umtx_key_release(&uq->uq_key);
2369 * Signal a userland condition variable.
2372 do_cv_signal(struct thread *td, struct ucond *cv)
2374 struct umtx_key key;
2375 int error, cnt, nwake;
2378 flags = fuword32(&cv->c_flags);
2379 if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0)
2383 cnt = umtxq_count(&key);
2384 nwake = umtxq_signal(&key, 1);
2388 __DEVOLATILE(uint32_t *, &cv->c_has_waiters), 0);
2393 umtx_key_release(&key);
2398 do_cv_broadcast(struct thread *td, struct ucond *cv)
2400 struct umtx_key key;
2404 flags = fuword32(&cv->c_flags);
2405 if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0)
2410 umtxq_signal(&key, INT_MAX);
2413 error = suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 0);
2419 umtx_key_release(&key);
2424 do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag, int timo)
2427 uint32_t flags, wrflags;
2428 int32_t state, oldstate;
2429 int32_t blocked_readers;
2433 flags = fuword32(&rwlock->rw_flags);
2434 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
2438 wrflags = URWLOCK_WRITE_OWNER;
2439 if (!(fflag & URWLOCK_PREFER_READER) && !(flags & URWLOCK_PREFER_READER))
2440 wrflags |= URWLOCK_WRITE_WAITERS;
2443 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2444 /* try to lock it */
2445 while (!(state & wrflags)) {
2446 if (__predict_false(URWLOCK_READER_COUNT(state) == URWLOCK_MAX_READERS)) {
2447 umtx_key_release(&uq->uq_key);
2450 oldstate = casuword32(&rwlock->rw_state, state, state + 1);
2451 if (oldstate == state) {
2452 umtx_key_release(&uq->uq_key);
2461 /* grab monitor lock */
2462 umtxq_lock(&uq->uq_key);
2463 umtxq_busy(&uq->uq_key);
2464 umtxq_unlock(&uq->uq_key);
2466 /* set read contention bit */
2467 while ((state & wrflags) && !(state & URWLOCK_READ_WAITERS)) {
2468 oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_READ_WAITERS);
2469 if (oldstate == state)
2474 /* state is changed while setting flags, restart */
2475 if (!(state & wrflags)) {
2476 umtxq_lock(&uq->uq_key);
2477 umtxq_unbusy(&uq->uq_key);
2478 umtxq_unlock(&uq->uq_key);
2483 /* contention bit is set, before sleeping, increase read waiter count */
2484 blocked_readers = fuword32(&rwlock->rw_blocked_readers);
2485 suword32(&rwlock->rw_blocked_readers, blocked_readers+1);
2487 while (state & wrflags) {
2488 umtxq_lock(&uq->uq_key);
2490 umtxq_unbusy(&uq->uq_key);
2492 error = umtxq_sleep(uq, "urdlck", timo);
2494 umtxq_busy(&uq->uq_key);
2496 umtxq_unlock(&uq->uq_key);
2499 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2502 /* decrease read waiter count, and may clear read contention bit */
2503 blocked_readers = fuword32(&rwlock->rw_blocked_readers);
2504 suword32(&rwlock->rw_blocked_readers, blocked_readers-1);
2505 if (blocked_readers == 1) {
2506 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2508 oldstate = casuword32(&rwlock->rw_state, state,
2509 state & ~URWLOCK_READ_WAITERS);
2510 if (oldstate == state)
2516 umtxq_lock(&uq->uq_key);
2517 umtxq_unbusy(&uq->uq_key);
2518 umtxq_unlock(&uq->uq_key);
2520 umtx_key_release(&uq->uq_key);
2525 do_rw_rdlock2(struct thread *td, void *obj, long val, struct timespec *timeout)
2527 struct timespec ts, ts2, ts3;
2532 timespecadd(&ts, timeout);
2533 TIMESPEC_TO_TIMEVAL(&tv, timeout);
2535 error = do_rw_rdlock(td, obj, val, tvtohz(&tv));
2536 if (error != ETIMEDOUT)
2538 getnanouptime(&ts2);
2539 if (timespeccmp(&ts2, &ts, >=)) {
2544 timespecsub(&ts3, &ts2);
2545 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
2547 if (error == ERESTART)
2553 do_rw_wrlock(struct thread *td, struct urwlock *rwlock, int timo)
2557 int32_t state, oldstate;
2558 int32_t blocked_writers;
2562 flags = fuword32(&rwlock->rw_flags);
2563 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
2568 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2569 while (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) {
2570 oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_WRITE_OWNER);
2571 if (oldstate == state) {
2572 umtx_key_release(&uq->uq_key);
2581 /* grab monitor lock */
2582 umtxq_lock(&uq->uq_key);
2583 umtxq_busy(&uq->uq_key);
2584 umtxq_unlock(&uq->uq_key);
2586 while (((state & URWLOCK_WRITE_OWNER) || URWLOCK_READER_COUNT(state) != 0) &&
2587 (state & URWLOCK_WRITE_WAITERS) == 0) {
2588 oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_WRITE_WAITERS);
2589 if (oldstate == state)
2594 if (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) {
2595 umtxq_lock(&uq->uq_key);
2596 umtxq_unbusy(&uq->uq_key);
2597 umtxq_unlock(&uq->uq_key);
2601 blocked_writers = fuword32(&rwlock->rw_blocked_writers);
2602 suword32(&rwlock->rw_blocked_writers, blocked_writers+1);
2604 while ((state & URWLOCK_WRITE_OWNER) || URWLOCK_READER_COUNT(state) != 0) {
2605 umtxq_lock(&uq->uq_key);
2606 umtxq_insert_queue(uq, UMTX_EXCLUSIVE_QUEUE);
2607 umtxq_unbusy(&uq->uq_key);
2609 error = umtxq_sleep(uq, "uwrlck", timo);
2611 umtxq_busy(&uq->uq_key);
2612 umtxq_remove_queue(uq, UMTX_EXCLUSIVE_QUEUE);
2613 umtxq_unlock(&uq->uq_key);
2616 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2619 blocked_writers = fuword32(&rwlock->rw_blocked_writers);
2620 suword32(&rwlock->rw_blocked_writers, blocked_writers-1);
2621 if (blocked_writers == 1) {
2622 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2624 oldstate = casuword32(&rwlock->rw_state, state,
2625 state & ~URWLOCK_WRITE_WAITERS);
2626 if (oldstate == state)
2632 umtxq_lock(&uq->uq_key);
2633 umtxq_unbusy(&uq->uq_key);
2634 umtxq_unlock(&uq->uq_key);
2637 umtx_key_release(&uq->uq_key);
2642 do_rw_wrlock2(struct thread *td, void *obj, struct timespec *timeout)
2644 struct timespec ts, ts2, ts3;
2649 timespecadd(&ts, timeout);
2650 TIMESPEC_TO_TIMEVAL(&tv, timeout);
2652 error = do_rw_wrlock(td, obj, tvtohz(&tv));
2653 if (error != ETIMEDOUT)
2655 getnanouptime(&ts2);
2656 if (timespeccmp(&ts2, &ts, >=)) {
2661 timespecsub(&ts3, &ts2);
2662 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
2664 if (error == ERESTART)
2670 do_rw_unlock(struct thread *td, struct urwlock *rwlock)
2674 int32_t state, oldstate;
2675 int error, q, count;
2678 flags = fuword32(&rwlock->rw_flags);
2679 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
2683 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
2684 if (state & URWLOCK_WRITE_OWNER) {
2686 oldstate = casuword32(&rwlock->rw_state, state,
2687 state & ~URWLOCK_WRITE_OWNER);
2688 if (oldstate != state) {
2690 if (!(oldstate & URWLOCK_WRITE_OWNER)) {
2697 } else if (URWLOCK_READER_COUNT(state) != 0) {
2699 oldstate = casuword32(&rwlock->rw_state, state,
2701 if (oldstate != state) {
2703 if (URWLOCK_READER_COUNT(oldstate) == 0) {
2718 if (!(flags & URWLOCK_PREFER_READER)) {
2719 if (state & URWLOCK_WRITE_WAITERS) {
2721 q = UMTX_EXCLUSIVE_QUEUE;
2722 } else if (state & URWLOCK_READ_WAITERS) {
2724 q = UMTX_SHARED_QUEUE;
2727 if (state & URWLOCK_READ_WAITERS) {
2729 q = UMTX_SHARED_QUEUE;
2730 } else if (state & URWLOCK_WRITE_WAITERS) {
2732 q = UMTX_EXCLUSIVE_QUEUE;
2737 umtxq_lock(&uq->uq_key);
2738 umtxq_busy(&uq->uq_key);
2739 umtxq_signal_queue(&uq->uq_key, count, q);
2740 umtxq_unbusy(&uq->uq_key);
2741 umtxq_unlock(&uq->uq_key);
2744 umtx_key_release(&uq->uq_key);
2749 _umtx_lock(struct thread *td, struct _umtx_lock_args *uap)
2750 /* struct umtx *umtx */
2752 return _do_lock_umtx(td, uap->umtx, td->td_tid, 0);
2756 _umtx_unlock(struct thread *td, struct _umtx_unlock_args *uap)
2757 /* struct umtx *umtx */
2759 return do_unlock_umtx(td, uap->umtx, td->td_tid);
2763 __umtx_op_lock_umtx(struct thread *td, struct _umtx_op_args *uap)
2765 struct timespec *ts, timeout;
2768 /* Allow a null timespec (wait forever). */
2769 if (uap->uaddr2 == NULL)
2772 error = copyin(uap->uaddr2, &timeout, sizeof(timeout));
2775 if (timeout.tv_nsec >= 1000000000 ||
2776 timeout.tv_nsec < 0) {
2781 return (do_lock_umtx(td, uap->obj, uap->val, ts));
2785 __umtx_op_unlock_umtx(struct thread *td, struct _umtx_op_args *uap)
2787 return (do_unlock_umtx(td, uap->obj, uap->val));
2791 __umtx_op_wait(struct thread *td, struct _umtx_op_args *uap)
2793 struct timespec *ts, timeout;
2796 if (uap->uaddr2 == NULL)
2799 error = copyin(uap->uaddr2, &timeout, sizeof(timeout));
2802 if (timeout.tv_nsec >= 1000000000 ||
2803 timeout.tv_nsec < 0)
2807 return do_wait(td, uap->obj, uap->val, ts, 0, 0);
2811 __umtx_op_wait_uint(struct thread *td, struct _umtx_op_args *uap)
2813 struct timespec *ts, timeout;
2816 if (uap->uaddr2 == NULL)
2819 error = copyin(uap->uaddr2, &timeout, sizeof(timeout));
2822 if (timeout.tv_nsec >= 1000000000 ||
2823 timeout.tv_nsec < 0)
2827 return do_wait(td, uap->obj, uap->val, ts, 1, 0);
2831 __umtx_op_wait_uint_private(struct thread *td, struct _umtx_op_args *uap)
2833 struct timespec *ts, timeout;
2836 if (uap->uaddr2 == NULL)
2839 error = copyin(uap->uaddr2, &timeout, sizeof(timeout));
2842 if (timeout.tv_nsec >= 1000000000 ||
2843 timeout.tv_nsec < 0)
2847 return do_wait(td, uap->obj, uap->val, ts, 1, 1);
2851 __umtx_op_wake(struct thread *td, struct _umtx_op_args *uap)
2853 return (kern_umtx_wake(td, uap->obj, uap->val, 0));
2857 __umtx_op_wake_private(struct thread *td, struct _umtx_op_args *uap)
2859 return (kern_umtx_wake(td, uap->obj, uap->val, 1));
2863 __umtx_op_lock_umutex(struct thread *td, struct _umtx_op_args *uap)
2865 struct timespec *ts, timeout;
2868 /* Allow a null timespec (wait forever). */
2869 if (uap->uaddr2 == NULL)
2872 error = copyin(uap->uaddr2, &timeout,
2876 if (timeout.tv_nsec >= 1000000000 ||
2877 timeout.tv_nsec < 0) {
2882 return do_lock_umutex(td, uap->obj, ts, 0);
2886 __umtx_op_trylock_umutex(struct thread *td, struct _umtx_op_args *uap)
2888 return do_lock_umutex(td, uap->obj, NULL, _UMUTEX_TRY);
2892 __umtx_op_wait_umutex(struct thread *td, struct _umtx_op_args *uap)
2894 struct timespec *ts, timeout;
2897 /* Allow a null timespec (wait forever). */
2898 if (uap->uaddr2 == NULL)
2901 error = copyin(uap->uaddr2, &timeout,
2905 if (timeout.tv_nsec >= 1000000000 ||
2906 timeout.tv_nsec < 0) {
2911 return do_lock_umutex(td, uap->obj, ts, _UMUTEX_WAIT);
2915 __umtx_op_wake_umutex(struct thread *td, struct _umtx_op_args *uap)
2917 return do_wake_umutex(td, uap->obj);
2921 __umtx_op_unlock_umutex(struct thread *td, struct _umtx_op_args *uap)
2923 return do_unlock_umutex(td, uap->obj);
2927 __umtx_op_set_ceiling(struct thread *td, struct _umtx_op_args *uap)
2929 return do_set_ceiling(td, uap->obj, uap->val, uap->uaddr1);
2933 __umtx_op_cv_wait(struct thread *td, struct _umtx_op_args *uap)
2935 struct timespec *ts, timeout;
2938 /* Allow a null timespec (wait forever). */
2939 if (uap->uaddr2 == NULL)
2942 error = copyin(uap->uaddr2, &timeout,
2946 if (timeout.tv_nsec >= 1000000000 ||
2947 timeout.tv_nsec < 0) {
2952 return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val));
2956 __umtx_op_cv_signal(struct thread *td, struct _umtx_op_args *uap)
2958 return do_cv_signal(td, uap->obj);
2962 __umtx_op_cv_broadcast(struct thread *td, struct _umtx_op_args *uap)
2964 return do_cv_broadcast(td, uap->obj);
2968 __umtx_op_rw_rdlock(struct thread *td, struct _umtx_op_args *uap)
2970 struct timespec timeout;
2973 /* Allow a null timespec (wait forever). */
2974 if (uap->uaddr2 == NULL) {
2975 error = do_rw_rdlock(td, uap->obj, uap->val, 0);
2977 error = copyin(uap->uaddr2, &timeout,
2981 if (timeout.tv_nsec >= 1000000000 ||
2982 timeout.tv_nsec < 0) {
2985 error = do_rw_rdlock2(td, uap->obj, uap->val, &timeout);
2991 __umtx_op_rw_wrlock(struct thread *td, struct _umtx_op_args *uap)
2993 struct timespec timeout;
2996 /* Allow a null timespec (wait forever). */
2997 if (uap->uaddr2 == NULL) {
2998 error = do_rw_wrlock(td, uap->obj, 0);
3000 error = copyin(uap->uaddr2, &timeout,
3004 if (timeout.tv_nsec >= 1000000000 ||
3005 timeout.tv_nsec < 0) {
3009 error = do_rw_wrlock2(td, uap->obj, &timeout);
3015 __umtx_op_rw_unlock(struct thread *td, struct _umtx_op_args *uap)
3017 return do_rw_unlock(td, uap->obj);
3020 typedef int (*_umtx_op_func)(struct thread *td, struct _umtx_op_args *uap);
3022 static _umtx_op_func op_table[] = {
3023 __umtx_op_lock_umtx, /* UMTX_OP_LOCK */
3024 __umtx_op_unlock_umtx, /* UMTX_OP_UNLOCK */
3025 __umtx_op_wait, /* UMTX_OP_WAIT */
3026 __umtx_op_wake, /* UMTX_OP_WAKE */
3027 __umtx_op_trylock_umutex, /* UMTX_OP_MUTEX_TRYLOCK */
3028 __umtx_op_lock_umutex, /* UMTX_OP_MUTEX_LOCK */
3029 __umtx_op_unlock_umutex, /* UMTX_OP_MUTEX_UNLOCK */
3030 __umtx_op_set_ceiling, /* UMTX_OP_SET_CEILING */
3031 __umtx_op_cv_wait, /* UMTX_OP_CV_WAIT*/
3032 __umtx_op_cv_signal, /* UMTX_OP_CV_SIGNAL */
3033 __umtx_op_cv_broadcast, /* UMTX_OP_CV_BROADCAST */
3034 __umtx_op_wait_uint, /* UMTX_OP_WAIT_UINT */
3035 __umtx_op_rw_rdlock, /* UMTX_OP_RW_RDLOCK */
3036 __umtx_op_rw_wrlock, /* UMTX_OP_RW_WRLOCK */
3037 __umtx_op_rw_unlock, /* UMTX_OP_RW_UNLOCK */
3038 __umtx_op_wait_uint_private, /* UMTX_OP_WAIT_UINT_PRIVATE */
3039 __umtx_op_wake_private, /* UMTX_OP_WAKE_PRIVATE */
3040 __umtx_op_wait_umutex, /* UMTX_OP_UMUTEX_WAIT */
3041 __umtx_op_wake_umutex /* UMTX_OP_UMUTEX_WAKE */
3045 _umtx_op(struct thread *td, struct _umtx_op_args *uap)
3047 if ((unsigned)uap->op < UMTX_OP_MAX)
3048 return (*op_table[uap->op])(td, uap);
3054 freebsd32_umtx_lock(struct thread *td, struct freebsd32_umtx_lock_args *uap)
3055 /* struct umtx *umtx */
3057 return (do_lock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid, NULL));
3061 freebsd32_umtx_unlock(struct thread *td, struct freebsd32_umtx_unlock_args *uap)
3062 /* struct umtx *umtx */
3064 return (do_unlock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid));
3073 copyin_timeout32(void *addr, struct timespec *tsp)
3075 struct timespec32 ts32;
3078 error = copyin(addr, &ts32, sizeof(struct timespec32));
3080 tsp->tv_sec = ts32.tv_sec;
3081 tsp->tv_nsec = ts32.tv_nsec;
3087 __umtx_op_lock_umtx_compat32(struct thread *td, struct _umtx_op_args *uap)
3089 struct timespec *ts, timeout;
3092 /* Allow a null timespec (wait forever). */
3093 if (uap->uaddr2 == NULL)
3096 error = copyin_timeout32(uap->uaddr2, &timeout);
3099 if (timeout.tv_nsec >= 1000000000 ||
3100 timeout.tv_nsec < 0) {
3105 return (do_lock_umtx32(td, uap->obj, uap->val, ts));
3109 __umtx_op_unlock_umtx_compat32(struct thread *td, struct _umtx_op_args *uap)
3111 return (do_unlock_umtx32(td, uap->obj, (uint32_t)uap->val));
3115 __umtx_op_wait_compat32(struct thread *td, struct _umtx_op_args *uap)
3117 struct timespec *ts, timeout;
3120 if (uap->uaddr2 == NULL)
3123 error = copyin_timeout32(uap->uaddr2, &timeout);
3126 if (timeout.tv_nsec >= 1000000000 ||
3127 timeout.tv_nsec < 0)
3131 return do_wait(td, uap->obj, uap->val, ts, 1, 0);
3135 __umtx_op_lock_umutex_compat32(struct thread *td, struct _umtx_op_args *uap)
3137 struct timespec *ts, timeout;
3140 /* Allow a null timespec (wait forever). */
3141 if (uap->uaddr2 == NULL)
3144 error = copyin_timeout32(uap->uaddr2, &timeout);
3147 if (timeout.tv_nsec >= 1000000000 ||
3148 timeout.tv_nsec < 0)
3152 return do_lock_umutex(td, uap->obj, ts, 0);
3156 __umtx_op_wait_umutex_compat32(struct thread *td, struct _umtx_op_args *uap)
3158 struct timespec *ts, timeout;
3161 /* Allow a null timespec (wait forever). */
3162 if (uap->uaddr2 == NULL)
3165 error = copyin_timeout32(uap->uaddr2, &timeout);
3168 if (timeout.tv_nsec >= 1000000000 ||
3169 timeout.tv_nsec < 0)
3173 return do_lock_umutex(td, uap->obj, ts, _UMUTEX_WAIT);
3177 __umtx_op_cv_wait_compat32(struct thread *td, struct _umtx_op_args *uap)
3179 struct timespec *ts, timeout;
3182 /* Allow a null timespec (wait forever). */
3183 if (uap->uaddr2 == NULL)
3186 error = copyin_timeout32(uap->uaddr2, &timeout);
3189 if (timeout.tv_nsec >= 1000000000 ||
3190 timeout.tv_nsec < 0)
3194 return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val));
3198 __umtx_op_rw_rdlock_compat32(struct thread *td, struct _umtx_op_args *uap)
3200 struct timespec timeout;
3203 /* Allow a null timespec (wait forever). */
3204 if (uap->uaddr2 == NULL) {
3205 error = do_rw_rdlock(td, uap->obj, uap->val, 0);
3207 error = copyin(uap->uaddr2, &timeout,
3211 if (timeout.tv_nsec >= 1000000000 ||
3212 timeout.tv_nsec < 0) {
3215 error = do_rw_rdlock2(td, uap->obj, uap->val, &timeout);
3221 __umtx_op_rw_wrlock_compat32(struct thread *td, struct _umtx_op_args *uap)
3223 struct timespec timeout;
3226 /* Allow a null timespec (wait forever). */
3227 if (uap->uaddr2 == NULL) {
3228 error = do_rw_wrlock(td, uap->obj, 0);
3230 error = copyin_timeout32(uap->uaddr2, &timeout);
3233 if (timeout.tv_nsec >= 1000000000 ||
3234 timeout.tv_nsec < 0) {
3238 error = do_rw_wrlock2(td, uap->obj, &timeout);
3244 __umtx_op_wait_uint_private_compat32(struct thread *td, struct _umtx_op_args *uap)
3246 struct timespec *ts, timeout;
3249 if (uap->uaddr2 == NULL)
3252 error = copyin_timeout32(uap->uaddr2, &timeout);
3255 if (timeout.tv_nsec >= 1000000000 ||
3256 timeout.tv_nsec < 0)
3260 return do_wait(td, uap->obj, uap->val, ts, 1, 1);
3263 static _umtx_op_func op_table_compat32[] = {
3264 __umtx_op_lock_umtx_compat32, /* UMTX_OP_LOCK */
3265 __umtx_op_unlock_umtx_compat32, /* UMTX_OP_UNLOCK */
3266 __umtx_op_wait_compat32, /* UMTX_OP_WAIT */
3267 __umtx_op_wake, /* UMTX_OP_WAKE */
3268 __umtx_op_trylock_umutex, /* UMTX_OP_MUTEX_LOCK */
3269 __umtx_op_lock_umutex_compat32, /* UMTX_OP_MUTEX_TRYLOCK */
3270 __umtx_op_unlock_umutex, /* UMTX_OP_MUTEX_UNLOCK */
3271 __umtx_op_set_ceiling, /* UMTX_OP_SET_CEILING */
3272 __umtx_op_cv_wait_compat32, /* UMTX_OP_CV_WAIT*/
3273 __umtx_op_cv_signal, /* UMTX_OP_CV_SIGNAL */
3274 __umtx_op_cv_broadcast, /* UMTX_OP_CV_BROADCAST */
3275 __umtx_op_wait_compat32, /* UMTX_OP_WAIT_UINT */
3276 __umtx_op_rw_rdlock_compat32, /* UMTX_OP_RW_RDLOCK */
3277 __umtx_op_rw_wrlock_compat32, /* UMTX_OP_RW_WRLOCK */
3278 __umtx_op_rw_unlock, /* UMTX_OP_RW_UNLOCK */
3279 __umtx_op_wait_uint_private_compat32, /* UMTX_OP_WAIT_UINT_PRIVATE */
3280 __umtx_op_wake_private, /* UMTX_OP_WAKE_PRIVATE */
3281 __umtx_op_wait_umutex_compat32, /* UMTX_OP_UMUTEX_WAIT */
3282 __umtx_op_wake_umutex /* UMTX_OP_UMUTEX_WAKE */
3286 freebsd32_umtx_op(struct thread *td, struct freebsd32_umtx_op_args *uap)
3288 if ((unsigned)uap->op < UMTX_OP_MAX)
3289 return (*op_table_compat32[uap->op])(td,
3290 (struct _umtx_op_args *)uap);
3296 umtx_thread_init(struct thread *td)
3298 td->td_umtxq = umtxq_alloc();
3299 td->td_umtxq->uq_thread = td;
3303 umtx_thread_fini(struct thread *td)
3305 umtxq_free(td->td_umtxq);
3309 * It will be called when new thread is created, e.g fork().
3312 umtx_thread_alloc(struct thread *td)
3317 uq->uq_inherited_pri = PRI_MAX;
3319 KASSERT(uq->uq_flags == 0, ("uq_flags != 0"));
3320 KASSERT(uq->uq_thread == td, ("uq_thread != td"));
3321 KASSERT(uq->uq_pi_blocked == NULL, ("uq_pi_blocked != NULL"));
3322 KASSERT(TAILQ_EMPTY(&uq->uq_pi_contested), ("uq_pi_contested is not empty"));
3329 umtx_exec_hook(void *arg __unused, struct proc *p __unused,
3330 struct image_params *imgp __unused)
3332 umtx_thread_cleanup(curthread);
3336 * thread_exit() hook.
3339 umtx_thread_exit(struct thread *td)
3341 umtx_thread_cleanup(td);
3345 * clean up umtx data.
3348 umtx_thread_cleanup(struct thread *td)
3353 if ((uq = td->td_umtxq) == NULL)
3356 mtx_lock_spin(&umtx_lock);
3357 uq->uq_inherited_pri = PRI_MAX;
3358 while ((pi = TAILQ_FIRST(&uq->uq_pi_contested)) != NULL) {
3359 pi->pi_owner = NULL;
3360 TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link);
3363 td->td_flags &= ~TDF_UBORROWING;
3365 mtx_unlock_spin(&umtx_lock);