2 * Copyright (c) 2004, David Xu <davidxu@freebsd.org>
3 * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org>
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice unmodified, this list of conditions, and the following
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
31 #include "opt_compat.h"
32 #include <sys/param.h>
33 #include <sys/kernel.h>
34 #include <sys/limits.h>
36 #include <sys/malloc.h>
37 #include <sys/mutex.h>
40 #include <sys/sched.h>
42 #include <sys/sysctl.h>
43 #include <sys/sysent.h>
44 #include <sys/systm.h>
45 #include <sys/sysproto.h>
46 #include <sys/eventhandler.h>
50 #include <vm/vm_param.h>
52 #include <vm/vm_map.h>
53 #include <vm/vm_object.h>
55 #include <machine/cpu.h>
58 #include <compat/freebsd32/freebsd32_proto.h>
61 #define TYPE_SIMPLE_LOCK 0
62 #define TYPE_SIMPLE_WAIT 1
63 #define TYPE_NORMAL_UMUTEX 2
64 #define TYPE_PI_UMUTEX 3
65 #define TYPE_PP_UMUTEX 4
68 /* Key to represent a unique userland synchronous object */
89 /* Priority inheritance mutex info. */
92 struct thread *pi_owner;
97 /* List entry to link umtx holding by thread */
98 TAILQ_ENTRY(umtx_pi) pi_link;
100 /* List entry in hash */
101 TAILQ_ENTRY(umtx_pi) pi_hashlink;
103 /* List for waiters */
104 TAILQ_HEAD(,umtx_q) pi_blocked;
106 /* Identify a userland lock object */
107 struct umtx_key pi_key;
110 /* A userland synchronous object user. */
112 /* Linked list for the hash. */
113 TAILQ_ENTRY(umtx_q) uq_link;
116 struct umtx_key uq_key;
120 #define UQF_UMTXQ 0x0001
122 /* The thread waits on. */
123 struct thread *uq_thread;
126 * Blocked on PI mutex. read can use chain lock
127 * or umtx_lock, write must have both chain lock and
128 * umtx_lock being hold.
130 struct umtx_pi *uq_pi_blocked;
132 /* On blocked list */
133 TAILQ_ENTRY(umtx_q) uq_lockq;
135 /* Thread contending with us */
136 TAILQ_HEAD(,umtx_pi) uq_pi_contested;
138 /* Inherited priority from PP mutex */
139 u_char uq_inherited_pri;
142 TAILQ_HEAD(umtxq_head, umtx_q);
144 /* Userland lock object's wait-queue chain */
146 /* Lock for this chain. */
149 /* List of sleep queues. */
150 struct umtxq_head uc_queue;
155 /* Chain lock waiters */
158 /* All PI in the list */
159 TAILQ_HEAD(,umtx_pi) uc_pi_list;
162 #define UMTXQ_LOCKED_ASSERT(uc) mtx_assert(&(uc)->uc_lock, MA_OWNED)
165 * Don't propagate time-sharing priority, there is a security reason,
166 * a user can simply introduce PI-mutex, let thread A lock the mutex,
167 * and let another thread B block on the mutex, because B is
168 * sleeping, its priority will be boosted, this causes A's priority to
169 * be boosted via priority propagating too and will never be lowered even
170 * if it is using 100%CPU, this is unfair to other processes.
173 #define UPRI(td) (((td)->td_user_pri >= PRI_MIN_TIMESHARE &&\
174 (td)->td_user_pri <= PRI_MAX_TIMESHARE) ?\
175 PRI_MAX_TIMESHARE : (td)->td_user_pri)
177 #define GOLDEN_RATIO_PRIME 2654404609U
178 #define UMTX_CHAINS 128
179 #define UMTX_SHIFTS (__WORD_BIT - 7)
181 #define THREAD_SHARE 0
182 #define PROCESS_SHARE 1
185 #define GET_SHARE(flags) \
186 (((flags) & USYNC_PROCESS_SHARED) == 0 ? THREAD_SHARE : PROCESS_SHARE)
188 static uma_zone_t umtx_pi_zone;
189 static struct umtxq_chain umtxq_chains[UMTX_CHAINS];
190 static MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory");
191 static int umtx_pi_allocated;
193 SYSCTL_NODE(_debug, OID_AUTO, umtx, CTLFLAG_RW, 0, "umtx debug");
194 SYSCTL_INT(_debug_umtx, OID_AUTO, umtx_pi_allocated, CTLFLAG_RD,
195 &umtx_pi_allocated, 0, "Allocated umtx_pi");
197 static void umtxq_sysinit(void *);
198 static void umtxq_hash(struct umtx_key *key);
199 static struct umtxq_chain *umtxq_getchain(struct umtx_key *key);
200 static void umtxq_lock(struct umtx_key *key);
201 static void umtxq_unlock(struct umtx_key *key);
202 static void umtxq_busy(struct umtx_key *key);
203 static void umtxq_unbusy(struct umtx_key *key);
204 static void umtxq_insert(struct umtx_q *uq);
205 static void umtxq_remove(struct umtx_q *uq);
206 static int umtxq_sleep(struct umtx_q *uq, const char *wmesg, int timo);
207 static int umtxq_count(struct umtx_key *key);
208 static int umtxq_signal(struct umtx_key *key, int nr_wakeup);
209 static int umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2);
210 static int umtx_key_get(void *addr, int type, int share,
211 struct umtx_key *key);
212 static void umtx_key_release(struct umtx_key *key);
213 static struct umtx_pi *umtx_pi_alloc(int);
214 static void umtx_pi_free(struct umtx_pi *pi);
215 static int do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags);
216 static void umtx_thread_cleanup(struct thread *td);
217 static void umtx_exec_hook(void *arg __unused, struct proc *p __unused,
218 struct image_params *imgp __unused);
219 SYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_sysinit, NULL);
221 static struct mtx umtx_lock;
224 umtxq_sysinit(void *arg __unused)
228 umtx_pi_zone = uma_zcreate("umtx pi", sizeof(struct umtx_pi),
229 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
230 for (i = 0; i < UMTX_CHAINS; ++i) {
231 mtx_init(&umtxq_chains[i].uc_lock, "umtxql", NULL,
232 MTX_DEF | MTX_DUPOK);
233 TAILQ_INIT(&umtxq_chains[i].uc_queue);
234 TAILQ_INIT(&umtxq_chains[i].uc_pi_list);
235 umtxq_chains[i].uc_busy = 0;
236 umtxq_chains[i].uc_waiters = 0;
238 mtx_init(&umtx_lock, "umtx lock", NULL, MTX_SPIN);
239 EVENTHANDLER_REGISTER(process_exec, umtx_exec_hook, NULL,
240 EVENTHANDLER_PRI_ANY);
248 uq = malloc(sizeof(struct umtx_q), M_UMTX, M_WAITOK | M_ZERO);
249 TAILQ_INIT(&uq->uq_pi_contested);
250 uq->uq_inherited_pri = PRI_MAX;
255 umtxq_free(struct umtx_q *uq)
261 umtxq_hash(struct umtx_key *key)
263 unsigned n = (uintptr_t)key->info.both.a + key->info.both.b;
264 key->hash = ((n * GOLDEN_RATIO_PRIME) >> UMTX_SHIFTS) % UMTX_CHAINS;
268 umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2)
270 return (k1->type == k2->type &&
271 k1->info.both.a == k2->info.both.a &&
272 k1->info.both.b == k2->info.both.b);
275 static inline struct umtxq_chain *
276 umtxq_getchain(struct umtx_key *key)
278 return (&umtxq_chains[key->hash]);
282 * Set chain to busy state when following operation
283 * may be blocked (kernel mutex can not be used).
286 umtxq_busy(struct umtx_key *key)
288 struct umtxq_chain *uc;
290 uc = umtxq_getchain(key);
291 mtx_assert(&uc->uc_lock, MA_OWNED);
292 while (uc->uc_busy != 0) {
294 msleep(uc, &uc->uc_lock, 0, "umtxqb", 0);
304 umtxq_unbusy(struct umtx_key *key)
306 struct umtxq_chain *uc;
308 uc = umtxq_getchain(key);
309 mtx_assert(&uc->uc_lock, MA_OWNED);
310 KASSERT(uc->uc_busy != 0, ("not busy"));
320 umtxq_lock(struct umtx_key *key)
322 struct umtxq_chain *uc;
324 uc = umtxq_getchain(key);
325 mtx_lock(&uc->uc_lock);
332 umtxq_unlock(struct umtx_key *key)
334 struct umtxq_chain *uc;
336 uc = umtxq_getchain(key);
337 mtx_unlock(&uc->uc_lock);
341 * Insert a thread onto the umtx queue.
344 umtxq_insert(struct umtx_q *uq)
346 struct umtxq_chain *uc;
348 uc = umtxq_getchain(&uq->uq_key);
349 UMTXQ_LOCKED_ASSERT(uc);
350 TAILQ_INSERT_TAIL(&uc->uc_queue, uq, uq_link);
351 uq->uq_flags |= UQF_UMTXQ;
355 * Remove thread from the umtx queue.
358 umtxq_remove(struct umtx_q *uq)
360 struct umtxq_chain *uc;
362 uc = umtxq_getchain(&uq->uq_key);
363 UMTXQ_LOCKED_ASSERT(uc);
364 if (uq->uq_flags & UQF_UMTXQ) {
365 TAILQ_REMOVE(&uc->uc_queue, uq, uq_link);
366 uq->uq_flags &= ~UQF_UMTXQ;
371 * Check if there are multiple waiters
374 umtxq_count(struct umtx_key *key)
376 struct umtxq_chain *uc;
380 uc = umtxq_getchain(key);
381 UMTXQ_LOCKED_ASSERT(uc);
382 TAILQ_FOREACH(uq, &uc->uc_queue, uq_link) {
383 if (umtx_key_match(&uq->uq_key, key)) {
392 * Check if there are multiple PI waiters and returns first
396 umtxq_count_pi(struct umtx_key *key, struct umtx_q **first)
398 struct umtxq_chain *uc;
403 uc = umtxq_getchain(key);
404 UMTXQ_LOCKED_ASSERT(uc);
405 TAILQ_FOREACH(uq, &uc->uc_queue, uq_link) {
406 if (umtx_key_match(&uq->uq_key, key)) {
416 * Wake up threads waiting on an userland object.
419 umtxq_signal(struct umtx_key *key, int n_wake)
421 struct umtxq_chain *uc;
422 struct umtx_q *uq, *next;
426 uc = umtxq_getchain(key);
427 UMTXQ_LOCKED_ASSERT(uc);
428 TAILQ_FOREACH_SAFE(uq, &uc->uc_queue, uq_link, next) {
429 if (umtx_key_match(&uq->uq_key, key)) {
440 * Wake up specified thread.
443 umtxq_signal_thread(struct umtx_q *uq)
445 struct umtxq_chain *uc;
447 uc = umtxq_getchain(&uq->uq_key);
448 UMTXQ_LOCKED_ASSERT(uc);
454 * Put thread into sleep state, before sleeping, check if
455 * thread was removed from umtx queue.
458 umtxq_sleep(struct umtx_q *uq, const char *wmesg, int timo)
460 struct umtxq_chain *uc;
463 uc = umtxq_getchain(&uq->uq_key);
464 UMTXQ_LOCKED_ASSERT(uc);
465 if (!(uq->uq_flags & UQF_UMTXQ))
467 error = msleep(uq, &uc->uc_lock, PCATCH, wmesg, timo);
468 if (error == EWOULDBLOCK)
474 * Convert userspace address into unique logical address.
477 umtx_key_get(void *addr, int type, int share, struct umtx_key *key)
479 struct thread *td = curthread;
481 vm_map_entry_t entry;
487 if (share == THREAD_SHARE) {
489 key->info.private.vs = td->td_proc->p_vmspace;
490 key->info.private.addr = (uintptr_t)addr;
492 MPASS(share == PROCESS_SHARE || share == AUTO_SHARE);
493 map = &td->td_proc->p_vmspace->vm_map;
494 if (vm_map_lookup(&map, (vm_offset_t)addr, VM_PROT_WRITE,
495 &entry, &key->info.shared.object, &pindex, &prot,
496 &wired) != KERN_SUCCESS) {
500 if ((share == PROCESS_SHARE) ||
501 (share == AUTO_SHARE &&
502 VM_INHERIT_SHARE == entry->inheritance)) {
504 key->info.shared.offset = entry->offset + entry->start -
506 vm_object_reference(key->info.shared.object);
509 key->info.private.vs = td->td_proc->p_vmspace;
510 key->info.private.addr = (uintptr_t)addr;
512 vm_map_lookup_done(map, entry);
523 umtx_key_release(struct umtx_key *key)
526 vm_object_deallocate(key->info.shared.object);
530 * Lock a umtx object.
533 _do_lock_umtx(struct thread *td, struct umtx *umtx, u_long id, int timo)
543 * Care must be exercised when dealing with umtx structure. It
544 * can fault on any access.
548 * Try the uncontested case. This should be done in userland.
550 owner = casuword(&umtx->u_owner, UMTX_UNOWNED, id);
552 /* The acquire succeeded. */
553 if (owner == UMTX_UNOWNED)
556 /* The address was invalid. */
560 /* If no one owns it but it is contested try to acquire it. */
561 if (owner == UMTX_CONTESTED) {
562 owner = casuword(&umtx->u_owner,
563 UMTX_CONTESTED, id | UMTX_CONTESTED);
565 if (owner == UMTX_CONTESTED)
568 /* The address was invalid. */
572 /* If this failed the lock has changed, restart. */
577 * If we caught a signal, we have retried and now
583 if ((error = umtx_key_get(umtx, TYPE_SIMPLE_LOCK,
584 AUTO_SHARE, &uq->uq_key)) != 0)
587 umtxq_lock(&uq->uq_key);
588 umtxq_busy(&uq->uq_key);
590 umtxq_unbusy(&uq->uq_key);
591 umtxq_unlock(&uq->uq_key);
594 * Set the contested bit so that a release in user space
595 * knows to use the system call for unlock. If this fails
596 * either some one else has acquired the lock or it has been
599 old = casuword(&umtx->u_owner, owner, owner | UMTX_CONTESTED);
601 /* The address was invalid. */
603 umtxq_lock(&uq->uq_key);
605 umtxq_unlock(&uq->uq_key);
606 umtx_key_release(&uq->uq_key);
611 * We set the contested bit, sleep. Otherwise the lock changed
612 * and we need to retry or we lost a race to the thread
613 * unlocking the umtx.
615 umtxq_lock(&uq->uq_key);
617 error = umtxq_sleep(uq, "umtx", timo);
619 umtxq_unlock(&uq->uq_key);
620 umtx_key_release(&uq->uq_key);
627 * Lock a umtx object.
630 do_lock_umtx(struct thread *td, struct umtx *umtx, u_long id,
631 struct timespec *timeout)
633 struct timespec ts, ts2, ts3;
637 if (timeout == NULL) {
638 error = _do_lock_umtx(td, umtx, id, 0);
639 /* Mutex locking is restarted if it is interrupted. */
644 timespecadd(&ts, timeout);
645 TIMESPEC_TO_TIMEVAL(&tv, timeout);
647 error = _do_lock_umtx(td, umtx, id, tvtohz(&tv));
648 if (error != ETIMEDOUT)
651 if (timespeccmp(&ts2, &ts, >=)) {
656 timespecsub(&ts3, &ts2);
657 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
659 /* Timed-locking is not restarted. */
660 if (error == ERESTART)
667 * Unlock a umtx object.
670 do_unlock_umtx(struct thread *td, struct umtx *umtx, u_long id)
679 * Make sure we own this mtx.
681 owner = fuword(__DEVOLATILE(u_long *, &umtx->u_owner));
685 if ((owner & ~UMTX_CONTESTED) != id)
688 /* This should be done in userland */
689 if ((owner & UMTX_CONTESTED) == 0) {
690 old = casuword(&umtx->u_owner, owner, UMTX_UNOWNED);
698 /* We should only ever be in here for contested locks */
699 if ((error = umtx_key_get(umtx, TYPE_SIMPLE_LOCK, AUTO_SHARE,
705 count = umtxq_count(&key);
709 * When unlocking the umtx, it must be marked as unowned if
710 * there is zero or one thread only waiting for it.
711 * Otherwise, it must be marked as contested.
713 old = casuword(&umtx->u_owner, owner,
714 count <= 1 ? UMTX_UNOWNED : UMTX_CONTESTED);
716 umtxq_signal(&key,1);
719 umtx_key_release(&key);
730 * Lock a umtx object.
733 _do_lock_umtx32(struct thread *td, uint32_t *m, uint32_t id, int timo)
743 * Care must be exercised when dealing with umtx structure. It
744 * can fault on any access.
748 * Try the uncontested case. This should be done in userland.
750 owner = casuword32(m, UMUTEX_UNOWNED, id);
752 /* The acquire succeeded. */
753 if (owner == UMUTEX_UNOWNED)
756 /* The address was invalid. */
760 /* If no one owns it but it is contested try to acquire it. */
761 if (owner == UMUTEX_CONTESTED) {
762 owner = casuword32(m,
763 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
764 if (owner == UMUTEX_CONTESTED)
767 /* The address was invalid. */
771 /* If this failed the lock has changed, restart. */
776 * If we caught a signal, we have retried and now
782 if ((error = umtx_key_get(m, TYPE_SIMPLE_LOCK,
783 AUTO_SHARE, &uq->uq_key)) != 0)
786 umtxq_lock(&uq->uq_key);
787 umtxq_busy(&uq->uq_key);
789 umtxq_unbusy(&uq->uq_key);
790 umtxq_unlock(&uq->uq_key);
793 * Set the contested bit so that a release in user space
794 * knows to use the system call for unlock. If this fails
795 * either some one else has acquired the lock or it has been
798 old = casuword32(m, owner, owner | UMUTEX_CONTESTED);
800 /* The address was invalid. */
802 umtxq_lock(&uq->uq_key);
804 umtxq_unlock(&uq->uq_key);
805 umtx_key_release(&uq->uq_key);
810 * We set the contested bit, sleep. Otherwise the lock changed
811 * and we need to retry or we lost a race to the thread
812 * unlocking the umtx.
814 umtxq_lock(&uq->uq_key);
816 error = umtxq_sleep(uq, "umtx", timo);
818 umtxq_unlock(&uq->uq_key);
819 umtx_key_release(&uq->uq_key);
826 * Lock a umtx object.
829 do_lock_umtx32(struct thread *td, void *m, uint32_t id,
830 struct timespec *timeout)
832 struct timespec ts, ts2, ts3;
836 if (timeout == NULL) {
837 error = _do_lock_umtx32(td, m, id, 0);
838 /* Mutex locking is restarted if it is interrupted. */
843 timespecadd(&ts, timeout);
844 TIMESPEC_TO_TIMEVAL(&tv, timeout);
846 error = _do_lock_umtx32(td, m, id, tvtohz(&tv));
847 if (error != ETIMEDOUT)
850 if (timespeccmp(&ts2, &ts, >=)) {
855 timespecsub(&ts3, &ts2);
856 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
858 /* Timed-locking is not restarted. */
859 if (error == ERESTART)
866 * Unlock a umtx object.
869 do_unlock_umtx32(struct thread *td, uint32_t *m, uint32_t id)
878 * Make sure we own this mtx.
884 if ((owner & ~UMUTEX_CONTESTED) != id)
887 /* This should be done in userland */
888 if ((owner & UMUTEX_CONTESTED) == 0) {
889 old = casuword32(m, owner, UMUTEX_UNOWNED);
897 /* We should only ever be in here for contested locks */
898 if ((error = umtx_key_get(m, TYPE_SIMPLE_LOCK, AUTO_SHARE,
904 count = umtxq_count(&key);
908 * When unlocking the umtx, it must be marked as unowned if
909 * there is zero or one thread only waiting for it.
910 * Otherwise, it must be marked as contested.
912 old = casuword32(m, owner,
913 count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
915 umtxq_signal(&key,1);
918 umtx_key_release(&key);
928 * Fetch and compare value, sleep on the address if value is not changed.
931 do_wait(struct thread *td, void *addr, u_long id,
932 struct timespec *timeout, int compat32)
935 struct timespec ts, ts2, ts3;
941 if ((error = umtx_key_get(addr, TYPE_SIMPLE_WAIT, AUTO_SHARE,
945 umtxq_lock(&uq->uq_key);
947 umtxq_unlock(&uq->uq_key);
951 tmp = fuword32(addr);
953 umtxq_lock(&uq->uq_key);
955 umtxq_unlock(&uq->uq_key);
956 } else if (timeout == NULL) {
957 umtxq_lock(&uq->uq_key);
958 error = umtxq_sleep(uq, "uwait", 0);
960 umtxq_unlock(&uq->uq_key);
963 timespecadd(&ts, timeout);
964 TIMESPEC_TO_TIMEVAL(&tv, timeout);
965 umtxq_lock(&uq->uq_key);
967 error = umtxq_sleep(uq, "uwait", tvtohz(&tv));
968 if (!(uq->uq_flags & UQF_UMTXQ))
970 if (error != ETIMEDOUT)
972 umtxq_unlock(&uq->uq_key);
974 if (timespeccmp(&ts2, &ts, >=)) {
976 umtxq_lock(&uq->uq_key);
980 timespecsub(&ts3, &ts2);
981 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
982 umtxq_lock(&uq->uq_key);
985 umtxq_unlock(&uq->uq_key);
987 umtx_key_release(&uq->uq_key);
988 if (error == ERESTART)
994 * Wake up threads sleeping on the specified address.
997 kern_umtx_wake(struct thread *td, void *uaddr, int n_wake)
1002 if ((ret = umtx_key_get(uaddr, TYPE_SIMPLE_WAIT, AUTO_SHARE,
1006 ret = umtxq_signal(&key, n_wake);
1008 umtx_key_release(&key);
1013 * Lock PTHREAD_PRIO_NONE protocol POSIX mutex.
1016 _do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, int timo,
1020 uint32_t owner, old, id;
1027 * Care must be exercised when dealing with umtx structure. It
1028 * can fault on any access.
1032 * Try the uncontested case. This should be done in userland.
1034 owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id);
1036 /* The acquire succeeded. */
1037 if (owner == UMUTEX_UNOWNED)
1040 /* The address was invalid. */
1044 /* If no one owns it but it is contested try to acquire it. */
1045 if (owner == UMUTEX_CONTESTED) {
1046 owner = casuword32(&m->m_owner,
1047 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
1049 if (owner == UMUTEX_CONTESTED)
1052 /* The address was invalid. */
1056 /* If this failed the lock has changed, restart. */
1060 if ((flags & UMUTEX_ERROR_CHECK) != 0 &&
1061 (owner & ~UMUTEX_CONTESTED) == id)
1068 * If we caught a signal, we have retried and now
1074 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX,
1075 GET_SHARE(flags), &uq->uq_key)) != 0)
1078 umtxq_lock(&uq->uq_key);
1079 umtxq_busy(&uq->uq_key);
1081 umtxq_unbusy(&uq->uq_key);
1082 umtxq_unlock(&uq->uq_key);
1085 * Set the contested bit so that a release in user space
1086 * knows to use the system call for unlock. If this fails
1087 * either some one else has acquired the lock or it has been
1090 old = casuword32(&m->m_owner, owner, owner | UMUTEX_CONTESTED);
1092 /* The address was invalid. */
1094 umtxq_lock(&uq->uq_key);
1096 umtxq_unlock(&uq->uq_key);
1097 umtx_key_release(&uq->uq_key);
1102 * We set the contested bit, sleep. Otherwise the lock changed
1103 * and we need to retry or we lost a race to the thread
1104 * unlocking the umtx.
1106 umtxq_lock(&uq->uq_key);
1108 error = umtxq_sleep(uq, "umtxn", timo);
1110 umtxq_unlock(&uq->uq_key);
1111 umtx_key_release(&uq->uq_key);
1118 * Lock PTHREAD_PRIO_NONE protocol POSIX mutex.
1121 * Unlock PTHREAD_PRIO_NONE protocol POSIX mutex.
1124 do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags)
1126 struct umtx_key key;
1127 uint32_t owner, old, id;
1133 * Make sure we own this mtx.
1135 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
1139 if ((owner & ~UMUTEX_CONTESTED) != id)
1142 /* This should be done in userland */
1143 if ((owner & UMUTEX_CONTESTED) == 0) {
1144 old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED);
1152 /* We should only ever be in here for contested locks */
1153 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags),
1159 count = umtxq_count(&key);
1163 * When unlocking the umtx, it must be marked as unowned if
1164 * there is zero or one thread only waiting for it.
1165 * Otherwise, it must be marked as contested.
1167 old = casuword32(&m->m_owner, owner,
1168 count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
1170 umtxq_signal(&key,1);
1173 umtx_key_release(&key);
1181 static inline struct umtx_pi *
1182 umtx_pi_alloc(int flags)
1186 pi = uma_zalloc(umtx_pi_zone, M_ZERO | flags);
1187 TAILQ_INIT(&pi->pi_blocked);
1188 atomic_add_int(&umtx_pi_allocated, 1);
1193 umtx_pi_free(struct umtx_pi *pi)
1195 uma_zfree(umtx_pi_zone, pi);
1196 atomic_add_int(&umtx_pi_allocated, -1);
1200 * Adjust the thread's position on a pi_state after its priority has been
1204 umtx_pi_adjust_thread(struct umtx_pi *pi, struct thread *td)
1206 struct umtx_q *uq, *uq1, *uq2;
1209 mtx_assert(&umtx_lock, MA_OWNED);
1216 * Check if the thread needs to be moved on the blocked chain.
1217 * It needs to be moved if either its priority is lower than
1218 * the previous thread or higher than the next thread.
1220 uq1 = TAILQ_PREV(uq, umtxq_head, uq_lockq);
1221 uq2 = TAILQ_NEXT(uq, uq_lockq);
1222 if ((uq1 != NULL && UPRI(td) < UPRI(uq1->uq_thread)) ||
1223 (uq2 != NULL && UPRI(td) > UPRI(uq2->uq_thread))) {
1225 * Remove thread from blocked chain and determine where
1226 * it should be moved to.
1228 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
1229 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) {
1230 td1 = uq1->uq_thread;
1231 MPASS(td1->td_proc->p_magic == P_MAGIC);
1232 if (UPRI(td1) > UPRI(td))
1237 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq);
1239 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq);
1245 * Propagate priority when a thread is blocked on POSIX
1249 umtx_propagate_priority(struct thread *td)
1255 mtx_assert(&umtx_lock, MA_OWNED);
1258 pi = uq->uq_pi_blocked;
1267 MPASS(td->td_proc != NULL);
1268 MPASS(td->td_proc->p_magic == P_MAGIC);
1270 if (UPRI(td) <= pri)
1274 sched_lend_user_prio(td, pri);
1278 * Pick up the lock that td is blocked on.
1281 pi = uq->uq_pi_blocked;
1282 /* Resort td on the list if needed. */
1283 if (!umtx_pi_adjust_thread(pi, td))
1289 * Unpropagate priority for a PI mutex when a thread blocked on
1290 * it is interrupted by signal or resumed by others.
1293 umtx_unpropagate_priority(struct umtx_pi *pi)
1295 struct umtx_q *uq, *uq_owner;
1296 struct umtx_pi *pi2;
1299 mtx_assert(&umtx_lock, MA_OWNED);
1301 while (pi != NULL && pi->pi_owner != NULL) {
1303 uq_owner = pi->pi_owner->td_umtxq;
1305 TAILQ_FOREACH(pi2, &uq_owner->uq_pi_contested, pi_link) {
1306 uq = TAILQ_FIRST(&pi2->pi_blocked);
1308 if (pri > UPRI(uq->uq_thread))
1309 pri = UPRI(uq->uq_thread);
1313 if (pri > uq_owner->uq_inherited_pri)
1314 pri = uq_owner->uq_inherited_pri;
1315 thread_lock(pi->pi_owner);
1316 sched_unlend_user_prio(pi->pi_owner, pri);
1317 thread_unlock(pi->pi_owner);
1318 pi = uq_owner->uq_pi_blocked;
1323 * Insert a PI mutex into owned list.
1326 umtx_pi_setowner(struct umtx_pi *pi, struct thread *owner)
1328 struct umtx_q *uq_owner;
1330 uq_owner = owner->td_umtxq;
1331 mtx_assert(&umtx_lock, MA_OWNED);
1332 if (pi->pi_owner != NULL)
1333 panic("pi_ower != NULL");
1334 pi->pi_owner = owner;
1335 TAILQ_INSERT_TAIL(&uq_owner->uq_pi_contested, pi, pi_link);
1339 * Claim ownership of a PI mutex.
1342 umtx_pi_claim(struct umtx_pi *pi, struct thread *owner)
1344 struct umtx_q *uq, *uq_owner;
1346 uq_owner = owner->td_umtxq;
1347 mtx_lock_spin(&umtx_lock);
1348 if (pi->pi_owner == owner) {
1349 mtx_unlock_spin(&umtx_lock);
1353 if (pi->pi_owner != NULL) {
1355 * userland may have already messed the mutex, sigh.
1357 mtx_unlock_spin(&umtx_lock);
1360 umtx_pi_setowner(pi, owner);
1361 uq = TAILQ_FIRST(&pi->pi_blocked);
1365 pri = UPRI(uq->uq_thread);
1367 if (pri < UPRI(owner))
1368 sched_lend_user_prio(owner, pri);
1369 thread_unlock(owner);
1371 mtx_unlock_spin(&umtx_lock);
1376 * Adjust a thread's order position in its blocked PI mutex,
1377 * this may result new priority propagating process.
1380 umtx_pi_adjust(struct thread *td, u_char oldpri)
1387 mtx_assert(&umtx_lock, MA_OWNED);
1388 MPASS(TD_ON_UPILOCK(td));
1391 * Pick up the lock that td is blocked on.
1393 pi = uq->uq_pi_blocked;
1396 /* Resort the turnstile on the list. */
1397 if (!umtx_pi_adjust_thread(pi, td))
1401 * If our priority was lowered and we are at the head of the
1402 * turnstile, then propagate our new priority up the chain.
1404 if (uq == TAILQ_FIRST(&pi->pi_blocked) && UPRI(td) < oldpri)
1405 umtx_propagate_priority(td);
1409 * Sleep on a PI mutex.
1412 umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi,
1413 uint32_t owner, const char *wmesg, int timo)
1415 struct umtxq_chain *uc;
1416 struct thread *td, *td1;
1422 KASSERT(td == curthread, ("inconsistent uq_thread"));
1423 uc = umtxq_getchain(&uq->uq_key);
1424 UMTXQ_LOCKED_ASSERT(uc);
1426 if (pi->pi_owner == NULL) {
1428 * Current, We only support process private PI-mutex,
1429 * non-contended PI-mutexes are locked in userland.
1430 * Process shared PI-mutex should always be initialized
1431 * by kernel and be registered in kernel, locking should
1432 * always be done by kernel to avoid security problems.
1433 * For process private PI-mutex, we can find owner
1434 * thread and boost its priority safely.
1437 td1 = thread_find(curproc, owner);
1438 mtx_lock_spin(&umtx_lock);
1439 if (td1 != NULL && pi->pi_owner == NULL) {
1440 uq1 = td1->td_umtxq;
1441 umtx_pi_setowner(pi, td1);
1443 PROC_UNLOCK(curproc);
1445 mtx_lock_spin(&umtx_lock);
1448 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) {
1449 pri = UPRI(uq1->uq_thread);
1455 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq);
1457 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq);
1459 uq->uq_pi_blocked = pi;
1460 td->td_flags |= TDF_UPIBLOCKED;
1461 mtx_unlock_spin(&umtx_lock);
1462 umtxq_unlock(&uq->uq_key);
1464 mtx_lock_spin(&umtx_lock);
1465 umtx_propagate_priority(td);
1466 mtx_unlock_spin(&umtx_lock);
1468 umtxq_lock(&uq->uq_key);
1469 if (uq->uq_flags & UQF_UMTXQ) {
1470 error = msleep(uq, &uc->uc_lock, PCATCH, wmesg, timo);
1471 if (error == EWOULDBLOCK)
1473 if (uq->uq_flags & UQF_UMTXQ) {
1474 umtxq_busy(&uq->uq_key);
1476 umtxq_unbusy(&uq->uq_key);
1479 umtxq_unlock(&uq->uq_key);
1481 mtx_lock_spin(&umtx_lock);
1482 uq->uq_pi_blocked = NULL;
1483 td->td_flags &= ~TDF_UPIBLOCKED;
1484 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
1485 umtx_unpropagate_priority(pi);
1486 mtx_unlock_spin(&umtx_lock);
1488 umtxq_lock(&uq->uq_key);
1494 * Add reference count for a PI mutex.
1497 umtx_pi_ref(struct umtx_pi *pi)
1499 struct umtxq_chain *uc;
1501 uc = umtxq_getchain(&pi->pi_key);
1502 UMTXQ_LOCKED_ASSERT(uc);
1507 * Decrease reference count for a PI mutex, if the counter
1508 * is decreased to zero, its memory space is freed.
1511 umtx_pi_unref(struct umtx_pi *pi)
1513 struct umtxq_chain *uc;
1516 uc = umtxq_getchain(&pi->pi_key);
1517 UMTXQ_LOCKED_ASSERT(uc);
1518 KASSERT(pi->pi_refcount > 0, ("invalid reference count"));
1519 if (--pi->pi_refcount == 0) {
1520 mtx_lock_spin(&umtx_lock);
1521 if (pi->pi_owner != NULL) {
1522 TAILQ_REMOVE(&pi->pi_owner->td_umtxq->uq_pi_contested,
1524 pi->pi_owner = NULL;
1526 KASSERT(TAILQ_EMPTY(&pi->pi_blocked),
1527 ("blocked queue not empty"));
1528 mtx_unlock_spin(&umtx_lock);
1529 TAILQ_REMOVE(&uc->uc_pi_list, pi, pi_hashlink);
1537 * Find a PI mutex in hash table.
1539 static struct umtx_pi *
1540 umtx_pi_lookup(struct umtx_key *key)
1542 struct umtxq_chain *uc;
1545 uc = umtxq_getchain(key);
1546 UMTXQ_LOCKED_ASSERT(uc);
1548 TAILQ_FOREACH(pi, &uc->uc_pi_list, pi_hashlink) {
1549 if (umtx_key_match(&pi->pi_key, key)) {
1557 * Insert a PI mutex into hash table.
1560 umtx_pi_insert(struct umtx_pi *pi)
1562 struct umtxq_chain *uc;
1564 uc = umtxq_getchain(&pi->pi_key);
1565 UMTXQ_LOCKED_ASSERT(uc);
1566 TAILQ_INSERT_TAIL(&uc->uc_pi_list, pi, pi_hashlink);
1573 _do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, int timo,
1577 struct umtx_pi *pi, *new_pi;
1578 uint32_t id, owner, old;
1584 if ((error = umtx_key_get(m, TYPE_PI_UMUTEX, GET_SHARE(flags),
1587 umtxq_lock(&uq->uq_key);
1588 pi = umtx_pi_lookup(&uq->uq_key);
1590 new_pi = umtx_pi_alloc(M_NOWAIT);
1591 if (new_pi == NULL) {
1592 umtxq_unlock(&uq->uq_key);
1593 new_pi = umtx_pi_alloc(M_WAITOK);
1594 new_pi->pi_key = uq->uq_key;
1595 umtxq_lock(&uq->uq_key);
1596 pi = umtx_pi_lookup(&uq->uq_key);
1598 umtx_pi_free(new_pi);
1602 if (new_pi != NULL) {
1603 new_pi->pi_key = uq->uq_key;
1604 umtx_pi_insert(new_pi);
1609 umtxq_unlock(&uq->uq_key);
1612 * Care must be exercised when dealing with umtx structure. It
1613 * can fault on any access.
1617 * Try the uncontested case. This should be done in userland.
1619 owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id);
1621 /* The acquire succeeded. */
1622 if (owner == UMUTEX_UNOWNED) {
1627 /* The address was invalid. */
1633 /* If no one owns it but it is contested try to acquire it. */
1634 if (owner == UMUTEX_CONTESTED) {
1635 owner = casuword32(&m->m_owner,
1636 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
1638 if (owner == UMUTEX_CONTESTED) {
1639 umtxq_lock(&uq->uq_key);
1640 error = umtx_pi_claim(pi, td);
1641 umtxq_unlock(&uq->uq_key);
1645 /* The address was invalid. */
1651 /* If this failed the lock has changed, restart. */
1655 if ((flags & UMUTEX_ERROR_CHECK) != 0 &&
1656 (owner & ~UMUTEX_CONTESTED) == id) {
1667 * If we caught a signal, we have retried and now
1673 umtxq_lock(&uq->uq_key);
1674 umtxq_busy(&uq->uq_key);
1675 umtxq_unlock(&uq->uq_key);
1678 * Set the contested bit so that a release in user space
1679 * knows to use the system call for unlock. If this fails
1680 * either some one else has acquired the lock or it has been
1683 old = casuword32(&m->m_owner, owner, owner | UMUTEX_CONTESTED);
1685 /* The address was invalid. */
1687 umtxq_lock(&uq->uq_key);
1688 umtxq_unbusy(&uq->uq_key);
1689 umtxq_unlock(&uq->uq_key);
1694 umtxq_lock(&uq->uq_key);
1695 umtxq_unbusy(&uq->uq_key);
1697 * We set the contested bit, sleep. Otherwise the lock changed
1698 * and we need to retry or we lost a race to the thread
1699 * unlocking the umtx.
1702 error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED,
1704 umtxq_unlock(&uq->uq_key);
1707 umtxq_lock(&uq->uq_key);
1709 umtxq_unlock(&uq->uq_key);
1711 umtx_key_release(&uq->uq_key);
1716 * Unlock a PI mutex.
1719 do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags)
1721 struct umtx_key key;
1722 struct umtx_q *uq_first, *uq_first2, *uq_me;
1723 struct umtx_pi *pi, *pi2;
1724 uint32_t owner, old, id;
1731 * Make sure we own this mtx.
1733 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
1737 if ((owner & ~UMUTEX_CONTESTED) != id)
1740 /* This should be done in userland */
1741 if ((owner & UMUTEX_CONTESTED) == 0) {
1742 old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED);
1750 /* We should only ever be in here for contested locks */
1751 if ((error = umtx_key_get(m, TYPE_PI_UMUTEX, GET_SHARE(flags),
1757 count = umtxq_count_pi(&key, &uq_first);
1758 if (uq_first != NULL) {
1759 pi = uq_first->uq_pi_blocked;
1760 if (pi->pi_owner != curthread) {
1763 /* userland messed the mutex */
1766 uq_me = curthread->td_umtxq;
1767 mtx_lock_spin(&umtx_lock);
1768 pi->pi_owner = NULL;
1769 TAILQ_REMOVE(&uq_me->uq_pi_contested, pi, pi_link);
1770 uq_first = TAILQ_FIRST(&pi->pi_blocked);
1772 TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) {
1773 uq_first2 = TAILQ_FIRST(&pi2->pi_blocked);
1774 if (uq_first2 != NULL) {
1775 if (pri > UPRI(uq_first2->uq_thread))
1776 pri = UPRI(uq_first2->uq_thread);
1779 thread_lock(curthread);
1780 sched_unlend_user_prio(curthread, pri);
1781 thread_unlock(curthread);
1782 mtx_unlock_spin(&umtx_lock);
1787 * When unlocking the umtx, it must be marked as unowned if
1788 * there is zero or one thread only waiting for it.
1789 * Otherwise, it must be marked as contested.
1791 old = casuword32(&m->m_owner, owner,
1792 count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
1795 if (uq_first != NULL)
1796 umtxq_signal_thread(uq_first);
1799 umtx_key_release(&key);
1811 _do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, int timo,
1814 struct umtx_q *uq, *uq2;
1818 int error, pri, old_inherited_pri, su;
1822 if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags),
1825 su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0);
1827 old_inherited_pri = uq->uq_inherited_pri;
1828 umtxq_lock(&uq->uq_key);
1829 umtxq_busy(&uq->uq_key);
1830 umtxq_unlock(&uq->uq_key);
1832 ceiling = RTP_PRIO_MAX - fuword32(&m->m_ceilings[0]);
1833 if (ceiling > RTP_PRIO_MAX) {
1838 mtx_lock_spin(&umtx_lock);
1839 if (UPRI(td) < PRI_MIN_REALTIME + ceiling) {
1840 mtx_unlock_spin(&umtx_lock);
1844 if (su && PRI_MIN_REALTIME + ceiling < uq->uq_inherited_pri) {
1845 uq->uq_inherited_pri = PRI_MIN_REALTIME + ceiling;
1847 if (uq->uq_inherited_pri < UPRI(td))
1848 sched_lend_user_prio(td, uq->uq_inherited_pri);
1851 mtx_unlock_spin(&umtx_lock);
1853 owner = casuword32(&m->m_owner,
1854 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
1856 if (owner == UMUTEX_CONTESTED) {
1861 /* The address was invalid. */
1867 if ((flags & UMUTEX_ERROR_CHECK) != 0 &&
1868 (owner & ~UMUTEX_CONTESTED) == id) {
1879 * If we caught a signal, we have retried and now
1885 umtxq_lock(&uq->uq_key);
1887 umtxq_unbusy(&uq->uq_key);
1888 error = umtxq_sleep(uq, "umtxpp", timo);
1890 umtxq_unlock(&uq->uq_key);
1892 mtx_lock_spin(&umtx_lock);
1893 uq->uq_inherited_pri = old_inherited_pri;
1895 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
1896 uq2 = TAILQ_FIRST(&pi->pi_blocked);
1898 if (pri > UPRI(uq2->uq_thread))
1899 pri = UPRI(uq2->uq_thread);
1902 if (pri > uq->uq_inherited_pri)
1903 pri = uq->uq_inherited_pri;
1905 sched_unlend_user_prio(td, pri);
1907 mtx_unlock_spin(&umtx_lock);
1911 mtx_lock_spin(&umtx_lock);
1912 uq->uq_inherited_pri = old_inherited_pri;
1914 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
1915 uq2 = TAILQ_FIRST(&pi->pi_blocked);
1917 if (pri > UPRI(uq2->uq_thread))
1918 pri = UPRI(uq2->uq_thread);
1921 if (pri > uq->uq_inherited_pri)
1922 pri = uq->uq_inherited_pri;
1924 sched_unlend_user_prio(td, pri);
1926 mtx_unlock_spin(&umtx_lock);
1930 umtxq_lock(&uq->uq_key);
1931 umtxq_unbusy(&uq->uq_key);
1932 umtxq_unlock(&uq->uq_key);
1933 umtx_key_release(&uq->uq_key);
1938 * Unlock a PP mutex.
1941 do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags)
1943 struct umtx_key key;
1944 struct umtx_q *uq, *uq2;
1948 int error, pri, new_inherited_pri, su;
1952 su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0);
1955 * Make sure we own this mtx.
1957 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
1961 if ((owner & ~UMUTEX_CONTESTED) != id)
1964 error = copyin(&m->m_ceilings[1], &rceiling, sizeof(uint32_t));
1969 new_inherited_pri = PRI_MAX;
1971 rceiling = RTP_PRIO_MAX - rceiling;
1972 if (rceiling > RTP_PRIO_MAX)
1974 new_inherited_pri = PRI_MIN_REALTIME + rceiling;
1977 if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags),
1984 * For priority protected mutex, always set unlocked state
1985 * to UMUTEX_CONTESTED, so that userland always enters kernel
1986 * to lock the mutex, it is necessary because thread priority
1987 * has to be adjusted for such mutex.
1989 error = suword32(__DEVOLATILE(uint32_t *, &m->m_owner),
1994 umtxq_signal(&key, 1);
2001 mtx_lock_spin(&umtx_lock);
2003 uq->uq_inherited_pri = new_inherited_pri;
2005 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2006 uq2 = TAILQ_FIRST(&pi->pi_blocked);
2008 if (pri > UPRI(uq2->uq_thread))
2009 pri = UPRI(uq2->uq_thread);
2012 if (pri > uq->uq_inherited_pri)
2013 pri = uq->uq_inherited_pri;
2015 sched_unlend_user_prio(td, pri);
2017 mtx_unlock_spin(&umtx_lock);
2019 umtx_key_release(&key);
2024 do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling,
2025 uint32_t *old_ceiling)
2028 uint32_t save_ceiling;
2033 flags = fuword32(&m->m_flags);
2034 if ((flags & UMUTEX_PRIO_PROTECT) == 0)
2036 if (ceiling > RTP_PRIO_MAX)
2040 if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags),
2044 umtxq_lock(&uq->uq_key);
2045 umtxq_busy(&uq->uq_key);
2046 umtxq_unlock(&uq->uq_key);
2048 save_ceiling = fuword32(&m->m_ceilings[0]);
2050 owner = casuword32(&m->m_owner,
2051 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
2053 if (owner == UMUTEX_CONTESTED) {
2054 suword32(&m->m_ceilings[0], ceiling);
2055 suword32(__DEVOLATILE(uint32_t *, &m->m_owner),
2061 /* The address was invalid. */
2067 if ((owner & ~UMUTEX_CONTESTED) == id) {
2068 suword32(&m->m_ceilings[0], ceiling);
2074 * If we caught a signal, we have retried and now
2081 * We set the contested bit, sleep. Otherwise the lock changed
2082 * and we need to retry or we lost a race to the thread
2083 * unlocking the umtx.
2085 umtxq_lock(&uq->uq_key);
2087 umtxq_unbusy(&uq->uq_key);
2088 error = umtxq_sleep(uq, "umtxpp", 0);
2090 umtxq_unlock(&uq->uq_key);
2092 umtxq_lock(&uq->uq_key);
2094 umtxq_signal(&uq->uq_key, INT_MAX);
2095 umtxq_unbusy(&uq->uq_key);
2096 umtxq_unlock(&uq->uq_key);
2097 umtx_key_release(&uq->uq_key);
2098 if (error == 0 && old_ceiling != NULL)
2099 suword32(old_ceiling, save_ceiling);
2104 _do_lock_umutex(struct thread *td, struct umutex *m, int flags, int timo,
2107 switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) {
2109 return (_do_lock_normal(td, m, flags, timo, try));
2110 case UMUTEX_PRIO_INHERIT:
2111 return (_do_lock_pi(td, m, flags, timo, try));
2112 case UMUTEX_PRIO_PROTECT:
2113 return (_do_lock_pp(td, m, flags, timo, try));
2119 * Lock a userland POSIX mutex.
2122 do_lock_umutex(struct thread *td, struct umutex *m,
2123 struct timespec *timeout, int try)
2125 struct timespec ts, ts2, ts3;
2130 flags = fuword32(&m->m_flags);
2134 if (timeout == NULL) {
2135 error = _do_lock_umutex(td, m, flags, 0, try);
2136 /* Mutex locking is restarted if it is interrupted. */
2141 timespecadd(&ts, timeout);
2142 TIMESPEC_TO_TIMEVAL(&tv, timeout);
2144 error = _do_lock_umutex(td, m, flags, tvtohz(&tv), try);
2145 if (error != ETIMEDOUT)
2147 getnanouptime(&ts2);
2148 if (timespeccmp(&ts2, &ts, >=)) {
2153 timespecsub(&ts3, &ts2);
2154 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
2156 /* Timed-locking is not restarted. */
2157 if (error == ERESTART)
2164 * Unlock a userland POSIX mutex.
2167 do_unlock_umutex(struct thread *td, struct umutex *m)
2171 flags = fuword32(&m->m_flags);
2175 switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) {
2177 return (do_unlock_normal(td, m, flags));
2178 case UMUTEX_PRIO_INHERIT:
2179 return (do_unlock_pi(td, m, flags));
2180 case UMUTEX_PRIO_PROTECT:
2181 return (do_unlock_pp(td, m, flags));
2188 do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m,
2189 struct timespec *timeout, u_long wflags)
2193 struct timespec cts, ets, tts;
2198 flags = fuword32(&cv->c_flags);
2199 error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key);
2202 umtxq_lock(&uq->uq_key);
2203 umtxq_busy(&uq->uq_key);
2205 umtxq_unlock(&uq->uq_key);
2208 * The magic thing is we should set c_has_waiters to 1 before
2209 * releasing user mutex.
2211 suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 1);
2213 umtxq_lock(&uq->uq_key);
2214 umtxq_unbusy(&uq->uq_key);
2215 umtxq_unlock(&uq->uq_key);
2217 error = do_unlock_umutex(td, m);
2219 umtxq_lock(&uq->uq_key);
2221 if ((wflags & UMTX_CHECK_UNPARKING) &&
2222 (td->td_pflags & TDP_WAKEUP)) {
2223 td->td_pflags &= ~TDP_WAKEUP;
2225 } else if (timeout == NULL) {
2226 error = umtxq_sleep(uq, "ucond", 0);
2228 getnanouptime(&ets);
2229 timespecadd(&ets, timeout);
2230 TIMESPEC_TO_TIMEVAL(&tv, timeout);
2232 error = umtxq_sleep(uq, "ucond", tvtohz(&tv));
2233 if (error != ETIMEDOUT)
2235 getnanouptime(&cts);
2236 if (timespeccmp(&cts, &ets, >=)) {
2241 timespecsub(&tts, &cts);
2242 TIMESPEC_TO_TIMEVAL(&tv, &tts);
2248 if ((uq->uq_flags & UQF_UMTXQ) == 0) {
2250 * If we concurrently got do_cv_signal()d
2251 * and we got an error or UNIX signals or a timeout,
2252 * then, perform another umtxq_signal to avoid
2253 * consuming the wakeup. This may cause supurious
2254 * wakeup for another thread which was just queued,
2255 * but SUSV3 explicitly allows supurious wakeup to
2256 * occur, and indeed a kernel based implementation
2259 if (!umtxq_signal(&uq->uq_key, 1))
2262 if (error == ERESTART)
2266 umtxq_unlock(&uq->uq_key);
2267 umtx_key_release(&uq->uq_key);
2272 * Signal a userland condition variable.
2275 do_cv_signal(struct thread *td, struct ucond *cv)
2277 struct umtx_key key;
2278 int error, cnt, nwake;
2281 flags = fuword32(&cv->c_flags);
2282 if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0)
2286 cnt = umtxq_count(&key);
2287 nwake = umtxq_signal(&key, 1);
2291 __DEVOLATILE(uint32_t *, &cv->c_has_waiters), 0);
2296 umtx_key_release(&key);
2301 do_cv_broadcast(struct thread *td, struct ucond *cv)
2303 struct umtx_key key;
2307 flags = fuword32(&cv->c_flags);
2308 if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0)
2313 umtxq_signal(&key, INT_MAX);
2316 error = suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 0);
2322 umtx_key_release(&key);
2327 _umtx_lock(struct thread *td, struct _umtx_lock_args *uap)
2328 /* struct umtx *umtx */
2330 return _do_lock_umtx(td, uap->umtx, td->td_tid, 0);
2334 _umtx_unlock(struct thread *td, struct _umtx_unlock_args *uap)
2335 /* struct umtx *umtx */
2337 return do_unlock_umtx(td, uap->umtx, td->td_tid);
2341 __umtx_op_lock_umtx(struct thread *td, struct _umtx_op_args *uap)
2343 struct timespec *ts, timeout;
2346 /* Allow a null timespec (wait forever). */
2347 if (uap->uaddr2 == NULL)
2350 error = copyin(uap->uaddr2, &timeout, sizeof(timeout));
2353 if (timeout.tv_nsec >= 1000000000 ||
2354 timeout.tv_nsec < 0) {
2359 return (do_lock_umtx(td, uap->obj, uap->val, ts));
2363 __umtx_op_unlock_umtx(struct thread *td, struct _umtx_op_args *uap)
2365 return (do_unlock_umtx(td, uap->obj, uap->val));
2369 __umtx_op_wait(struct thread *td, struct _umtx_op_args *uap)
2371 struct timespec *ts, timeout;
2374 if (uap->uaddr2 == NULL)
2377 error = copyin(uap->uaddr2, &timeout, sizeof(timeout));
2380 if (timeout.tv_nsec >= 1000000000 ||
2381 timeout.tv_nsec < 0)
2385 return do_wait(td, uap->obj, uap->val, ts, 0);
2389 __umtx_op_wake(struct thread *td, struct _umtx_op_args *uap)
2391 return (kern_umtx_wake(td, uap->obj, uap->val));
2395 __umtx_op_lock_umutex(struct thread *td, struct _umtx_op_args *uap)
2397 struct timespec *ts, timeout;
2400 /* Allow a null timespec (wait forever). */
2401 if (uap->uaddr2 == NULL)
2404 error = copyin(uap->uaddr2, &timeout,
2408 if (timeout.tv_nsec >= 1000000000 ||
2409 timeout.tv_nsec < 0) {
2414 return do_lock_umutex(td, uap->obj, ts, 0);
2418 __umtx_op_trylock_umutex(struct thread *td, struct _umtx_op_args *uap)
2420 return do_lock_umutex(td, uap->obj, NULL, 1);
2424 __umtx_op_unlock_umutex(struct thread *td, struct _umtx_op_args *uap)
2426 return do_unlock_umutex(td, uap->obj);
2430 __umtx_op_set_ceiling(struct thread *td, struct _umtx_op_args *uap)
2432 return do_set_ceiling(td, uap->obj, uap->val, uap->uaddr1);
2436 __umtx_op_cv_wait(struct thread *td, struct _umtx_op_args *uap)
2438 struct timespec *ts, timeout;
2441 /* Allow a null timespec (wait forever). */
2442 if (uap->uaddr2 == NULL)
2445 error = copyin(uap->uaddr2, &timeout,
2449 if (timeout.tv_nsec >= 1000000000 ||
2450 timeout.tv_nsec < 0) {
2455 return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val));
2459 __umtx_op_cv_signal(struct thread *td, struct _umtx_op_args *uap)
2461 return do_cv_signal(td, uap->obj);
2465 __umtx_op_cv_broadcast(struct thread *td, struct _umtx_op_args *uap)
2467 return do_cv_broadcast(td, uap->obj);
2470 typedef int (*_umtx_op_func)(struct thread *td, struct _umtx_op_args *uap);
2472 static _umtx_op_func op_table[] = {
2473 __umtx_op_lock_umtx, /* UMTX_OP_LOCK */
2474 __umtx_op_unlock_umtx, /* UMTX_OP_UNLOCK */
2475 __umtx_op_wait, /* UMTX_OP_WAIT */
2476 __umtx_op_wake, /* UMTX_OP_WAKE */
2477 __umtx_op_trylock_umutex, /* UMTX_OP_MUTEX_TRYLOCK */
2478 __umtx_op_lock_umutex, /* UMTX_OP_MUTEX_LOCK */
2479 __umtx_op_unlock_umutex, /* UMTX_OP_MUTEX_UNLOCK */
2480 __umtx_op_set_ceiling, /* UMTX_OP_SET_CEILING */
2481 __umtx_op_cv_wait, /* UMTX_OP_CV_WAIT*/
2482 __umtx_op_cv_signal, /* UMTX_OP_CV_SIGNAL */
2483 __umtx_op_cv_broadcast /* UMTX_OP_CV_BROADCAST */
2487 _umtx_op(struct thread *td, struct _umtx_op_args *uap)
2489 if ((unsigned)uap->op < UMTX_OP_MAX)
2490 return (*op_table[uap->op])(td, uap);
2496 freebsd32_umtx_lock(struct thread *td, struct freebsd32_umtx_lock_args *uap)
2497 /* struct umtx *umtx */
2499 return (do_lock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid, NULL));
2503 freebsd32_umtx_unlock(struct thread *td, struct freebsd32_umtx_unlock_args *uap)
2504 /* struct umtx *umtx */
2506 return (do_unlock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid));
2515 copyin_timeout32(void *addr, struct timespec *tsp)
2517 struct timespec32 ts32;
2520 error = copyin(addr, &ts32, sizeof(struct timespec32));
2522 tsp->tv_sec = ts32.tv_sec;
2523 tsp->tv_nsec = ts32.tv_nsec;
2529 __umtx_op_lock_umtx_compat32(struct thread *td, struct _umtx_op_args *uap)
2531 struct timespec *ts, timeout;
2534 /* Allow a null timespec (wait forever). */
2535 if (uap->uaddr2 == NULL)
2538 error = copyin_timeout32(uap->uaddr2, &timeout);
2541 if (timeout.tv_nsec >= 1000000000 ||
2542 timeout.tv_nsec < 0) {
2547 return (do_lock_umtx32(td, uap->obj, uap->val, ts));
2551 __umtx_op_unlock_umtx_compat32(struct thread *td, struct _umtx_op_args *uap)
2553 return (do_unlock_umtx32(td, uap->obj, (uint32_t)uap->val));
2557 __umtx_op_wait_compat32(struct thread *td, struct _umtx_op_args *uap)
2559 struct timespec *ts, timeout;
2562 if (uap->uaddr2 == NULL)
2565 error = copyin_timeout32(uap->uaddr2, &timeout);
2568 if (timeout.tv_nsec >= 1000000000 ||
2569 timeout.tv_nsec < 0)
2573 return do_wait(td, uap->obj, uap->val, ts, 1);
2577 __umtx_op_lock_umutex_compat32(struct thread *td, struct _umtx_op_args *uap)
2579 struct timespec *ts, timeout;
2582 /* Allow a null timespec (wait forever). */
2583 if (uap->uaddr2 == NULL)
2586 error = copyin_timeout32(uap->uaddr2, &timeout);
2589 if (timeout.tv_nsec >= 1000000000 ||
2590 timeout.tv_nsec < 0)
2594 return do_lock_umutex(td, uap->obj, ts, 0);
2598 __umtx_op_cv_wait_compat32(struct thread *td, struct _umtx_op_args *uap)
2600 struct timespec *ts, timeout;
2603 /* Allow a null timespec (wait forever). */
2604 if (uap->uaddr2 == NULL)
2607 error = copyin_timeout32(uap->uaddr2, &timeout);
2610 if (timeout.tv_nsec >= 1000000000 ||
2611 timeout.tv_nsec < 0)
2615 return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val));
2618 static _umtx_op_func op_table_compat32[] = {
2619 __umtx_op_lock_umtx_compat32, /* UMTX_OP_LOCK */
2620 __umtx_op_unlock_umtx_compat32, /* UMTX_OP_UNLOCK */
2621 __umtx_op_wait_compat32, /* UMTX_OP_WAIT */
2622 __umtx_op_wake, /* UMTX_OP_WAKE */
2623 __umtx_op_trylock_umutex, /* UMTX_OP_MUTEX_LOCK */
2624 __umtx_op_lock_umutex_compat32, /* UMTX_OP_MUTEX_TRYLOCK */
2625 __umtx_op_unlock_umutex, /* UMTX_OP_MUTEX_UNLOCK */
2626 __umtx_op_set_ceiling, /* UMTX_OP_SET_CEILING */
2627 __umtx_op_cv_wait_compat32, /* UMTX_OP_CV_WAIT*/
2628 __umtx_op_cv_signal, /* UMTX_OP_CV_SIGNAL */
2629 __umtx_op_cv_broadcast /* UMTX_OP_CV_BROADCAST */
2633 freebsd32_umtx_op(struct thread *td, struct freebsd32_umtx_op_args *uap)
2635 if ((unsigned)uap->op < UMTX_OP_MAX)
2636 return (*op_table_compat32[uap->op])(td,
2637 (struct _umtx_op_args *)uap);
2643 umtx_thread_init(struct thread *td)
2645 td->td_umtxq = umtxq_alloc();
2646 td->td_umtxq->uq_thread = td;
2650 umtx_thread_fini(struct thread *td)
2652 umtxq_free(td->td_umtxq);
2656 * It will be called when new thread is created, e.g fork().
2659 umtx_thread_alloc(struct thread *td)
2664 uq->uq_inherited_pri = PRI_MAX;
2666 KASSERT(uq->uq_flags == 0, ("uq_flags != 0"));
2667 KASSERT(uq->uq_thread == td, ("uq_thread != td"));
2668 KASSERT(uq->uq_pi_blocked == NULL, ("uq_pi_blocked != NULL"));
2669 KASSERT(TAILQ_EMPTY(&uq->uq_pi_contested), ("uq_pi_contested is not empty"));
2676 umtx_exec_hook(void *arg __unused, struct proc *p __unused,
2677 struct image_params *imgp __unused)
2679 umtx_thread_cleanup(curthread);
2683 * thread_exit() hook.
2686 umtx_thread_exit(struct thread *td)
2688 umtx_thread_cleanup(td);
2692 * clean up umtx data.
2695 umtx_thread_cleanup(struct thread *td)
2700 if ((uq = td->td_umtxq) == NULL)
2703 mtx_lock_spin(&umtx_lock);
2704 uq->uq_inherited_pri = PRI_MAX;
2705 while ((pi = TAILQ_FIRST(&uq->uq_pi_contested)) != NULL) {
2706 pi->pi_owner = NULL;
2707 TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link);
2709 td->td_flags &= ~TDF_UBORROWING;
2710 mtx_unlock_spin(&umtx_lock);