2 * Copyright (c) 1995 John Birrell <jb@cimlogic.com.au>.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. All advertising materials mentioning features or use of this software
14 * must display the following acknowledgement:
15 * This product includes software developed by John Birrell.
16 * 4. Neither the name of the author nor the names of any co-contributors
17 * may be used to endorse or promote products derived from this software
18 * without specific prior written permission.
20 * THIS SOFTWARE IS PROVIDED BY JOHN BIRRELL AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
37 #include <sys/param.h>
38 #include <sys/queue.h>
41 #include "pthread_private.h"
43 #if defined(_PTHREADS_INVARIANTS)
44 #define _MUTEX_INIT_LINK(m) do { \
45 (m)->m_qe.tqe_prev = NULL; \
46 (m)->m_qe.tqe_next = NULL; \
48 #define _MUTEX_ASSERT_IS_OWNED(m) do { \
49 if ((m)->m_qe.tqe_prev == NULL) \
50 PANIC("mutex is not on list"); \
52 #define _MUTEX_ASSERT_NOT_OWNED(m) do { \
53 if (((m)->m_qe.tqe_prev != NULL) || \
54 ((m)->m_qe.tqe_next != NULL)) \
55 PANIC("mutex is on list"); \
58 #define _MUTEX_INIT_LINK(m)
59 #define _MUTEX_ASSERT_IS_OWNED(m)
60 #define _MUTEX_ASSERT_NOT_OWNED(m)
66 static inline int mutex_self_trylock(pthread_mutex_t);
67 static inline int mutex_self_lock(pthread_mutex_t);
68 static inline int mutex_unlock_common(pthread_mutex_t *, int);
69 static void mutex_priority_adjust(pthread_mutex_t);
70 static void mutex_rescan_owned (pthread_t, pthread_mutex_t);
71 static inline pthread_t mutex_queue_deq(pthread_mutex_t);
72 static inline void mutex_queue_remove(pthread_mutex_t, pthread_t);
73 static inline void mutex_queue_enq(pthread_mutex_t, pthread_t);
76 static spinlock_t static_init_lock = _SPINLOCK_INITIALIZER;
78 /* Reinitialize a mutex to defaults. */
80 _mutex_reinit(pthread_mutex_t * mutex)
86 else if (*mutex == NULL)
87 ret = pthread_mutex_init(mutex, NULL);
90 * Initialize the mutex structure:
92 (*mutex)->m_type = PTHREAD_MUTEX_DEFAULT;
93 (*mutex)->m_protocol = PTHREAD_PRIO_NONE;
94 TAILQ_INIT(&(*mutex)->m_queue);
95 (*mutex)->m_owner = NULL;
96 (*mutex)->m_data.m_count = 0;
97 (*mutex)->m_flags &= MUTEX_FLAGS_PRIVATE;
98 (*mutex)->m_flags |= MUTEX_FLAGS_INITED;
99 (*mutex)->m_refcount = 0;
100 (*mutex)->m_prio = 0;
101 (*mutex)->m_saved_prio = 0;
102 _MUTEX_INIT_LINK(*mutex);
103 memset(&(*mutex)->lock, 0, sizeof((*mutex)->lock));
109 pthread_mutex_init(pthread_mutex_t * mutex,
110 const pthread_mutexattr_t * mutex_attr)
112 enum pthread_mutextype type;
115 pthread_mutex_t pmutex;
121 /* Check if default mutex attributes: */
122 else if (mutex_attr == NULL || *mutex_attr == NULL) {
123 /* Default to a (error checking) POSIX mutex: */
124 type = PTHREAD_MUTEX_ERRORCHECK;
125 protocol = PTHREAD_PRIO_NONE;
126 ceiling = PTHREAD_MAX_PRIORITY;
129 /* Check mutex type: */
130 else if (((*mutex_attr)->m_type < PTHREAD_MUTEX_ERRORCHECK) ||
131 ((*mutex_attr)->m_type >= MUTEX_TYPE_MAX))
132 /* Return an invalid argument error: */
135 /* Check mutex protocol: */
136 else if (((*mutex_attr)->m_protocol < PTHREAD_PRIO_NONE) ||
137 ((*mutex_attr)->m_protocol > PTHREAD_MUTEX_RECURSIVE))
138 /* Return an invalid argument error: */
142 /* Use the requested mutex type and protocol: */
143 type = (*mutex_attr)->m_type;
144 protocol = (*mutex_attr)->m_protocol;
145 ceiling = (*mutex_attr)->m_ceiling;
148 /* Check no errors so far: */
150 if ((pmutex = (pthread_mutex_t)
151 malloc(sizeof(struct pthread_mutex))) == NULL)
154 /* Reset the mutex flags: */
157 /* Process according to mutex type: */
159 /* case PTHREAD_MUTEX_DEFAULT: */
160 case PTHREAD_MUTEX_ERRORCHECK:
161 case PTHREAD_MUTEX_NORMAL:
162 /* Nothing to do here. */
165 /* Single UNIX Spec 2 recursive mutex: */
166 case PTHREAD_MUTEX_RECURSIVE:
167 /* Reset the mutex count: */
168 pmutex->m_data.m_count = 0;
171 /* Trap invalid mutex types: */
173 /* Return an invalid argument error: */
178 /* Initialise the rest of the mutex: */
179 TAILQ_INIT(&pmutex->m_queue);
180 pmutex->m_flags |= MUTEX_FLAGS_INITED;
181 pmutex->m_owner = NULL;
182 pmutex->m_type = type;
183 pmutex->m_protocol = protocol;
184 pmutex->m_refcount = 0;
185 if (protocol == PTHREAD_PRIO_PROTECT)
186 pmutex->m_prio = ceiling;
189 pmutex->m_saved_prio = 0;
190 _MUTEX_INIT_LINK(pmutex);
191 memset(&pmutex->lock, 0, sizeof(pmutex->lock));
199 /* Return the completion status: */
204 pthread_mutex_destroy(pthread_mutex_t * mutex)
208 if (mutex == NULL || *mutex == NULL)
211 /* Lock the mutex structure: */
212 _SPINLOCK(&(*mutex)->lock);
215 * Check to see if this mutex is in use:
217 if (((*mutex)->m_owner != NULL) ||
218 (TAILQ_FIRST(&(*mutex)->m_queue) != NULL) ||
219 ((*mutex)->m_refcount != 0)) {
222 /* Unlock the mutex structure: */
223 _SPINUNLOCK(&(*mutex)->lock);
227 * Free the memory allocated for the mutex
230 _MUTEX_ASSERT_NOT_OWNED(*mutex);
234 * Leave the caller's pointer NULL now that
235 * the mutex has been destroyed:
241 /* Return the completion status: */
246 init_static(pthread_mutex_t *mutex)
250 _SPINLOCK(&static_init_lock);
253 ret = pthread_mutex_init(mutex, NULL);
257 _SPINUNLOCK(&static_init_lock);
263 pthread_mutex_trylock(pthread_mutex_t * mutex)
271 * If the mutex is statically initialized, perform the dynamic
274 else if (*mutex != NULL || (ret = init_static(mutex)) == 0) {
276 * Defer signals to protect the scheduling queues from
277 * access by the signal handler:
279 _thread_kern_sig_defer();
281 /* Lock the mutex structure: */
282 _SPINLOCK(&(*mutex)->lock);
285 * If the mutex was statically allocated, properly
286 * initialize the tail queue.
288 if (((*mutex)->m_flags & MUTEX_FLAGS_INITED) == 0) {
289 TAILQ_INIT(&(*mutex)->m_queue);
290 _MUTEX_INIT_LINK(*mutex);
291 (*mutex)->m_flags |= MUTEX_FLAGS_INITED;
294 /* Process according to mutex type: */
295 switch ((*mutex)->m_protocol) {
296 /* Default POSIX mutex: */
297 case PTHREAD_PRIO_NONE:
298 /* Check if this mutex is not locked: */
299 if ((*mutex)->m_owner == NULL) {
300 /* Lock the mutex for the running thread: */
301 (*mutex)->m_owner = _thread_run;
303 /* Add to the list of owned mutexes: */
304 _MUTEX_ASSERT_NOT_OWNED(*mutex);
305 TAILQ_INSERT_TAIL(&_thread_run->mutexq,
307 } else if ((*mutex)->m_owner == _thread_run)
308 ret = mutex_self_trylock(*mutex);
310 /* Return a busy error: */
314 /* POSIX priority inheritence mutex: */
315 case PTHREAD_PRIO_INHERIT:
316 /* Check if this mutex is not locked: */
317 if ((*mutex)->m_owner == NULL) {
318 /* Lock the mutex for the running thread: */
319 (*mutex)->m_owner = _thread_run;
321 /* Track number of priority mutexes owned: */
322 _thread_run->priority_mutex_count++;
325 * The mutex takes on the attributes of the
326 * running thread when there are no waiters.
328 (*mutex)->m_prio = _thread_run->active_priority;
329 (*mutex)->m_saved_prio =
330 _thread_run->inherited_priority;
332 /* Add to the list of owned mutexes: */
333 _MUTEX_ASSERT_NOT_OWNED(*mutex);
334 TAILQ_INSERT_TAIL(&_thread_run->mutexq,
336 } else if ((*mutex)->m_owner == _thread_run)
337 ret = mutex_self_trylock(*mutex);
339 /* Return a busy error: */
343 /* POSIX priority protection mutex: */
344 case PTHREAD_PRIO_PROTECT:
345 /* Check for a priority ceiling violation: */
346 if (_thread_run->active_priority > (*mutex)->m_prio)
349 /* Check if this mutex is not locked: */
350 else if ((*mutex)->m_owner == NULL) {
351 /* Lock the mutex for the running thread: */
352 (*mutex)->m_owner = _thread_run;
354 /* Track number of priority mutexes owned: */
355 _thread_run->priority_mutex_count++;
358 * The running thread inherits the ceiling
359 * priority of the mutex and executes at that
362 _thread_run->active_priority = (*mutex)->m_prio;
363 (*mutex)->m_saved_prio =
364 _thread_run->inherited_priority;
365 _thread_run->inherited_priority =
368 /* Add to the list of owned mutexes: */
369 _MUTEX_ASSERT_NOT_OWNED(*mutex);
370 TAILQ_INSERT_TAIL(&_thread_run->mutexq,
372 } else if ((*mutex)->m_owner == _thread_run)
373 ret = mutex_self_trylock(*mutex);
375 /* Return a busy error: */
379 /* Trap invalid mutex types: */
381 /* Return an invalid argument error: */
386 /* Unlock the mutex structure: */
387 _SPINUNLOCK(&(*mutex)->lock);
390 * Undefer and handle pending signals, yielding if
393 _thread_kern_sig_undefer();
396 /* Return the completion status: */
401 pthread_mutex_lock(pthread_mutex_t * mutex)
405 if (_thread_initial == NULL)
412 * If the mutex is statically initialized, perform the dynamic
415 else if (*mutex != NULL || (ret = init_static(mutex)) == 0) {
417 * Defer signals to protect the scheduling queues from
418 * access by the signal handler:
420 _thread_kern_sig_defer();
422 /* Lock the mutex structure: */
423 _SPINLOCK(&(*mutex)->lock);
426 * If the mutex was statically allocated, properly
427 * initialize the tail queue.
429 if (((*mutex)->m_flags & MUTEX_FLAGS_INITED) == 0) {
430 TAILQ_INIT(&(*mutex)->m_queue);
431 (*mutex)->m_flags |= MUTEX_FLAGS_INITED;
432 _MUTEX_INIT_LINK(*mutex);
435 /* Reset the interrupted flag: */
436 _thread_run->interrupted = 0;
438 /* Process according to mutex type: */
439 switch ((*mutex)->m_protocol) {
440 /* Default POSIX mutex: */
441 case PTHREAD_PRIO_NONE:
442 if ((*mutex)->m_owner == NULL) {
443 /* Lock the mutex for this thread: */
444 (*mutex)->m_owner = _thread_run;
446 /* Add to the list of owned mutexes: */
447 _MUTEX_ASSERT_NOT_OWNED(*mutex);
448 TAILQ_INSERT_TAIL(&_thread_run->mutexq,
451 } else if ((*mutex)->m_owner == _thread_run)
452 ret = mutex_self_lock(*mutex);
455 * Join the queue of threads waiting to lock
458 mutex_queue_enq(*mutex, _thread_run);
461 * Keep a pointer to the mutex this thread
464 _thread_run->data.mutex = *mutex;
467 * Unlock the mutex structure and schedule the
470 _thread_kern_sched_state_unlock(PS_MUTEX_WAIT,
471 &(*mutex)->lock, __FILE__, __LINE__);
473 /* Lock the mutex structure again: */
474 _SPINLOCK(&(*mutex)->lock);
478 /* POSIX priority inheritence mutex: */
479 case PTHREAD_PRIO_INHERIT:
480 /* Check if this mutex is not locked: */
481 if ((*mutex)->m_owner == NULL) {
482 /* Lock the mutex for this thread: */
483 (*mutex)->m_owner = _thread_run;
485 /* Track number of priority mutexes owned: */
486 _thread_run->priority_mutex_count++;
489 * The mutex takes on attributes of the
490 * running thread when there are no waiters.
492 (*mutex)->m_prio = _thread_run->active_priority;
493 (*mutex)->m_saved_prio =
494 _thread_run->inherited_priority;
495 _thread_run->inherited_priority =
498 /* Add to the list of owned mutexes: */
499 _MUTEX_ASSERT_NOT_OWNED(*mutex);
500 TAILQ_INSERT_TAIL(&_thread_run->mutexq,
503 } else if ((*mutex)->m_owner == _thread_run)
504 ret = mutex_self_lock(*mutex);
507 * Join the queue of threads waiting to lock
510 mutex_queue_enq(*mutex, _thread_run);
513 * Keep a pointer to the mutex this thread
516 _thread_run->data.mutex = *mutex;
518 if (_thread_run->active_priority >
520 /* Adjust priorities: */
521 mutex_priority_adjust(*mutex);
524 * Unlock the mutex structure and schedule the
527 _thread_kern_sched_state_unlock(PS_MUTEX_WAIT,
528 &(*mutex)->lock, __FILE__, __LINE__);
530 /* Lock the mutex structure again: */
531 _SPINLOCK(&(*mutex)->lock);
535 /* POSIX priority protection mutex: */
536 case PTHREAD_PRIO_PROTECT:
537 /* Check for a priority ceiling violation: */
538 if (_thread_run->active_priority > (*mutex)->m_prio)
541 /* Check if this mutex is not locked: */
542 else if ((*mutex)->m_owner == NULL) {
544 * Lock the mutex for the running
547 (*mutex)->m_owner = _thread_run;
549 /* Track number of priority mutexes owned: */
550 _thread_run->priority_mutex_count++;
553 * The running thread inherits the ceiling
554 * priority of the mutex and executes at that
557 _thread_run->active_priority = (*mutex)->m_prio;
558 (*mutex)->m_saved_prio =
559 _thread_run->inherited_priority;
560 _thread_run->inherited_priority =
563 /* Add to the list of owned mutexes: */
564 _MUTEX_ASSERT_NOT_OWNED(*mutex);
565 TAILQ_INSERT_TAIL(&_thread_run->mutexq,
567 } else if ((*mutex)->m_owner == _thread_run)
568 ret = mutex_self_lock(*mutex);
571 * Join the queue of threads waiting to lock
574 mutex_queue_enq(*mutex, _thread_run);
577 * Keep a pointer to the mutex this thread
580 _thread_run->data.mutex = *mutex;
582 /* Clear any previous error: */
583 _thread_run->error = 0;
586 * Unlock the mutex structure and schedule the
589 _thread_kern_sched_state_unlock(PS_MUTEX_WAIT,
590 &(*mutex)->lock, __FILE__, __LINE__);
592 /* Lock the mutex structure again: */
593 _SPINLOCK(&(*mutex)->lock);
596 * The threads priority may have changed while
597 * waiting for the mutex causing a ceiling
600 ret = _thread_run->error;
601 _thread_run->error = 0;
605 /* Trap invalid mutex types: */
607 /* Return an invalid argument error: */
613 * Check to see if this thread was interrupted and
614 * is still in the mutex queue of waiting threads:
616 if (_thread_run->interrupted != 0)
617 mutex_queue_remove(*mutex, _thread_run);
619 /* Unlock the mutex structure: */
620 _SPINUNLOCK(&(*mutex)->lock);
623 * Undefer and handle pending signals, yielding if
626 _thread_kern_sig_undefer();
628 if (_thread_run->interrupted != 0 &&
629 _thread_run->continuation != NULL)
630 _thread_run->continuation((void *) _thread_run);
633 /* Return the completion status: */
638 pthread_mutex_unlock(pthread_mutex_t * mutex)
640 return (mutex_unlock_common(mutex, /* add reference */ 0));
644 _mutex_cv_unlock(pthread_mutex_t * mutex)
646 return (mutex_unlock_common(mutex, /* add reference */ 1));
650 _mutex_cv_lock(pthread_mutex_t * mutex)
653 if ((ret = pthread_mutex_lock(mutex)) == 0)
654 (*mutex)->m_refcount--;
659 mutex_self_trylock(pthread_mutex_t mutex)
663 switch (mutex->m_type) {
665 /* case PTHREAD_MUTEX_DEFAULT: */
666 case PTHREAD_MUTEX_ERRORCHECK:
667 case PTHREAD_MUTEX_NORMAL:
669 * POSIX specifies that mutexes should return EDEADLK if a
670 * recursive lock is detected.
675 case PTHREAD_MUTEX_RECURSIVE:
676 /* Increment the lock count: */
677 mutex->m_data.m_count++;
681 /* Trap invalid mutex types; */
689 mutex_self_lock(pthread_mutex_t mutex)
693 switch (mutex->m_type) {
694 /* case PTHREAD_MUTEX_DEFAULT: */
695 case PTHREAD_MUTEX_ERRORCHECK:
697 * POSIX specifies that mutexes should return EDEADLK if a
698 * recursive lock is detected.
703 case PTHREAD_MUTEX_NORMAL:
705 * What SS2 define as a 'normal' mutex. Intentionally
706 * deadlock on attempts to get a lock you already own.
708 _thread_kern_sched_state_unlock(PS_DEADLOCK,
709 &mutex->lock, __FILE__, __LINE__);
712 case PTHREAD_MUTEX_RECURSIVE:
713 /* Increment the lock count: */
714 mutex->m_data.m_count++;
718 /* Trap invalid mutex types; */
726 mutex_unlock_common(pthread_mutex_t * mutex, int add_reference)
730 if (mutex == NULL || *mutex == NULL) {
734 * Defer signals to protect the scheduling queues from
735 * access by the signal handler:
737 _thread_kern_sig_defer();
739 /* Lock the mutex structure: */
740 _SPINLOCK(&(*mutex)->lock);
742 /* Process according to mutex type: */
743 switch ((*mutex)->m_protocol) {
744 /* Default POSIX mutex: */
745 case PTHREAD_PRIO_NONE:
747 * Check if the running thread is not the owner of the
750 if ((*mutex)->m_owner != _thread_run) {
752 * Return an invalid argument error for no
753 * owner and a permission error otherwise:
755 ret = (*mutex)->m_owner == NULL ? EINVAL : EPERM;
757 else if (((*mutex)->m_type == PTHREAD_MUTEX_RECURSIVE) &&
758 ((*mutex)->m_data.m_count > 0)) {
759 /* Decrement the count: */
760 (*mutex)->m_data.m_count--;
763 * Clear the count in case this is recursive
766 (*mutex)->m_data.m_count = 0;
768 /* Remove the mutex from the threads queue. */
769 _MUTEX_ASSERT_IS_OWNED(*mutex);
770 TAILQ_REMOVE(&(*mutex)->m_owner->mutexq,
772 _MUTEX_INIT_LINK(*mutex);
775 * Get the next thread from the queue of
776 * threads waiting on the mutex:
778 if (((*mutex)->m_owner =
779 mutex_queue_deq(*mutex)) != NULL) {
781 * Unless the new owner of the mutex is
782 * currently suspended, allow the owner
783 * to run. If the thread is suspended,
784 * make a note that the thread isn't in
785 * a wait queue any more.
787 if (((*mutex)->m_owner->state !=
789 PTHREAD_NEW_STATE((*mutex)->m_owner,
792 (*mutex)->m_owner->suspended =
797 * Add the mutex to the threads list of
800 TAILQ_INSERT_TAIL(&(*mutex)->m_owner->mutexq,
804 * The owner is no longer waiting for
807 (*mutex)->m_owner->data.mutex = NULL;
812 /* POSIX priority inheritence mutex: */
813 case PTHREAD_PRIO_INHERIT:
815 * Check if the running thread is not the owner of the
818 if ((*mutex)->m_owner != _thread_run) {
820 * Return an invalid argument error for no
821 * owner and a permission error otherwise:
823 ret = (*mutex)->m_owner == NULL ? EINVAL : EPERM;
825 else if (((*mutex)->m_type == PTHREAD_MUTEX_RECURSIVE) &&
826 ((*mutex)->m_data.m_count > 0)) {
827 /* Decrement the count: */
828 (*mutex)->m_data.m_count--;
831 * Clear the count in case this is recursive
834 (*mutex)->m_data.m_count = 0;
837 * Restore the threads inherited priority and
838 * recompute the active priority (being careful
839 * not to override changes in the threads base
840 * priority subsequent to locking the mutex).
842 _thread_run->inherited_priority =
843 (*mutex)->m_saved_prio;
844 _thread_run->active_priority =
845 MAX(_thread_run->inherited_priority,
846 _thread_run->base_priority);
849 * This thread now owns one less priority mutex.
851 _thread_run->priority_mutex_count--;
853 /* Remove the mutex from the threads queue. */
854 _MUTEX_ASSERT_IS_OWNED(*mutex);
855 TAILQ_REMOVE(&(*mutex)->m_owner->mutexq,
857 _MUTEX_INIT_LINK(*mutex);
860 * Get the next thread from the queue of threads
861 * waiting on the mutex:
863 if (((*mutex)->m_owner =
864 mutex_queue_deq(*mutex)) == NULL)
865 /* This mutex has no priority. */
866 (*mutex)->m_prio = 0;
869 * Track number of priority mutexes owned:
871 (*mutex)->m_owner->priority_mutex_count++;
874 * Add the mutex to the threads list
877 TAILQ_INSERT_TAIL(&(*mutex)->m_owner->mutexq,
881 * The owner is no longer waiting for
884 (*mutex)->m_owner->data.mutex = NULL;
887 * Set the priority of the mutex. Since
888 * our waiting threads are in descending
889 * priority order, the priority of the
890 * mutex becomes the active priority of
891 * the thread we just dequeued.
894 (*mutex)->m_owner->active_priority;
897 * Save the owning threads inherited
900 (*mutex)->m_saved_prio =
901 (*mutex)->m_owner->inherited_priority;
904 * The owning threads inherited priority
905 * now becomes his active priority (the
906 * priority of the mutex).
908 (*mutex)->m_owner->inherited_priority =
912 * Unless the new owner of the mutex is
913 * currently suspended, allow the owner
914 * to run. If the thread is suspended,
915 * make a note that the thread isn't in
916 * a wait queue any more.
918 if (((*mutex)->m_owner->state !=
920 PTHREAD_NEW_STATE((*mutex)->m_owner,
923 (*mutex)->m_owner->suspended =
930 /* POSIX priority ceiling mutex: */
931 case PTHREAD_PRIO_PROTECT:
933 * Check if the running thread is not the owner of the
936 if ((*mutex)->m_owner != _thread_run) {
938 * Return an invalid argument error for no
939 * owner and a permission error otherwise:
941 ret = (*mutex)->m_owner == NULL ? EINVAL : EPERM;
943 else if (((*mutex)->m_type == PTHREAD_MUTEX_RECURSIVE) &&
944 ((*mutex)->m_data.m_count > 0)) {
945 /* Decrement the count: */
946 (*mutex)->m_data.m_count--;
949 * Clear the count in case this is recursive
952 (*mutex)->m_data.m_count = 0;
955 * Restore the threads inherited priority and
956 * recompute the active priority (being careful
957 * not to override changes in the threads base
958 * priority subsequent to locking the mutex).
960 _thread_run->inherited_priority =
961 (*mutex)->m_saved_prio;
962 _thread_run->active_priority =
963 MAX(_thread_run->inherited_priority,
964 _thread_run->base_priority);
967 * This thread now owns one less priority mutex.
969 _thread_run->priority_mutex_count--;
971 /* Remove the mutex from the threads queue. */
972 _MUTEX_ASSERT_IS_OWNED(*mutex);
973 TAILQ_REMOVE(&(*mutex)->m_owner->mutexq,
975 _MUTEX_INIT_LINK(*mutex);
978 * Enter a loop to find a waiting thread whose
979 * active priority will not cause a ceiling
982 while ((((*mutex)->m_owner =
983 mutex_queue_deq(*mutex)) != NULL) &&
984 ((*mutex)->m_owner->active_priority >
987 * Either the mutex ceiling priority
988 * been lowered and/or this threads
989 * priority has been raised subsequent
990 * to this thread being queued on the
993 (*mutex)->m_owner->error = EINVAL;
994 PTHREAD_NEW_STATE((*mutex)->m_owner,
997 * The thread is no longer waiting for
1000 (*mutex)->m_owner->data.mutex = NULL;
1003 /* Check for a new owner: */
1004 if ((*mutex)->m_owner != NULL) {
1006 * Track number of priority mutexes owned:
1008 (*mutex)->m_owner->priority_mutex_count++;
1011 * Add the mutex to the threads list
1014 TAILQ_INSERT_TAIL(&(*mutex)->m_owner->mutexq,
1018 * The owner is no longer waiting for
1021 (*mutex)->m_owner->data.mutex = NULL;
1024 * Save the owning threads inherited
1027 (*mutex)->m_saved_prio =
1028 (*mutex)->m_owner->inherited_priority;
1031 * The owning thread inherits the
1032 * ceiling priority of the mutex and
1033 * executes at that priority:
1035 (*mutex)->m_owner->inherited_priority =
1037 (*mutex)->m_owner->active_priority =
1041 * Unless the new owner of the mutex is
1042 * currently suspended, allow the owner
1043 * to run. If the thread is suspended,
1044 * make a note that the thread isn't in
1045 * a wait queue any more.
1047 if (((*mutex)->m_owner->state !=
1049 PTHREAD_NEW_STATE((*mutex)->m_owner,
1052 (*mutex)->m_owner->suspended =
1059 /* Trap invalid mutex types: */
1061 /* Return an invalid argument error: */
1066 if ((ret == 0) && (add_reference != 0)) {
1067 /* Increment the reference count: */
1068 (*mutex)->m_refcount++;
1071 /* Unlock the mutex structure: */
1072 _SPINUNLOCK(&(*mutex)->lock);
1075 * Undefer and handle pending signals, yielding if
1078 _thread_kern_sig_undefer();
1081 /* Return the completion status: */
1087 * This function is called when a change in base priority occurs for
1088 * a thread that is holding or waiting for a priority protection or
1089 * inheritence mutex. A change in a threads base priority can effect
1090 * changes to active priorities of other threads and to the ordering
1091 * of mutex locking by waiting threads.
1093 * This must be called while thread scheduling is deferred.
1096 _mutex_notify_priochange(pthread_t pthread)
1098 /* Adjust the priorites of any owned priority mutexes: */
1099 if (pthread->priority_mutex_count > 0) {
1101 * Rescan the mutexes owned by this thread and correct
1102 * their priorities to account for this threads change
1103 * in priority. This has the side effect of changing
1104 * the threads active priority.
1106 mutex_rescan_owned(pthread, /* rescan all owned */ NULL);
1110 * If this thread is waiting on a priority inheritence mutex,
1111 * check for priority adjustments. A change in priority can
1112 * also effect a ceiling violation(*) for a thread waiting on
1113 * a priority protection mutex; we don't perform the check here
1114 * as it is done in pthread_mutex_unlock.
1116 * (*) It should be noted that a priority change to a thread
1117 * _after_ taking and owning a priority ceiling mutex
1118 * does not affect ownership of that mutex; the ceiling
1119 * priority is only checked before mutex ownership occurs.
1121 if (pthread->state == PS_MUTEX_WAIT) {
1122 /* Lock the mutex structure: */
1123 _SPINLOCK(&pthread->data.mutex->lock);
1126 * Check to make sure this thread is still in the same state
1127 * (the spinlock above can yield the CPU to another thread):
1129 if (pthread->state == PS_MUTEX_WAIT) {
1131 * Remove and reinsert this thread into the list of
1132 * waiting threads to preserve decreasing priority
1135 mutex_queue_remove(pthread->data.mutex, pthread);
1136 mutex_queue_enq(pthread->data.mutex, pthread);
1138 if (pthread->data.mutex->m_protocol ==
1139 PTHREAD_PRIO_INHERIT) {
1140 /* Adjust priorities: */
1141 mutex_priority_adjust(pthread->data.mutex);
1145 /* Unlock the mutex structure: */
1146 _SPINUNLOCK(&pthread->data.mutex->lock);
1151 * Called when a new thread is added to the mutex waiting queue or
1152 * when a threads priority changes that is already in the mutex
1156 mutex_priority_adjust(pthread_mutex_t mutex)
1158 pthread_t pthread_next, pthread = mutex->m_owner;
1160 pthread_mutex_t m = mutex;
1163 * Calculate the mutex priority as the maximum of the highest
1164 * active priority of any waiting threads and the owning threads
1165 * active priority(*).
1167 * (*) Because the owning threads current active priority may
1168 * reflect priority inherited from this mutex (and the mutex
1169 * priority may have changed) we must recalculate the active
1170 * priority based on the threads saved inherited priority
1171 * and its base priority.
1173 pthread_next = TAILQ_FIRST(&m->m_queue); /* should never be NULL */
1174 temp_prio = MAX(pthread_next->active_priority,
1175 MAX(m->m_saved_prio, pthread->base_priority));
1177 /* See if this mutex really needs adjusting: */
1178 if (temp_prio == m->m_prio)
1179 /* No need to propagate the priority: */
1182 /* Set new priority of the mutex: */
1183 m->m_prio = temp_prio;
1187 * Save the threads priority before rescanning the
1190 temp_prio = pthread->active_priority;
1193 * Fix the priorities for all the mutexes this thread has
1194 * locked since taking this mutex. This also has a
1195 * potential side-effect of changing the threads priority.
1197 mutex_rescan_owned(pthread, m);
1200 * If the thread is currently waiting on a mutex, check
1201 * to see if the threads new priority has affected the
1202 * priority of the mutex.
1204 if ((temp_prio != pthread->active_priority) &&
1205 (pthread->state == PS_MUTEX_WAIT) &&
1206 (pthread->data.mutex->m_protocol == PTHREAD_PRIO_INHERIT)) {
1207 /* Grab the mutex this thread is waiting on: */
1208 m = pthread->data.mutex;
1211 * The priority for this thread has changed. Remove
1212 * and reinsert this thread into the list of waiting
1213 * threads to preserve decreasing priority order.
1215 mutex_queue_remove(m, pthread);
1216 mutex_queue_enq(m, pthread);
1218 /* Grab the waiting thread with highest priority: */
1219 pthread_next = TAILQ_FIRST(&m->m_queue);
1222 * Calculate the mutex priority as the maximum of the
1223 * highest active priority of any waiting threads and
1224 * the owning threads active priority.
1226 temp_prio = MAX(pthread_next->active_priority,
1227 MAX(m->m_saved_prio, m->m_owner->base_priority));
1229 if (temp_prio != m->m_prio) {
1231 * The priority needs to be propagated to the
1232 * mutex this thread is waiting on and up to
1233 * the owner of that mutex.
1235 m->m_prio = temp_prio;
1236 pthread = m->m_owner;
1250 mutex_rescan_owned(pthread_t pthread, pthread_mutex_t mutex)
1252 int active_prio, inherited_prio;
1254 pthread_t pthread_next;
1257 * Start walking the mutexes the thread has taken since
1258 * taking this mutex.
1260 if (mutex == NULL) {
1262 * A null mutex means start at the beginning of the owned
1265 m = TAILQ_FIRST(&pthread->mutexq);
1267 /* There is no inherited priority yet. */
1272 * The caller wants to start after a specific mutex. It
1273 * is assumed that this mutex is a priority inheritence
1274 * mutex and that its priority has been correctly
1277 m = TAILQ_NEXT(mutex, m_qe);
1279 /* Start inheriting priority from the specified mutex. */
1280 inherited_prio = mutex->m_prio;
1282 active_prio = MAX(inherited_prio, pthread->base_priority);
1286 * We only want to deal with priority inheritence
1287 * mutexes. This might be optimized by only placing
1288 * priority inheritence mutexes into the owned mutex
1289 * list, but it may prove to be useful having all
1290 * owned mutexes in this list. Consider a thread
1291 * exiting while holding mutexes...
1293 if (m->m_protocol == PTHREAD_PRIO_INHERIT) {
1295 * Fix the owners saved (inherited) priority to
1296 * reflect the priority of the previous mutex.
1298 m->m_saved_prio = inherited_prio;
1300 if ((pthread_next = TAILQ_FIRST(&m->m_queue)) != NULL)
1301 /* Recalculate the priority of the mutex: */
1302 m->m_prio = MAX(active_prio,
1303 pthread_next->active_priority);
1305 m->m_prio = active_prio;
1307 /* Recalculate new inherited and active priorities: */
1308 inherited_prio = m->m_prio;
1309 active_prio = MAX(m->m_prio, pthread->base_priority);
1312 /* Advance to the next mutex owned by this thread: */
1313 m = TAILQ_NEXT(m, m_qe);
1317 * Fix the threads inherited priority and recalculate its
1320 pthread->inherited_priority = inherited_prio;
1321 active_prio = MAX(inherited_prio, pthread->base_priority);
1323 if (active_prio != pthread->active_priority) {
1325 * If this thread is in the priority queue, it must be
1326 * removed and reinserted for its new priority.
1328 if (pthread->flags & PTHREAD_FLAGS_IN_PRIOQ) {
1330 * Remove the thread from the priority queue
1331 * before changing its priority:
1333 PTHREAD_PRIOQ_REMOVE(pthread);
1336 * POSIX states that if the priority is being
1337 * lowered, the thread must be inserted at the
1338 * head of the queue for its priority if it owns
1339 * any priority protection or inheritence mutexes.
1341 if ((active_prio < pthread->active_priority) &&
1342 (pthread->priority_mutex_count > 0)) {
1343 /* Set the new active priority. */
1344 pthread->active_priority = active_prio;
1346 PTHREAD_PRIOQ_INSERT_HEAD(pthread);
1349 /* Set the new active priority. */
1350 pthread->active_priority = active_prio;
1352 PTHREAD_PRIOQ_INSERT_TAIL(pthread);
1356 /* Set the new active priority. */
1357 pthread->active_priority = active_prio;
1363 _mutex_unlock_private(pthread_t pthread)
1365 struct pthread_mutex *m, *m_next;
1367 for (m = TAILQ_FIRST(&pthread->mutexq); m != NULL; m = m_next) {
1368 m_next = TAILQ_NEXT(m, m_qe);
1369 if ((m->m_flags & MUTEX_FLAGS_PRIVATE) != 0)
1370 pthread_mutex_unlock(&m);
1375 _mutex_lock_backout(pthread_t pthread)
1377 struct pthread_mutex *mutex;
1380 * Defer signals to protect the scheduling queues from
1381 * access by the signal handler:
1383 _thread_kern_sig_defer();
1384 if (pthread->state == PS_MUTEX_WAIT) {
1385 mutex = pthread->data.mutex;
1387 /* Lock the mutex structure: */
1388 _SPINLOCK(&mutex->lock);
1390 mutex_queue_remove(mutex, pthread);
1392 /* This thread is no longer waiting for the mutex: */
1393 mutex->m_owner->data.mutex = NULL;
1395 /* Unlock the mutex structure: */
1396 _SPINUNLOCK(&mutex->lock);
1400 * Undefer and handle pending signals, yielding if
1403 _thread_kern_sig_undefer();
1407 * Dequeue a waiting thread from the head of a mutex queue in descending
1410 static inline pthread_t
1411 mutex_queue_deq(pthread_mutex_t mutex)
1415 while ((pthread = TAILQ_FIRST(&mutex->m_queue)) != NULL) {
1416 TAILQ_REMOVE(&mutex->m_queue, pthread, sqe);
1417 pthread->flags &= ~PTHREAD_FLAGS_IN_MUTEXQ;
1420 * Only exit the loop if the thread hasn't been
1423 if (pthread->interrupted == 0)
1431 * Remove a waiting thread from a mutex queue in descending priority order.
1434 mutex_queue_remove(pthread_mutex_t mutex, pthread_t pthread)
1436 if ((pthread->flags & PTHREAD_FLAGS_IN_MUTEXQ) != 0) {
1437 TAILQ_REMOVE(&mutex->m_queue, pthread, sqe);
1438 pthread->flags &= ~PTHREAD_FLAGS_IN_MUTEXQ;
1443 * Enqueue a waiting thread to a queue in descending priority order.
1446 mutex_queue_enq(pthread_mutex_t mutex, pthread_t pthread)
1448 pthread_t tid = TAILQ_LAST(&mutex->m_queue, mutex_head);
1450 PTHREAD_ASSERT_NOT_IN_SYNCQ(pthread);
1452 * For the common case of all threads having equal priority,
1453 * we perform a quick check against the priority of the thread
1454 * at the tail of the queue.
1456 if ((tid == NULL) || (pthread->active_priority <= tid->active_priority))
1457 TAILQ_INSERT_TAIL(&mutex->m_queue, pthread, sqe);
1459 tid = TAILQ_FIRST(&mutex->m_queue);
1460 while (pthread->active_priority <= tid->active_priority)
1461 tid = TAILQ_NEXT(tid, sqe);
1462 TAILQ_INSERT_BEFORE(tid, pthread, sqe);
1464 pthread->flags |= PTHREAD_FLAGS_IN_MUTEXQ;