2 * Copyright (c) 1995 John Birrell <jb@cimlogic.com.au>.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. All advertising materials mentioning features or use of this software
14 * must display the following acknowledgement:
15 * This product includes software developed by John Birrell.
16 * 4. Neither the name of the author nor the names of any co-contributors
17 * may be used to endorse or promote products derived from this software
18 * without specific prior written permission.
20 * THIS SOFTWARE IS PROVIDED BY JOHN BIRRELL AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
37 #include <sys/param.h>
38 #include <sys/queue.h>
41 #include "pthread_private.h"
43 #if defined(_PTHREADS_INVARIANTS)
44 #define _MUTEX_INIT_LINK(m) do { \
45 (m)->m_qe.tqe_prev = NULL; \
46 (m)->m_qe.tqe_next = NULL; \
48 #define _MUTEX_ASSERT_IS_OWNED(m) do { \
49 if ((m)->m_qe.tqe_prev == NULL) \
50 PANIC("mutex is not on list"); \
52 #define _MUTEX_ASSERT_NOT_OWNED(m) do { \
53 if (((m)->m_qe.tqe_prev != NULL) || \
54 ((m)->m_qe.tqe_next != NULL)) \
55 PANIC("mutex is on list"); \
58 #define _MUTEX_INIT_LINK(m)
59 #define _MUTEX_ASSERT_IS_OWNED(m)
60 #define _MUTEX_ASSERT_NOT_OWNED(m)
66 static inline int mutex_self_trylock(pthread_mutex_t);
67 static inline int mutex_self_lock(pthread_mutex_t);
68 static inline int mutex_unlock_common(pthread_mutex_t *, int);
69 static void mutex_priority_adjust(pthread_mutex_t);
70 static void mutex_rescan_owned (pthread_t, pthread_mutex_t);
71 static inline pthread_t mutex_queue_deq(pthread_mutex_t);
72 static inline void mutex_queue_remove(pthread_mutex_t, pthread_t);
73 static inline void mutex_queue_enq(pthread_mutex_t, pthread_t);
76 static spinlock_t static_init_lock = _SPINLOCK_INITIALIZER;
78 /* Reinitialize a mutex to defaults. */
80 _mutex_reinit(pthread_mutex_t * mutex)
86 else if (*mutex == NULL)
87 ret = pthread_mutex_init(mutex, NULL);
90 * Initialize the mutex structure:
92 (*mutex)->m_type = PTHREAD_MUTEX_DEFAULT;
93 (*mutex)->m_protocol = PTHREAD_PRIO_NONE;
94 TAILQ_INIT(&(*mutex)->m_queue);
95 (*mutex)->m_owner = NULL;
96 (*mutex)->m_data.m_count = 0;
97 (*mutex)->m_flags = MUTEX_FLAGS_INITED;
98 (*mutex)->m_refcount = 0;
100 (*mutex)->m_saved_prio = 0;
101 _MUTEX_INIT_LINK(*mutex);
102 memset(&(*mutex)->lock, 0, sizeof((*mutex)->lock));
108 pthread_mutex_init(pthread_mutex_t * mutex,
109 const pthread_mutexattr_t * mutex_attr)
111 enum pthread_mutextype type;
114 pthread_mutex_t pmutex;
120 /* Check if default mutex attributes: */
121 else if (mutex_attr == NULL || *mutex_attr == NULL) {
122 /* Default to a (error checking) POSIX mutex: */
123 type = PTHREAD_MUTEX_ERRORCHECK;
124 protocol = PTHREAD_PRIO_NONE;
125 ceiling = PTHREAD_MAX_PRIORITY;
128 /* Check mutex type: */
129 else if (((*mutex_attr)->m_type < PTHREAD_MUTEX_ERRORCHECK) ||
130 ((*mutex_attr)->m_type >= MUTEX_TYPE_MAX))
131 /* Return an invalid argument error: */
134 /* Check mutex protocol: */
135 else if (((*mutex_attr)->m_protocol < PTHREAD_PRIO_NONE) ||
136 ((*mutex_attr)->m_protocol > PTHREAD_MUTEX_RECURSIVE))
137 /* Return an invalid argument error: */
141 /* Use the requested mutex type and protocol: */
142 type = (*mutex_attr)->m_type;
143 protocol = (*mutex_attr)->m_protocol;
144 ceiling = (*mutex_attr)->m_ceiling;
147 /* Check no errors so far: */
149 if ((pmutex = (pthread_mutex_t)
150 malloc(sizeof(struct pthread_mutex))) == NULL)
153 /* Reset the mutex flags: */
156 /* Process according to mutex type: */
158 /* case PTHREAD_MUTEX_DEFAULT: */
159 case PTHREAD_MUTEX_ERRORCHECK:
160 case PTHREAD_MUTEX_NORMAL:
161 /* Nothing to do here. */
164 /* Single UNIX Spec 2 recursive mutex: */
165 case PTHREAD_MUTEX_RECURSIVE:
166 /* Reset the mutex count: */
167 pmutex->m_data.m_count = 0;
170 /* Trap invalid mutex types: */
172 /* Return an invalid argument error: */
177 /* Initialise the rest of the mutex: */
178 TAILQ_INIT(&pmutex->m_queue);
179 pmutex->m_flags |= MUTEX_FLAGS_INITED;
180 pmutex->m_owner = NULL;
181 pmutex->m_type = type;
182 pmutex->m_protocol = protocol;
183 pmutex->m_refcount = 0;
184 if (protocol == PTHREAD_PRIO_PROTECT)
185 pmutex->m_prio = ceiling;
188 pmutex->m_saved_prio = 0;
189 _MUTEX_INIT_LINK(pmutex);
190 memset(&pmutex->lock, 0, sizeof(pmutex->lock));
198 /* Return the completion status: */
203 pthread_mutex_destroy(pthread_mutex_t * mutex)
207 if (mutex == NULL || *mutex == NULL)
210 /* Lock the mutex structure: */
211 _SPINLOCK(&(*mutex)->lock);
214 * Check to see if this mutex is in use:
216 if (((*mutex)->m_owner != NULL) ||
217 (TAILQ_FIRST(&(*mutex)->m_queue) != NULL) ||
218 ((*mutex)->m_refcount != 0)) {
221 /* Unlock the mutex structure: */
222 _SPINUNLOCK(&(*mutex)->lock);
226 * Free the memory allocated for the mutex
229 _MUTEX_ASSERT_NOT_OWNED(*mutex);
233 * Leave the caller's pointer NULL now that
234 * the mutex has been destroyed:
240 /* Return the completion status: */
245 init_static (pthread_mutex_t *mutex)
249 _SPINLOCK(&static_init_lock);
252 ret = pthread_mutex_init(mutex, NULL);
256 _SPINUNLOCK(&static_init_lock);
262 pthread_mutex_trylock(pthread_mutex_t * mutex)
270 * If the mutex is statically initialized, perform the dynamic
273 else if (*mutex != NULL || (ret = init_static(mutex)) == 0) {
275 * Defer signals to protect the scheduling queues from
276 * access by the signal handler:
278 _thread_kern_sig_defer();
280 /* Lock the mutex structure: */
281 _SPINLOCK(&(*mutex)->lock);
284 * If the mutex was statically allocated, properly
285 * initialize the tail queue.
287 if (((*mutex)->m_flags & MUTEX_FLAGS_INITED) == 0) {
288 TAILQ_INIT(&(*mutex)->m_queue);
289 _MUTEX_INIT_LINK(*mutex);
290 (*mutex)->m_flags |= MUTEX_FLAGS_INITED;
293 /* Process according to mutex type: */
294 switch ((*mutex)->m_protocol) {
295 /* Default POSIX mutex: */
296 case PTHREAD_PRIO_NONE:
297 /* Check if this mutex is not locked: */
298 if ((*mutex)->m_owner == NULL) {
299 /* Lock the mutex for the running thread: */
300 (*mutex)->m_owner = _thread_run;
302 /* Add to the list of owned mutexes: */
303 _MUTEX_ASSERT_NOT_OWNED(*mutex);
304 TAILQ_INSERT_TAIL(&_thread_run->mutexq,
306 } else if ((*mutex)->m_owner == _thread_run)
307 ret = mutex_self_trylock(*mutex);
309 /* Return a busy error: */
313 /* POSIX priority inheritence mutex: */
314 case PTHREAD_PRIO_INHERIT:
315 /* Check if this mutex is not locked: */
316 if ((*mutex)->m_owner == NULL) {
317 /* Lock the mutex for the running thread: */
318 (*mutex)->m_owner = _thread_run;
320 /* Track number of priority mutexes owned: */
321 _thread_run->priority_mutex_count++;
324 * The mutex takes on the attributes of the
325 * running thread when there are no waiters.
327 (*mutex)->m_prio = _thread_run->active_priority;
328 (*mutex)->m_saved_prio =
329 _thread_run->inherited_priority;
331 /* Add to the list of owned mutexes: */
332 _MUTEX_ASSERT_NOT_OWNED(*mutex);
333 TAILQ_INSERT_TAIL(&_thread_run->mutexq,
335 } else if ((*mutex)->m_owner == _thread_run)
336 ret = mutex_self_trylock(*mutex);
338 /* Return a busy error: */
342 /* POSIX priority protection mutex: */
343 case PTHREAD_PRIO_PROTECT:
344 /* Check for a priority ceiling violation: */
345 if (_thread_run->active_priority > (*mutex)->m_prio)
348 /* Check if this mutex is not locked: */
349 else if ((*mutex)->m_owner == NULL) {
350 /* Lock the mutex for the running thread: */
351 (*mutex)->m_owner = _thread_run;
353 /* Track number of priority mutexes owned: */
354 _thread_run->priority_mutex_count++;
357 * The running thread inherits the ceiling
358 * priority of the mutex and executes at that
361 _thread_run->active_priority = (*mutex)->m_prio;
362 (*mutex)->m_saved_prio =
363 _thread_run->inherited_priority;
364 _thread_run->inherited_priority =
367 /* Add to the list of owned mutexes: */
368 _MUTEX_ASSERT_NOT_OWNED(*mutex);
369 TAILQ_INSERT_TAIL(&_thread_run->mutexq,
371 } else if ((*mutex)->m_owner == _thread_run)
372 ret = mutex_self_trylock(*mutex);
374 /* Return a busy error: */
378 /* Trap invalid mutex types: */
380 /* Return an invalid argument error: */
385 /* Unlock the mutex structure: */
386 _SPINUNLOCK(&(*mutex)->lock);
389 * Undefer and handle pending signals, yielding if
392 _thread_kern_sig_undefer();
395 /* Return the completion status: */
400 pthread_mutex_lock(pthread_mutex_t * mutex)
408 * If the mutex is statically initialized, perform the dynamic
411 else if (*mutex != NULL || (ret = init_static(mutex)) == 0) {
413 * Defer signals to protect the scheduling queues from
414 * access by the signal handler:
416 _thread_kern_sig_defer();
418 /* Lock the mutex structure: */
419 _SPINLOCK(&(*mutex)->lock);
422 * If the mutex was statically allocated, properly
423 * initialize the tail queue.
425 if (((*mutex)->m_flags & MUTEX_FLAGS_INITED) == 0) {
426 TAILQ_INIT(&(*mutex)->m_queue);
427 (*mutex)->m_flags |= MUTEX_FLAGS_INITED;
428 _MUTEX_INIT_LINK(*mutex);
431 /* Process according to mutex type: */
432 switch ((*mutex)->m_protocol) {
433 /* Default POSIX mutex: */
434 case PTHREAD_PRIO_NONE:
435 if ((*mutex)->m_owner == NULL) {
436 /* Lock the mutex for this thread: */
437 (*mutex)->m_owner = _thread_run;
439 /* Add to the list of owned mutexes: */
440 _MUTEX_ASSERT_NOT_OWNED(*mutex);
441 TAILQ_INSERT_TAIL(&_thread_run->mutexq,
444 } else if ((*mutex)->m_owner == _thread_run)
445 ret = mutex_self_lock(*mutex);
448 * Join the queue of threads waiting to lock
451 mutex_queue_enq(*mutex, _thread_run);
454 * Keep a pointer to the mutex this thread
457 _thread_run->data.mutex = *mutex;
460 * Unlock the mutex structure and schedule the
463 _thread_kern_sched_state_unlock(PS_MUTEX_WAIT,
464 &(*mutex)->lock, __FILE__, __LINE__);
466 /* Lock the mutex structure again: */
467 _SPINLOCK(&(*mutex)->lock);
471 /* POSIX priority inheritence mutex: */
472 case PTHREAD_PRIO_INHERIT:
473 /* Check if this mutex is not locked: */
474 if ((*mutex)->m_owner == NULL) {
475 /* Lock the mutex for this thread: */
476 (*mutex)->m_owner = _thread_run;
478 /* Track number of priority mutexes owned: */
479 _thread_run->priority_mutex_count++;
482 * The mutex takes on attributes of the
483 * running thread when there are no waiters.
485 (*mutex)->m_prio = _thread_run->active_priority;
486 (*mutex)->m_saved_prio =
487 _thread_run->inherited_priority;
488 _thread_run->inherited_priority =
491 /* Add to the list of owned mutexes: */
492 _MUTEX_ASSERT_NOT_OWNED(*mutex);
493 TAILQ_INSERT_TAIL(&_thread_run->mutexq,
496 } else if ((*mutex)->m_owner == _thread_run)
497 ret = mutex_self_lock(*mutex);
500 * Join the queue of threads waiting to lock
503 mutex_queue_enq(*mutex, _thread_run);
506 * Keep a pointer to the mutex this thread
509 _thread_run->data.mutex = *mutex;
511 if (_thread_run->active_priority >
513 /* Adjust priorities: */
514 mutex_priority_adjust(*mutex);
517 * Unlock the mutex structure and schedule the
520 _thread_kern_sched_state_unlock(PS_MUTEX_WAIT,
521 &(*mutex)->lock, __FILE__, __LINE__);
523 /* Lock the mutex structure again: */
524 _SPINLOCK(&(*mutex)->lock);
528 /* POSIX priority protection mutex: */
529 case PTHREAD_PRIO_PROTECT:
530 /* Check for a priority ceiling violation: */
531 if (_thread_run->active_priority > (*mutex)->m_prio)
534 /* Check if this mutex is not locked: */
535 else if ((*mutex)->m_owner == NULL) {
537 * Lock the mutex for the running
540 (*mutex)->m_owner = _thread_run;
542 /* Track number of priority mutexes owned: */
543 _thread_run->priority_mutex_count++;
546 * The running thread inherits the ceiling
547 * priority of the mutex and executes at that
550 _thread_run->active_priority = (*mutex)->m_prio;
551 (*mutex)->m_saved_prio =
552 _thread_run->inherited_priority;
553 _thread_run->inherited_priority =
556 /* Add to the list of owned mutexes: */
557 _MUTEX_ASSERT_NOT_OWNED(*mutex);
558 TAILQ_INSERT_TAIL(&_thread_run->mutexq,
560 } else if ((*mutex)->m_owner == _thread_run)
561 ret = mutex_self_lock(*mutex);
564 * Join the queue of threads waiting to lock
567 mutex_queue_enq(*mutex, _thread_run);
570 * Keep a pointer to the mutex this thread
573 _thread_run->data.mutex = *mutex;
575 /* Clear any previous error: */
576 _thread_run->error = 0;
579 * Unlock the mutex structure and schedule the
582 _thread_kern_sched_state_unlock(PS_MUTEX_WAIT,
583 &(*mutex)->lock, __FILE__, __LINE__);
585 /* Lock the mutex structure again: */
586 _SPINLOCK(&(*mutex)->lock);
589 * The threads priority may have changed while
590 * waiting for the mutex causing a ceiling
593 ret = _thread_run->error;
594 _thread_run->error = 0;
598 /* Trap invalid mutex types: */
600 /* Return an invalid argument error: */
605 /* Unlock the mutex structure: */
606 _SPINUNLOCK(&(*mutex)->lock);
609 * Undefer and handle pending signals, yielding if
612 _thread_kern_sig_undefer();
615 /* Return the completion status: */
620 pthread_mutex_unlock(pthread_mutex_t * mutex)
622 return (mutex_unlock_common(mutex, /* add reference */ 0));
626 _mutex_cv_unlock(pthread_mutex_t * mutex)
628 return (mutex_unlock_common(mutex, /* add reference */ 1));
632 _mutex_cv_lock(pthread_mutex_t * mutex)
635 if ((ret = pthread_mutex_lock(mutex)) == 0)
636 (*mutex)->m_refcount--;
641 mutex_self_trylock(pthread_mutex_t mutex)
645 switch (mutex->m_type) {
647 /* case PTHREAD_MUTEX_DEFAULT: */
648 case PTHREAD_MUTEX_ERRORCHECK:
649 case PTHREAD_MUTEX_NORMAL:
651 * POSIX specifies that mutexes should return EDEADLK if a
652 * recursive lock is detected.
657 case PTHREAD_MUTEX_RECURSIVE:
658 /* Increment the lock count: */
659 mutex->m_data.m_count++;
663 /* Trap invalid mutex types; */
671 mutex_self_lock(pthread_mutex_t mutex)
675 switch (mutex->m_type) {
676 /* case PTHREAD_MUTEX_DEFAULT: */
677 case PTHREAD_MUTEX_ERRORCHECK:
679 * POSIX specifies that mutexes should return EDEADLK if a
680 * recursive lock is detected.
685 case PTHREAD_MUTEX_NORMAL:
687 * What SS2 define as a 'normal' mutex. Intentionally
688 * deadlock on attempts to get a lock you already own.
690 _thread_kern_sched_state_unlock(PS_DEADLOCK,
691 &mutex->lock, __FILE__, __LINE__);
694 case PTHREAD_MUTEX_RECURSIVE:
695 /* Increment the lock count: */
696 mutex->m_data.m_count++;
700 /* Trap invalid mutex types; */
708 mutex_unlock_common(pthread_mutex_t * mutex, int add_reference)
712 if (mutex == NULL || *mutex == NULL) {
716 * Defer signals to protect the scheduling queues from
717 * access by the signal handler:
719 _thread_kern_sig_defer();
721 /* Lock the mutex structure: */
722 _SPINLOCK(&(*mutex)->lock);
724 /* Process according to mutex type: */
725 switch ((*mutex)->m_protocol) {
726 /* Default POSIX mutex: */
727 case PTHREAD_PRIO_NONE:
729 * Check if the running thread is not the owner of the
732 if ((*mutex)->m_owner != _thread_run) {
734 * Return an invalid argument error for no
735 * owner and a permission error otherwise:
737 ret = (*mutex)->m_owner == NULL ? EINVAL : EPERM;
739 else if (((*mutex)->m_type == PTHREAD_MUTEX_RECURSIVE) &&
740 ((*mutex)->m_data.m_count > 1)) {
741 /* Decrement the count: */
742 (*mutex)->m_data.m_count--;
745 * Clear the count in case this is recursive
748 (*mutex)->m_data.m_count = 0;
750 /* Remove the mutex from the threads queue. */
751 _MUTEX_ASSERT_IS_OWNED(*mutex);
752 TAILQ_REMOVE(&(*mutex)->m_owner->mutexq,
754 _MUTEX_INIT_LINK(*mutex);
757 * Get the next thread from the queue of
758 * threads waiting on the mutex:
760 if (((*mutex)->m_owner =
761 mutex_queue_deq(*mutex)) != NULL) {
763 * Allow the new owner of the mutex to
766 PTHREAD_NEW_STATE((*mutex)->m_owner,
770 * Add the mutex to the threads list of
773 TAILQ_INSERT_TAIL(&(*mutex)->m_owner->mutexq,
777 * The owner is no longer waiting for
780 (*mutex)->m_owner->data.mutex = NULL;
785 /* POSIX priority inheritence mutex: */
786 case PTHREAD_PRIO_INHERIT:
788 * Check if the running thread is not the owner of the
791 if ((*mutex)->m_owner != _thread_run) {
793 * Return an invalid argument error for no
794 * owner and a permission error otherwise:
796 ret = (*mutex)->m_owner == NULL ? EINVAL : EPERM;
798 else if (((*mutex)->m_type == PTHREAD_MUTEX_RECURSIVE) &&
799 ((*mutex)->m_data.m_count > 1)) {
800 /* Decrement the count: */
801 (*mutex)->m_data.m_count--;
804 * Clear the count in case this is recursive
807 (*mutex)->m_data.m_count = 0;
810 * Restore the threads inherited priority and
811 * recompute the active priority (being careful
812 * not to override changes in the threads base
813 * priority subsequent to locking the mutex).
815 _thread_run->inherited_priority =
816 (*mutex)->m_saved_prio;
817 _thread_run->active_priority =
818 MAX(_thread_run->inherited_priority,
819 _thread_run->base_priority);
822 * This thread now owns one less priority mutex.
824 _thread_run->priority_mutex_count--;
826 /* Remove the mutex from the threads queue. */
827 _MUTEX_ASSERT_IS_OWNED(*mutex);
828 TAILQ_REMOVE(&(*mutex)->m_owner->mutexq,
830 _MUTEX_INIT_LINK(*mutex);
833 * Get the next thread from the queue of threads
834 * waiting on the mutex:
836 if (((*mutex)->m_owner =
837 mutex_queue_deq(*mutex)) == NULL)
838 /* This mutex has no priority. */
839 (*mutex)->m_prio = 0;
842 * Track number of priority mutexes owned:
844 (*mutex)->m_owner->priority_mutex_count++;
847 * Add the mutex to the threads list
850 TAILQ_INSERT_TAIL(&(*mutex)->m_owner->mutexq,
854 * The owner is no longer waiting for
857 (*mutex)->m_owner->data.mutex = NULL;
860 * Set the priority of the mutex. Since
861 * our waiting threads are in descending
862 * priority order, the priority of the
863 * mutex becomes the active priority of
864 * the thread we just dequeued.
867 (*mutex)->m_owner->active_priority;
870 * Save the owning threads inherited
873 (*mutex)->m_saved_prio =
874 (*mutex)->m_owner->inherited_priority;
877 * The owning threads inherited priority
878 * now becomes his active priority (the
879 * priority of the mutex).
881 (*mutex)->m_owner->inherited_priority =
885 * Allow the new owner of the mutex to
888 PTHREAD_NEW_STATE((*mutex)->m_owner,
894 /* POSIX priority ceiling mutex: */
895 case PTHREAD_PRIO_PROTECT:
897 * Check if the running thread is not the owner of the
900 if ((*mutex)->m_owner != _thread_run) {
902 * Return an invalid argument error for no
903 * owner and a permission error otherwise:
905 ret = (*mutex)->m_owner == NULL ? EINVAL : EPERM;
907 else if (((*mutex)->m_type == PTHREAD_MUTEX_RECURSIVE) &&
908 ((*mutex)->m_data.m_count > 1)) {
909 /* Decrement the count: */
910 (*mutex)->m_data.m_count--;
913 * Clear the count in case this is recursive
916 (*mutex)->m_data.m_count = 0;
919 * Restore the threads inherited priority and
920 * recompute the active priority (being careful
921 * not to override changes in the threads base
922 * priority subsequent to locking the mutex).
924 _thread_run->inherited_priority =
925 (*mutex)->m_saved_prio;
926 _thread_run->active_priority =
927 MAX(_thread_run->inherited_priority,
928 _thread_run->base_priority);
931 * This thread now owns one less priority mutex.
933 _thread_run->priority_mutex_count--;
935 /* Remove the mutex from the threads queue. */
936 _MUTEX_ASSERT_IS_OWNED(*mutex);
937 TAILQ_REMOVE(&(*mutex)->m_owner->mutexq,
939 _MUTEX_INIT_LINK(*mutex);
942 * Enter a loop to find a waiting thread whose
943 * active priority will not cause a ceiling
946 while ((((*mutex)->m_owner =
947 mutex_queue_deq(*mutex)) != NULL) &&
948 ((*mutex)->m_owner->active_priority >
951 * Either the mutex ceiling priority
952 * been lowered and/or this threads
953 * priority has been raised subsequent
954 * to this thread being queued on the
957 (*mutex)->m_owner->error = EINVAL;
958 PTHREAD_NEW_STATE((*mutex)->m_owner,
961 * The thread is no longer waiting for
964 (*mutex)->m_owner->data.mutex = NULL;
967 /* Check for a new owner: */
968 if ((*mutex)->m_owner != NULL) {
970 * Track number of priority mutexes owned:
972 (*mutex)->m_owner->priority_mutex_count++;
975 * Add the mutex to the threads list
978 TAILQ_INSERT_TAIL(&(*mutex)->m_owner->mutexq,
982 * The owner is no longer waiting for
985 (*mutex)->m_owner->data.mutex = NULL;
988 * Save the owning threads inherited
991 (*mutex)->m_saved_prio =
992 (*mutex)->m_owner->inherited_priority;
995 * The owning thread inherits the
996 * ceiling priority of the mutex and
997 * executes at that priority:
999 (*mutex)->m_owner->inherited_priority =
1001 (*mutex)->m_owner->active_priority =
1005 * Allow the new owner of the mutex to
1008 PTHREAD_NEW_STATE((*mutex)->m_owner,
1014 /* Trap invalid mutex types: */
1016 /* Return an invalid argument error: */
1021 if ((ret == 0) && (add_reference != 0)) {
1022 /* Increment the reference count: */
1023 (*mutex)->m_refcount++;
1026 /* Unlock the mutex structure: */
1027 _SPINUNLOCK(&(*mutex)->lock);
1030 * Undefer and handle pending signals, yielding if
1033 _thread_kern_sig_undefer();
1036 /* Return the completion status: */
1042 * This function is called when a change in base priority occurs for
1043 * a thread that is holding or waiting for a priority protection or
1044 * inheritence mutex. A change in a threads base priority can effect
1045 * changes to active priorities of other threads and to the ordering
1046 * of mutex locking by waiting threads.
1048 * This must be called while thread scheduling is deferred.
1051 _mutex_notify_priochange(pthread_t pthread)
1053 /* Adjust the priorites of any owned priority mutexes: */
1054 if (pthread->priority_mutex_count > 0) {
1056 * Rescan the mutexes owned by this thread and correct
1057 * their priorities to account for this threads change
1058 * in priority. This has the side effect of changing
1059 * the threads active priority.
1061 mutex_rescan_owned(pthread, /* rescan all owned */ NULL);
1065 * If this thread is waiting on a priority inheritence mutex,
1066 * check for priority adjustments. A change in priority can
1067 * also effect a ceiling violation(*) for a thread waiting on
1068 * a priority protection mutex; we don't perform the check here
1069 * as it is done in pthread_mutex_unlock.
1071 * (*) It should be noted that a priority change to a thread
1072 * _after_ taking and owning a priority ceiling mutex
1073 * does not affect ownership of that mutex; the ceiling
1074 * priority is only checked before mutex ownership occurs.
1076 if (pthread->state == PS_MUTEX_WAIT) {
1077 /* Lock the mutex structure: */
1078 _SPINLOCK(&pthread->data.mutex->lock);
1081 * Check to make sure this thread is still in the same state
1082 * (the spinlock above can yield the CPU to another thread):
1084 if (pthread->state == PS_MUTEX_WAIT) {
1086 * Remove and reinsert this thread into the list of
1087 * waiting threads to preserve decreasing priority
1090 mutex_queue_remove(pthread->data.mutex, pthread);
1091 mutex_queue_enq(pthread->data.mutex, pthread);
1093 if (pthread->data.mutex->m_protocol ==
1094 PTHREAD_PRIO_INHERIT) {
1095 /* Adjust priorities: */
1096 mutex_priority_adjust(pthread->data.mutex);
1100 /* Unlock the mutex structure: */
1101 _SPINUNLOCK(&pthread->data.mutex->lock);
1106 * Called when a new thread is added to the mutex waiting queue or
1107 * when a threads priority changes that is already in the mutex
1111 mutex_priority_adjust(pthread_mutex_t mutex)
1113 pthread_t pthread_next, pthread = mutex->m_owner;
1115 pthread_mutex_t m = mutex;
1118 * Calculate the mutex priority as the maximum of the highest
1119 * active priority of any waiting threads and the owning threads
1120 * active priority(*).
1122 * (*) Because the owning threads current active priority may
1123 * reflect priority inherited from this mutex (and the mutex
1124 * priority may have changed) we must recalculate the active
1125 * priority based on the threads saved inherited priority
1126 * and its base priority.
1128 pthread_next = TAILQ_FIRST(&m->m_queue); /* should never be NULL */
1129 temp_prio = MAX(pthread_next->active_priority,
1130 MAX(m->m_saved_prio, pthread->base_priority));
1132 /* See if this mutex really needs adjusting: */
1133 if (temp_prio == m->m_prio)
1134 /* No need to propagate the priority: */
1137 /* Set new priority of the mutex: */
1138 m->m_prio = temp_prio;
1142 * Save the threads priority before rescanning the
1145 temp_prio = pthread->active_priority;
1148 * Fix the priorities for all the mutexes this thread has
1149 * locked since taking this mutex. This also has a
1150 * potential side-effect of changing the threads priority.
1152 mutex_rescan_owned(pthread, m);
1155 * If the thread is currently waiting on a mutex, check
1156 * to see if the threads new priority has affected the
1157 * priority of the mutex.
1159 if ((temp_prio != pthread->active_priority) &&
1160 (pthread->state == PS_MUTEX_WAIT) &&
1161 (pthread->data.mutex->m_protocol == PTHREAD_PRIO_INHERIT)) {
1162 /* Grab the mutex this thread is waiting on: */
1163 m = pthread->data.mutex;
1166 * The priority for this thread has changed. Remove
1167 * and reinsert this thread into the list of waiting
1168 * threads to preserve decreasing priority order.
1170 mutex_queue_remove(m, pthread);
1171 mutex_queue_enq(m, pthread);
1173 /* Grab the waiting thread with highest priority: */
1174 pthread_next = TAILQ_FIRST(&m->m_queue);
1177 * Calculate the mutex priority as the maximum of the
1178 * highest active priority of any waiting threads and
1179 * the owning threads active priority.
1181 temp_prio = MAX(pthread_next->active_priority,
1182 MAX(m->m_saved_prio, m->m_owner->base_priority));
1184 if (temp_prio != m->m_prio) {
1186 * The priority needs to be propagated to the
1187 * mutex this thread is waiting on and up to
1188 * the owner of that mutex.
1190 m->m_prio = temp_prio;
1191 pthread = m->m_owner;
1205 mutex_rescan_owned (pthread_t pthread, pthread_mutex_t mutex)
1207 int active_prio, inherited_prio;
1209 pthread_t pthread_next;
1212 * Start walking the mutexes the thread has taken since
1213 * taking this mutex.
1215 if (mutex == NULL) {
1217 * A null mutex means start at the beginning of the owned
1220 m = TAILQ_FIRST(&pthread->mutexq);
1222 /* There is no inherited priority yet. */
1227 * The caller wants to start after a specific mutex. It
1228 * is assumed that this mutex is a priority inheritence
1229 * mutex and that its priority has been correctly
1232 m = TAILQ_NEXT(mutex, m_qe);
1234 /* Start inheriting priority from the specified mutex. */
1235 inherited_prio = mutex->m_prio;
1237 active_prio = MAX(inherited_prio, pthread->base_priority);
1241 * We only want to deal with priority inheritence
1242 * mutexes. This might be optimized by only placing
1243 * priority inheritence mutexes into the owned mutex
1244 * list, but it may prove to be useful having all
1245 * owned mutexes in this list. Consider a thread
1246 * exiting while holding mutexes...
1248 if (m->m_protocol == PTHREAD_PRIO_INHERIT) {
1250 * Fix the owners saved (inherited) priority to
1251 * reflect the priority of the previous mutex.
1253 m->m_saved_prio = inherited_prio;
1255 if ((pthread_next = TAILQ_FIRST(&m->m_queue)) != NULL)
1256 /* Recalculate the priority of the mutex: */
1257 m->m_prio = MAX(active_prio,
1258 pthread_next->active_priority);
1260 m->m_prio = active_prio;
1262 /* Recalculate new inherited and active priorities: */
1263 inherited_prio = m->m_prio;
1264 active_prio = MAX(m->m_prio, pthread->base_priority);
1267 /* Advance to the next mutex owned by this thread: */
1268 m = TAILQ_NEXT(m, m_qe);
1272 * Fix the threads inherited priority and recalculate its
1275 pthread->inherited_priority = inherited_prio;
1276 active_prio = MAX(inherited_prio, pthread->base_priority);
1278 if (active_prio != pthread->active_priority) {
1280 * If this thread is in the priority queue, it must be
1281 * removed and reinserted for its new priority.
1283 if (pthread->flags & PTHREAD_FLAGS_IN_PRIOQ) {
1285 * Remove the thread from the priority queue
1286 * before changing its priority:
1288 PTHREAD_PRIOQ_REMOVE(pthread);
1291 * POSIX states that if the priority is being
1292 * lowered, the thread must be inserted at the
1293 * head of the queue for its priority if it owns
1294 * any priority protection or inheritence mutexes.
1296 if ((active_prio < pthread->active_priority) &&
1297 (pthread->priority_mutex_count > 0)) {
1298 /* Set the new active priority. */
1299 pthread->active_priority = active_prio;
1301 PTHREAD_PRIOQ_INSERT_HEAD(pthread);
1304 /* Set the new active priority. */
1305 pthread->active_priority = active_prio;
1307 PTHREAD_PRIOQ_INSERT_TAIL(pthread);
1311 /* Set the new active priority. */
1312 pthread->active_priority = active_prio;
1318 * Dequeue a waiting thread from the head of a mutex queue in descending
1321 static inline pthread_t
1322 mutex_queue_deq(pthread_mutex_t mutex)
1326 if ((pthread = TAILQ_FIRST(&mutex->m_queue)) != NULL)
1327 TAILQ_REMOVE(&mutex->m_queue, pthread, qe);
1333 * Remove a waiting thread from a mutex queue in descending priority order.
1336 mutex_queue_remove(pthread_mutex_t mutex, pthread_t pthread)
1338 TAILQ_REMOVE(&mutex->m_queue, pthread, qe);
1342 * Enqueue a waiting thread to a queue in descending priority order.
1345 mutex_queue_enq(pthread_mutex_t mutex, pthread_t pthread)
1347 pthread_t tid = TAILQ_LAST(&mutex->m_queue, mutex_head);
1350 * For the common case of all threads having equal priority,
1351 * we perform a quick check against the priority of the thread
1352 * at the tail of the queue.
1354 if ((tid == NULL) || (pthread->active_priority <= tid->active_priority))
1355 TAILQ_INSERT_TAIL(&mutex->m_queue, pthread, qe);
1357 tid = TAILQ_FIRST(&mutex->m_queue);
1358 while (pthread->active_priority <= tid->active_priority)
1359 tid = TAILQ_NEXT(tid, qe);
1360 TAILQ_INSERT_BEFORE(tid, pthread, qe);