lib/libthr/thread/thr_mutex.c

   1 /*
   2  * Copyright (c) 1995 John Birrell <jb@cimlogic.com.au>.
   3  * Copyright (c) 2006 David Xu <davidxu@freebsd.org>.
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  * 3. All advertising materials mentioning features or use of this software
  15  *    must display the following acknowledgement:
  16  *      This product includes software developed by John Birrell.
  17  * 4. Neither the name of the author nor the names of any co-contributors
  18  *    may be used to endorse or promote products derived from this software
  19  *    without specific prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY JOHN BIRRELL AND CONTRIBUTORS ``AS IS'' AND
  22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  24  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  25  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  30  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  31  * SUCH DAMAGE.
  32  *
  33  * $FreeBSD$
  34  */
  35
  36 #include "namespace.h"
  37 #include <stdlib.h>
  38 #include <errno.h>
  39 #include <string.h>
  40 #include <sys/param.h>
  41 #include <sys/queue.h>
  42 #include <pthread.h>
  43 #include <pthread_np.h>
  44 #include "un-namespace.h"
  45
  46 #include "thr_private.h"
  47
  48 #if defined(_PTHREADS_INVARIANTS)
  49 #define MUTEX_INIT_LINK(m)              do {            \
  50         (m)->m_qe.tqe_prev = NULL;                      \
  51         (m)->m_qe.tqe_next = NULL;                      \
  52 } while (0)
  53 #define MUTEX_ASSERT_IS_OWNED(m)        do {            \
  54         if (__predict_false((m)->m_qe.tqe_prev == NULL))\
  55                 PANIC("mutex is not on list");          \
  56 } while (0)
  57 #define MUTEX_ASSERT_NOT_OWNED(m)       do {            \
  58         if (__predict_false((m)->m_qe.tqe_prev != NULL ||       \
  59             (m)->m_qe.tqe_next != NULL))        \
  60                 PANIC("mutex is on list");              \
  61 } while (0)
  62 #else
  63 #define MUTEX_INIT_LINK(m)
  64 #define MUTEX_ASSERT_IS_OWNED(m)
  65 #define MUTEX_ASSERT_NOT_OWNED(m)
  66 #endif
  67
  68 /*
  69  * For adaptive mutexes, how many times to spin doing trylock2
  70  * before entering the kernel to block
  71  */
  72 #define MUTEX_ADAPTIVE_SPINS    2000
  73
  74 /*
  75  * Prototypes
  76  */
  77 int     __pthread_mutex_init(pthread_mutex_t *mutex,
  78                 const pthread_mutexattr_t *mutex_attr);
  79 int     __pthread_mutex_trylock(pthread_mutex_t *mutex);
  80 int     __pthread_mutex_lock(pthread_mutex_t *mutex);
  81 int     __pthread_mutex_timedlock(pthread_mutex_t *mutex,
  82                 const struct timespec *abstime);
  83 int     _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
  84                 void *(calloc_cb)(size_t, size_t));
  85 int     _pthread_mutex_getspinloops_np(pthread_mutex_t *mutex, int *count);
  86 int     _pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count);
  87 int     __pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count);
  88 int     _pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count);
  89 int     _pthread_mutex_getyieldloops_np(pthread_mutex_t *mutex, int *count);
  90 int     __pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count);
  91
  92 static int      mutex_self_trylock(pthread_mutex_t);
  93 static int      mutex_self_lock(pthread_mutex_t,
  94                                 const struct timespec *abstime);
  95 static int      mutex_unlock_common(pthread_mutex_t *);
  96 static int      mutex_lock_sleep(struct pthread *, pthread_mutex_t,
  97                                 const struct timespec *);
  98
  99 __weak_reference(__pthread_mutex_init, pthread_mutex_init);
 100 __strong_reference(__pthread_mutex_init, _pthread_mutex_init);
 101 __weak_reference(__pthread_mutex_lock, pthread_mutex_lock);
 102 __strong_reference(__pthread_mutex_lock, _pthread_mutex_lock);
 103 __weak_reference(__pthread_mutex_timedlock, pthread_mutex_timedlock);
 104 __strong_reference(__pthread_mutex_timedlock, _pthread_mutex_timedlock);
 105 __weak_reference(__pthread_mutex_trylock, pthread_mutex_trylock);
 106 __strong_reference(__pthread_mutex_trylock, _pthread_mutex_trylock);
 107
 108 /* Single underscore versions provided for libc internal usage: */
 109 /* No difference between libc and application usage of these: */
 110 __weak_reference(_pthread_mutex_destroy, pthread_mutex_destroy);
 111 __weak_reference(_pthread_mutex_unlock, pthread_mutex_unlock);
 112
 113 __weak_reference(_pthread_mutex_getprioceiling, pthread_mutex_getprioceiling);
 114 __weak_reference(_pthread_mutex_setprioceiling, pthread_mutex_setprioceiling);
 115
 116 __weak_reference(__pthread_mutex_setspinloops_np, pthread_mutex_setspinloops_np);
 117 __strong_reference(__pthread_mutex_setspinloops_np, _pthread_mutex_setspinloops_np);
 118 __weak_reference(_pthread_mutex_getspinloops_np, pthread_mutex_getspinloops_np);
 119
 120 __weak_reference(__pthread_mutex_setyieldloops_np, pthread_mutex_setyieldloops_np);
 121 __strong_reference(__pthread_mutex_setyieldloops_np, _pthread_mutex_setyieldloops_np);
 122 __weak_reference(_pthread_mutex_getyieldloops_np, pthread_mutex_getyieldloops_np);
 123 __weak_reference(_pthread_mutex_isowned_np, pthread_mutex_isowned_np);
 124
 125 static int
 126 mutex_init(pthread_mutex_t *mutex,
 127     const struct pthread_mutex_attr *mutex_attr,
 128     void *(calloc_cb)(size_t, size_t))
 129 {
 130         const struct pthread_mutex_attr *attr;
 131         struct pthread_mutex *pmutex;
 132
 133         if (mutex_attr == NULL) {
 134                 attr = &_pthread_mutexattr_default;
 135         } else {
 136                 attr = mutex_attr;
 137                 if (attr->m_type < PTHREAD_MUTEX_ERRORCHECK ||
 138                     attr->m_type >= PTHREAD_MUTEX_TYPE_MAX)
 139                         return (EINVAL);
 140                 if (attr->m_protocol < PTHREAD_PRIO_NONE ||
 141                     attr->m_protocol > PTHREAD_PRIO_PROTECT)
 142                         return (EINVAL);
 143         }
 144         if ((pmutex = (pthread_mutex_t)
 145                 calloc_cb(1, sizeof(struct pthread_mutex))) == NULL)
 146                 return (ENOMEM);
 147
 148         pmutex->m_type = attr->m_type;
 149         pmutex->m_owner = NULL;
 150         pmutex->m_count = 0;
 151         pmutex->m_refcount = 0;
 152         pmutex->m_spinloops = 0;
 153         pmutex->m_yieldloops = 0;
 154         MUTEX_INIT_LINK(pmutex);
 155         switch(attr->m_protocol) {
 156         case PTHREAD_PRIO_NONE:
 157                 pmutex->m_lock.m_owner = UMUTEX_UNOWNED;
 158                 pmutex->m_lock.m_flags = 0;
 159                 break;
 160         case PTHREAD_PRIO_INHERIT:
 161                 pmutex->m_lock.m_owner = UMUTEX_UNOWNED;
 162                 pmutex->m_lock.m_flags = UMUTEX_PRIO_INHERIT;
 163                 break;
 164         case PTHREAD_PRIO_PROTECT:
 165                 pmutex->m_lock.m_owner = UMUTEX_CONTESTED;
 166                 pmutex->m_lock.m_flags = UMUTEX_PRIO_PROTECT;
 167                 pmutex->m_lock.m_ceilings[0] = attr->m_ceiling;
 168                 break;
 169         }
 170
 171         if (pmutex->m_type == PTHREAD_MUTEX_ADAPTIVE_NP) {
 172                 pmutex->m_spinloops =
 173                     _thr_spinloops ? _thr_spinloops: MUTEX_ADAPTIVE_SPINS;
 174                 pmutex->m_yieldloops = _thr_yieldloops;
 175         }
 176
 177         *mutex = pmutex;
 178         return (0);
 179 }
 180
 181 static int
 182 init_static(struct pthread *thread, pthread_mutex_t *mutex)
 183 {
 184         int ret;
 185
 186         THR_LOCK_ACQUIRE(thread, &_mutex_static_lock);
 187
 188         if (*mutex == THR_MUTEX_INITIALIZER)
 189                 ret = mutex_init(mutex, &_pthread_mutexattr_default, calloc);
 190         else if (*mutex == THR_ADAPTIVE_MUTEX_INITIALIZER)
 191                 ret = mutex_init(mutex, &_pthread_mutexattr_adaptive_default, calloc);
 192         else
 193                 ret = 0;
 194         THR_LOCK_RELEASE(thread, &_mutex_static_lock);
 195
 196         return (ret);
 197 }
 198
 199 static void
 200 set_inherited_priority(struct pthread *curthread, struct pthread_mutex *m)
 201 {
 202         struct pthread_mutex *m2;
 203
 204         m2 = TAILQ_LAST(&curthread->pp_mutexq, mutex_queue);
 205         if (m2 != NULL)
 206                 m->m_lock.m_ceilings[1] = m2->m_lock.m_ceilings[0];
 207         else
 208                 m->m_lock.m_ceilings[1] = -1;
 209 }
 210
 211 int
 212 __pthread_mutex_init(pthread_mutex_t *mutex,
 213     const pthread_mutexattr_t *mutex_attr)
 214 {
 215         return mutex_init(mutex, mutex_attr ? *mutex_attr : NULL, calloc);
 216 }
 217
 218 /* This function is used internally by malloc. */
 219 int
 220 _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
 221     void *(calloc_cb)(size_t, size_t))
 222 {
 223         static const struct pthread_mutex_attr attr = {
 224                 .m_type = PTHREAD_MUTEX_NORMAL,
 225                 .m_protocol = PTHREAD_PRIO_NONE,
 226                 .m_ceiling = 0
 227         };
 228
 229         return mutex_init(mutex, &attr, calloc_cb);
 230 }
 231
 232 void
 233 _mutex_fork(struct pthread *curthread)
 234 {
 235         struct pthread_mutex *m;
 236
 237         /*
 238          * Fix mutex ownership for child process.
 239          * note that process shared mutex should not
 240          * be inherited because owner is forking thread
 241          * which is in parent process, they should be
 242          * removed from the owned mutex list, current,
 243          * process shared mutex is not supported, so I
 244          * am not worried.
 245          */
 246
 247         TAILQ_FOREACH(m, &curthread->mutexq, m_qe)
 248                 m->m_lock.m_owner = TID(curthread);
 249         TAILQ_FOREACH(m, &curthread->pp_mutexq, m_qe)
 250                 m->m_lock.m_owner = TID(curthread) | UMUTEX_CONTESTED;
 251 }
 252
 253 int
 254 _pthread_mutex_destroy(pthread_mutex_t *mutex)
 255 {
 256         struct pthread *curthread = _get_curthread();
 257         pthread_mutex_t m;
 258         uint32_t id;
 259         int ret = 0;
 260
 261         m = *mutex;
 262         if (m < THR_MUTEX_DESTROYED) {
 263                 ret = 0;
 264         } else if (m == THR_MUTEX_DESTROYED) {
 265                 ret = EINVAL;
 266         } else {
 267                 id = TID(curthread);
 268
 269                 /*
 270                  * Try to lock the mutex structure, we only need to
 271                  * try once, if failed, the mutex is in used.
 272                  */
 273                 ret = _thr_umutex_trylock(&m->m_lock, id);
 274                 if (ret)
 275                         return (ret);
 276                 /*
 277                  * Check mutex other fields to see if this mutex is
 278                  * in use. Mostly for prority mutex types, or there
 279                  * are condition variables referencing it.
 280                  */
 281                 if (m->m_owner != NULL || m->m_refcount != 0) {
 282                         if (m->m_lock.m_flags & UMUTEX_PRIO_PROTECT)
 283                                 set_inherited_priority(curthread, m);
 284                         _thr_umutex_unlock(&m->m_lock, id);
 285                         ret = EBUSY;
 286                 } else {
 287                         *mutex = THR_MUTEX_DESTROYED;
 288
 289                         if (m->m_lock.m_flags & UMUTEX_PRIO_PROTECT)
 290                                 set_inherited_priority(curthread, m);
 291                         _thr_umutex_unlock(&m->m_lock, id);
 292
 293                         MUTEX_ASSERT_NOT_OWNED(m);
 294                         free(m);
 295                 }
 296         }
 297
 298         return (ret);
 299 }
 300
 301 #define ENQUEUE_MUTEX(curthread, m)                                     \
 302         do {                                                            \
 303                 (m)->m_owner = curthread;                               \
 304                 /* Add to the list of owned mutexes: */                 \
 305                 MUTEX_ASSERT_NOT_OWNED((m));                            \
 306                 if (((m)->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)   \
 307                         TAILQ_INSERT_TAIL(&curthread->mutexq, (m), m_qe);\
 308                 else                                                    \
 309                         TAILQ_INSERT_TAIL(&curthread->pp_mutexq, (m), m_qe);\
 310         } while (0)
 311
 312 #define CHECK_AND_INIT_MUTEX                                            \
 313         if (__predict_false((m = *mutex) <= THR_MUTEX_DESTROYED)) {     \
 314                 if (m == THR_MUTEX_DESTROYED)                           \
 315                         return (EINVAL);                                \
 316                 int ret;                                                \
 317                 ret = init_static(_get_curthread(), mutex);             \
 318                 if (ret)                                                \
 319                         return (ret);                                   \
 320                 m = *mutex;                                             \
 321         }
 322
 323 static int
 324 mutex_trylock_common(pthread_mutex_t *mutex)
 325 {
 326         struct pthread *curthread = _get_curthread();
 327         struct pthread_mutex *m = *mutex;
 328         uint32_t id;
 329         int ret;
 330
 331         id = TID(curthread);
 332         ret = _thr_umutex_trylock(&m->m_lock, id);
 333         if (__predict_true(ret == 0)) {
 334                 ENQUEUE_MUTEX(curthread, m);
 335         } else if (m->m_owner == curthread) {
 336                 ret = mutex_self_trylock(m);
 337         } /* else {} */
 338
 339         return (ret);
 340 }
 341
 342 int
 343 __pthread_mutex_trylock(pthread_mutex_t *mutex)
 344 {
 345         struct pthread_mutex *m;
 346
 347         CHECK_AND_INIT_MUTEX
 348
 349         return (mutex_trylock_common(mutex));
 350 }
 351
 352 static int
 353 mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m,
 354         const struct timespec *abstime)
 355 {
 356         uint32_t        id, owner;
 357         int     count;
 358         int     ret;
 359
 360         if (m->m_owner == curthread)
 361                 return mutex_self_lock(m, abstime);
 362
 363         id = TID(curthread);
 364         /*
 365          * For adaptive mutexes, spin for a bit in the expectation
 366          * that if the application requests this mutex type then
 367          * the lock is likely to be released quickly and it is
 368          * faster than entering the kernel
 369          */
 370         if (__predict_false(
 371                 (m->m_lock.m_flags &
 372                  (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) != 0))
 373                         goto sleep_in_kernel;
 374
 375         if (!_thr_is_smp)
 376                 goto yield_loop;
 377
 378         count = m->m_spinloops;
 379         while (count--) {
 380                 owner = m->m_lock.m_owner;
 381                 if ((owner & ~UMUTEX_CONTESTED) == 0) {
 382                         if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner, id|owner)) {
 383                                 ret = 0;
 384                                 goto done;
 385                         }
 386                 }
 387                 CPU_SPINWAIT;
 388         }
 389
 390 yield_loop:
 391         count = m->m_yieldloops;
 392         while (count--) {
 393                 _sched_yield();
 394                 owner = m->m_lock.m_owner;
 395                 if ((owner & ~UMUTEX_CONTESTED) == 0) {
 396                         if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner, id|owner)) {
 397                                 ret = 0;
 398                                 goto done;
 399                         }
 400                 }
 401         }
 402
 403 sleep_in_kernel:
 404         if (abstime == NULL) {
 405                 ret = __thr_umutex_lock(&m->m_lock, id);
 406         } else if (__predict_false(
 407                    abstime->tv_nsec < 0 ||
 408                    abstime->tv_nsec >= 1000000000)) {
 409                 ret = EINVAL;
 410         } else {
 411                 ret = __thr_umutex_timedlock(&m->m_lock, id, abstime);
 412         }
 413 done:
 414         if (ret == 0)
 415                 ENQUEUE_MUTEX(curthread, m);
 416
 417         return (ret);
 418 }
 419
 420 static inline int
 421 mutex_lock_common(struct pthread_mutex *m,
 422         const struct timespec *abstime)
 423 {
 424         struct pthread *curthread  = _get_curthread();
 425
 426         if (_thr_umutex_trylock2(&m->m_lock, TID(curthread)) == 0) {
 427                 ENQUEUE_MUTEX(curthread, m);
 428                 return (0);
 429         }
 430
 431         return (mutex_lock_sleep(curthread, m, abstime));
 432 }
 433
 434 int
 435 __pthread_mutex_lock(pthread_mutex_t *mutex)
 436 {
 437         struct pthread_mutex    *m;
 438
 439         _thr_check_init();
 440
 441         CHECK_AND_INIT_MUTEX
 442
 443         return (mutex_lock_common(m, NULL));
 444 }
 445
 446 int
 447 __pthread_mutex_timedlock(pthread_mutex_t *mutex, const struct timespec *abstime)
 448 {
 449         struct pthread_mutex    *m;
 450
 451         _thr_check_init();
 452
 453         CHECK_AND_INIT_MUTEX
 454
 455         return (mutex_lock_common(m, abstime));
 456 }
 457
 458 int
 459 _pthread_mutex_unlock(pthread_mutex_t *m)
 460 {
 461         return (mutex_unlock_common(m));
 462 }
 463
 464 int
 465 _mutex_cv_lock(pthread_mutex_t *mutex, int count)
 466 {
 467         struct pthread_mutex    *m;
 468         int     ret;
 469
 470         m = *mutex;
 471         ret = mutex_lock_common(m, NULL);
 472         if (ret == 0) {
 473                 m->m_refcount--;
 474                 m->m_count += count;
 475         }
 476         return (ret);
 477 }
 478
 479 static int
 480 mutex_self_trylock(struct pthread_mutex *m)
 481 {
 482         int     ret;
 483
 484         switch (m->m_type) {
 485         case PTHREAD_MUTEX_ERRORCHECK:
 486         case PTHREAD_MUTEX_NORMAL:
 487         case PTHREAD_MUTEX_ADAPTIVE_NP:
 488                 ret = EBUSY;
 489                 break;
 490
 491         case PTHREAD_MUTEX_RECURSIVE:
 492                 /* Increment the lock count: */
 493                 if (m->m_count + 1 > 0) {
 494                         m->m_count++;
 495                         ret = 0;
 496                 } else
 497                         ret = EAGAIN;
 498                 break;
 499
 500         default:
 501                 /* Trap invalid mutex types; */
 502                 ret = EINVAL;
 503         }
 504
 505         return (ret);
 506 }
 507
 508 static int
 509 mutex_self_lock(struct pthread_mutex *m, const struct timespec *abstime)
 510 {
 511         struct timespec ts1, ts2;
 512         int     ret;
 513
 514         switch (m->m_type) {
 515         case PTHREAD_MUTEX_ERRORCHECK:
 516         case PTHREAD_MUTEX_ADAPTIVE_NP:
 517                 if (abstime) {
 518                         if (abstime->tv_sec < 0 || abstime->tv_nsec < 0 ||
 519                             abstime->tv_nsec >= 1000000000) {
 520                                 ret = EINVAL;
 521                         } else {
 522                                 clock_gettime(CLOCK_REALTIME, &ts1);
 523                                 TIMESPEC_SUB(&ts2, abstime, &ts1);
 524                                 __sys_nanosleep(&ts2, NULL);
 525                                 ret = ETIMEDOUT;
 526                         }
 527                 } else {
 528                         /*
 529                          * POSIX specifies that mutexes should return
 530                          * EDEADLK if a recursive lock is detected.
 531                          */
 532                         ret = EDEADLK;
 533                 }
 534                 break;
 535
 536         case PTHREAD_MUTEX_NORMAL:
 537                 /*
 538                  * What SS2 define as a 'normal' mutex.  Intentionally
 539                  * deadlock on attempts to get a lock you already own.
 540                  */
 541                 ret = 0;
 542                 if (abstime) {
 543                         if (abstime->tv_sec < 0 || abstime->tv_nsec < 0 ||
 544                             abstime->tv_nsec >= 1000000000) {
 545                                 ret = EINVAL;
 546                         } else {
 547                                 clock_gettime(CLOCK_REALTIME, &ts1);
 548                                 TIMESPEC_SUB(&ts2, abstime, &ts1);
 549                                 __sys_nanosleep(&ts2, NULL);
 550                                 ret = ETIMEDOUT;
 551                         }
 552                 } else {
 553                         ts1.tv_sec = 30;
 554                         ts1.tv_nsec = 0;
 555                         for (;;)
 556                                 __sys_nanosleep(&ts1, NULL);
 557                 }
 558                 break;
 559
 560         case PTHREAD_MUTEX_RECURSIVE:
 561                 /* Increment the lock count: */
 562                 if (m->m_count + 1 > 0) {
 563                         m->m_count++;
 564                         ret = 0;
 565                 } else
 566                         ret = EAGAIN;
 567                 break;
 568
 569         default:
 570                 /* Trap invalid mutex types; */
 571                 ret = EINVAL;
 572         }
 573
 574         return (ret);
 575 }
 576
 577 static int
 578 mutex_unlock_common(pthread_mutex_t *mutex)
 579 {
 580         struct pthread *curthread = _get_curthread();
 581         struct pthread_mutex *m;
 582         uint32_t id;
 583
 584         m = *mutex;
 585         if (__predict_false(m <= THR_MUTEX_DESTROYED)) {
 586                 if (m == THR_MUTEX_DESTROYED)
 587                         return (EINVAL);
 588                 return (EPERM);
 589         }
 590
 591         /*
 592          * Check if the running thread is not the owner of the mutex.
 593          */
 594         if (__predict_false(m->m_owner != curthread))
 595                 return (EPERM);
 596
 597         id = TID(curthread);
 598         if (__predict_false(
 599                 m->m_type == PTHREAD_MUTEX_RECURSIVE &&
 600                 m->m_count > 0)) {
 601                 m->m_count--;
 602         } else {
 603                 m->m_owner = NULL;
 604                 /* Remove the mutex from the threads queue. */
 605                 MUTEX_ASSERT_IS_OWNED(m);
 606                 if (__predict_true((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0))
 607                         TAILQ_REMOVE(&curthread->mutexq, m, m_qe);
 608                 else {
 609                         TAILQ_REMOVE(&curthread->pp_mutexq, m, m_qe);
 610                         set_inherited_priority(curthread, m);
 611                 }
 612                 MUTEX_INIT_LINK(m);
 613                 _thr_umutex_unlock(&m->m_lock, id);
 614         }
 615         return (0);
 616 }
 617
 618 int
 619 _mutex_cv_unlock(pthread_mutex_t *mutex, int *count)
 620 {
 621         struct pthread *curthread = _get_curthread();
 622         struct pthread_mutex *m;
 623
 624         m = *mutex;
 625         if (__predict_false(m <= THR_MUTEX_DESTROYED)) {
 626                 if (m == THR_MUTEX_DESTROYED)
 627                         return (EINVAL);
 628                 return (EPERM);
 629         }
 630
 631         /*
 632          * Check if the running thread is not the owner of the mutex.
 633          */
 634         if (__predict_false(m->m_owner != curthread))
 635                 return (EPERM);
 636
 637         /*
 638          * Clear the count in case this is a recursive mutex.
 639          */
 640         *count = m->m_count;
 641         m->m_refcount++;
 642         m->m_count = 0;
 643         m->m_owner = NULL;
 644         /* Remove the mutex from the threads queue. */
 645         MUTEX_ASSERT_IS_OWNED(m);
 646         if (__predict_true((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0))
 647                 TAILQ_REMOVE(&curthread->mutexq, m, m_qe);
 648         else {
 649                 TAILQ_REMOVE(&curthread->pp_mutexq, m, m_qe);
 650                 set_inherited_priority(curthread, m);
 651         }
 652         MUTEX_INIT_LINK(m);
 653         _thr_umutex_unlock(&m->m_lock, TID(curthread));
 654         return (0);
 655 }
 656
 657 int
 658 _pthread_mutex_getprioceiling(pthread_mutex_t *mutex,
 659                               int *prioceiling)
 660 {
 661         struct pthread_mutex *m;
 662         int ret;
 663
 664         m = *mutex;
 665         if ((m <= THR_MUTEX_DESTROYED) ||
 666             (m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)
 667                 ret = EINVAL;
 668         else {
 669                 *prioceiling = m->m_lock.m_ceilings[0];
 670                 ret = 0;
 671         }
 672
 673         return (ret);
 674 }
 675
 676 int
 677 _pthread_mutex_setprioceiling(pthread_mutex_t *mutex,
 678                               int ceiling, int *old_ceiling)
 679 {
 680         struct pthread *curthread = _get_curthread();
 681         struct pthread_mutex *m, *m1, *m2;
 682         int ret;
 683
 684         m = *mutex;
 685         if ((m <= THR_MUTEX_DESTROYED) ||
 686             (m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)
 687                 return (EINVAL);
 688
 689         ret = __thr_umutex_set_ceiling(&m->m_lock, ceiling, old_ceiling);
 690         if (ret != 0)
 691                 return (ret);
 692
 693         if (m->m_owner == curthread) {
 694                 MUTEX_ASSERT_IS_OWNED(m);
 695                 m1 = TAILQ_PREV(m, mutex_queue, m_qe);
 696                 m2 = TAILQ_NEXT(m, m_qe);
 697                 if ((m1 != NULL && m1->m_lock.m_ceilings[0] > (u_int)ceiling) ||
 698                     (m2 != NULL && m2->m_lock.m_ceilings[0] < (u_int)ceiling)) {
 699                         TAILQ_REMOVE(&curthread->pp_mutexq, m, m_qe);
 700                         TAILQ_FOREACH(m2, &curthread->pp_mutexq, m_qe) {
 701                                 if (m2->m_lock.m_ceilings[0] > (u_int)ceiling) {
 702                                         TAILQ_INSERT_BEFORE(m2, m, m_qe);
 703                                         return (0);
 704                                 }
 705                         }
 706                         TAILQ_INSERT_TAIL(&curthread->pp_mutexq, m, m_qe);
 707                 }
 708         }
 709         return (0);
 710 }
 711
 712 int
 713 _pthread_mutex_getspinloops_np(pthread_mutex_t *mutex, int *count)
 714 {
 715         struct pthread_mutex    *m;
 716
 717         CHECK_AND_INIT_MUTEX
 718
 719         *count = m->m_spinloops;
 720         return (0);
 721 }
 722
 723 int
 724 __pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count)
 725 {
 726         struct pthread_mutex    *m;
 727
 728         CHECK_AND_INIT_MUTEX
 729
 730         m->m_spinloops = count;
 731         return (0);
 732 }
 733
 734 int
 735 _pthread_mutex_getyieldloops_np(pthread_mutex_t *mutex, int *count)
 736 {
 737         struct pthread_mutex    *m;
 738
 739         CHECK_AND_INIT_MUTEX
 740
 741         *count = m->m_yieldloops;
 742         return (0);
 743 }
 744
 745 int
 746 __pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count)
 747 {
 748         struct pthread_mutex    *m;
 749
 750         CHECK_AND_INIT_MUTEX
 751
 752         m->m_yieldloops = count;
 753         return (0);
 754 }
 755
 756 int
 757 _pthread_mutex_isowned_np(pthread_mutex_t *mutex)
 758 {
 759         struct pthread_mutex    *m;
 760
 761         m = *mutex;
 762         if (m <= THR_MUTEX_DESTROYED)
 763                 return (0);
 764         return (m->m_owner == _get_curthread());
 765 }