lib/libthr/thread/thr_mutex.c

   1 /*
   2  * Copyright (c) 1995 John Birrell <jb@cimlogic.com.au>.
   3  * Copyright (c) 2006 David Xu <davidxu@freebsd.org>.
   4  * Copyright (c) 2015 The FreeBSD Foundation
   5  *
   6  * All rights reserved.
   7  *
   8  * Portions of this software were developed by Konstantin Belousov
   9  * under sponsorship from the FreeBSD Foundation.
  10  *
  11  * Redistribution and use in source and binary forms, with or without
  12  * modification, are permitted provided that the following conditions
  13  * are met:
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in the
  18  *    documentation and/or other materials provided with the distribution.
  19  * 3. All advertising materials mentioning features or use of this software
  20  *    must display the following acknowledgement:
  21  *      This product includes software developed by John Birrell.
  22  * 4. Neither the name of the author nor the names of any co-contributors
  23  *    may be used to endorse or promote products derived from this software
  24  *    without specific prior written permission.
  25  *
  26  * THIS SOFTWARE IS PROVIDED BY JOHN BIRRELL AND CONTRIBUTORS ``AS IS'' AND
  27  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  28  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  29  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  30  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  31  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  32  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  33  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  34  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  35  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  36  * SUCH DAMAGE.
  37  *
  38  * $FreeBSD$
  39  */
  40
  41 #include <stdbool.h>
  42 #include "namespace.h"
  43 #include <stdlib.h>
  44 #include <errno.h>
  45 #include <string.h>
  46 #include <sys/param.h>
  47 #include <sys/queue.h>
  48 #include <pthread.h>
  49 #include <pthread_np.h>
  50 #include "un-namespace.h"
  51
  52 #include "thr_private.h"
  53
  54 /*
  55  * For adaptive mutexes, how many times to spin doing trylock2
  56  * before entering the kernel to block
  57  */
  58 #define MUTEX_ADAPTIVE_SPINS    2000
  59
  60 /*
  61  * Prototypes
  62  */
  63 int     __pthread_mutex_init(pthread_mutex_t *mutex,
  64                 const pthread_mutexattr_t *mutex_attr);
  65 int     __pthread_mutex_trylock(pthread_mutex_t *mutex);
  66 int     __pthread_mutex_lock(pthread_mutex_t *mutex);
  67 int     __pthread_mutex_timedlock(pthread_mutex_t *mutex,
  68                 const struct timespec *abstime);
  69 int     _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
  70                 void *(calloc_cb)(size_t, size_t));
  71 int     _pthread_mutex_getspinloops_np(pthread_mutex_t *mutex, int *count);
  72 int     _pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count);
  73 int     __pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count);
  74 int     _pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count);
  75 int     _pthread_mutex_getyieldloops_np(pthread_mutex_t *mutex, int *count);
  76 int     __pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count);
  77
  78 static int      mutex_self_trylock(pthread_mutex_t);
  79 static int      mutex_self_lock(pthread_mutex_t,
  80                                 const struct timespec *abstime);
  81 static int      mutex_unlock_common(struct pthread_mutex *, int, int *);
  82 static int      mutex_lock_sleep(struct pthread *, pthread_mutex_t,
  83                                 const struct timespec *);
  84
  85 __weak_reference(__pthread_mutex_init, pthread_mutex_init);
  86 __strong_reference(__pthread_mutex_init, _pthread_mutex_init);
  87 __weak_reference(__pthread_mutex_lock, pthread_mutex_lock);
  88 __strong_reference(__pthread_mutex_lock, _pthread_mutex_lock);
  89 __weak_reference(__pthread_mutex_timedlock, pthread_mutex_timedlock);
  90 __strong_reference(__pthread_mutex_timedlock, _pthread_mutex_timedlock);
  91 __weak_reference(__pthread_mutex_trylock, pthread_mutex_trylock);
  92 __strong_reference(__pthread_mutex_trylock, _pthread_mutex_trylock);
  93
  94 /* Single underscore versions provided for libc internal usage: */
  95 /* No difference between libc and application usage of these: */
  96 __weak_reference(_pthread_mutex_destroy, pthread_mutex_destroy);
  97 __weak_reference(_pthread_mutex_unlock, pthread_mutex_unlock);
  98
  99 __weak_reference(_pthread_mutex_getprioceiling, pthread_mutex_getprioceiling);
 100 __weak_reference(_pthread_mutex_setprioceiling, pthread_mutex_setprioceiling);
 101
 102 __weak_reference(__pthread_mutex_setspinloops_np, pthread_mutex_setspinloops_np);
 103 __strong_reference(__pthread_mutex_setspinloops_np, _pthread_mutex_setspinloops_np);
 104 __weak_reference(_pthread_mutex_getspinloops_np, pthread_mutex_getspinloops_np);
 105
 106 __weak_reference(__pthread_mutex_setyieldloops_np, pthread_mutex_setyieldloops_np);
 107 __strong_reference(__pthread_mutex_setyieldloops_np, _pthread_mutex_setyieldloops_np);
 108 __weak_reference(_pthread_mutex_getyieldloops_np, pthread_mutex_getyieldloops_np);
 109 __weak_reference(_pthread_mutex_isowned_np, pthread_mutex_isowned_np);
 110
 111 static void
 112 mutex_init_link(struct pthread_mutex *m)
 113 {
 114
 115 #if defined(_PTHREADS_INVARIANTS)
 116         m->m_qe.tqe_prev = NULL;
 117         m->m_qe.tqe_next = NULL;
 118         m->m_pqe.tqe_prev = NULL;
 119         m->m_pqe.tqe_next = NULL;
 120 #endif
 121 }
 122
 123 static void
 124 mutex_assert_is_owned(struct pthread_mutex *m)
 125 {
 126
 127 #if defined(_PTHREADS_INVARIANTS)
 128         if (__predict_false(m->m_qe.tqe_prev == NULL)) {
 129                 char msg[128];
 130                 snprintf(msg, sizeof(msg),
 131                     "mutex %p own %#x %#x is not on list %p %p",
 132                     m, m->m_lock.m_owner, m->m_owner, m->m_qe.tqe_prev,
 133                     m->m_qe.tqe_next);
 134                 PANIC(msg);
 135         }
 136 #endif
 137 }
 138
 139 static void
 140 mutex_assert_not_owned(struct pthread_mutex *m)
 141 {
 142
 143 #if defined(_PTHREADS_INVARIANTS)
 144         if (__predict_false(m->m_qe.tqe_prev != NULL ||
 145             m->m_qe.tqe_next != NULL)) {
 146                 char msg[128];
 147                 snprintf(msg, sizeof(msg),
 148                     "mutex %p own %#x %#x is on list %p %p",
 149                     m, m->m_lock.m_owner, m->m_owner, m->m_qe.tqe_prev,
 150                     m->m_qe.tqe_next);
 151                 PANIC(msg);
 152         }
 153 #endif
 154 }
 155
 156 static int
 157 is_pshared_mutex(struct pthread_mutex *m)
 158 {
 159
 160         return ((m->m_lock.m_flags & USYNC_PROCESS_SHARED) != 0);
 161 }
 162
 163 static int
 164 mutex_check_attr(const struct pthread_mutex_attr *attr)
 165 {
 166
 167         if (attr->m_type < PTHREAD_MUTEX_ERRORCHECK ||
 168             attr->m_type >= PTHREAD_MUTEX_TYPE_MAX)
 169                 return (EINVAL);
 170         if (attr->m_protocol < PTHREAD_PRIO_NONE ||
 171             attr->m_protocol > PTHREAD_PRIO_PROTECT)
 172                 return (EINVAL);
 173         return (0);
 174 }
 175
 176 static void
 177 mutex_init_body(struct pthread_mutex *pmutex,
 178     const struct pthread_mutex_attr *attr)
 179 {
 180
 181         pmutex->m_flags = attr->m_type;
 182         pmutex->m_owner = 0;
 183         pmutex->m_count = 0;
 184         pmutex->m_spinloops = 0;
 185         pmutex->m_yieldloops = 0;
 186         mutex_init_link(pmutex);
 187         switch (attr->m_protocol) {
 188         case PTHREAD_PRIO_NONE:
 189                 pmutex->m_lock.m_owner = UMUTEX_UNOWNED;
 190                 pmutex->m_lock.m_flags = 0;
 191                 break;
 192         case PTHREAD_PRIO_INHERIT:
 193                 pmutex->m_lock.m_owner = UMUTEX_UNOWNED;
 194                 pmutex->m_lock.m_flags = UMUTEX_PRIO_INHERIT;
 195                 break;
 196         case PTHREAD_PRIO_PROTECT:
 197                 pmutex->m_lock.m_owner = UMUTEX_CONTESTED;
 198                 pmutex->m_lock.m_flags = UMUTEX_PRIO_PROTECT;
 199                 pmutex->m_lock.m_ceilings[0] = attr->m_ceiling;
 200                 break;
 201         }
 202         if (attr->m_pshared == PTHREAD_PROCESS_SHARED)
 203                 pmutex->m_lock.m_flags |= USYNC_PROCESS_SHARED;
 204
 205         if (PMUTEX_TYPE(pmutex->m_flags) == PTHREAD_MUTEX_ADAPTIVE_NP) {
 206                 pmutex->m_spinloops =
 207                     _thr_spinloops ? _thr_spinloops: MUTEX_ADAPTIVE_SPINS;
 208                 pmutex->m_yieldloops = _thr_yieldloops;
 209         }
 210 }
 211
 212 static int
 213 mutex_init(pthread_mutex_t *mutex,
 214     const struct pthread_mutex_attr *mutex_attr,
 215     void *(calloc_cb)(size_t, size_t))
 216 {
 217         const struct pthread_mutex_attr *attr;
 218         struct pthread_mutex *pmutex;
 219         int error;
 220
 221         if (mutex_attr == NULL) {
 222                 attr = &_pthread_mutexattr_default;
 223         } else {
 224                 attr = mutex_attr;
 225                 error = mutex_check_attr(attr);
 226                 if (error != 0)
 227                         return (error);
 228         }
 229         if ((pmutex = (pthread_mutex_t)
 230                 calloc_cb(1, sizeof(struct pthread_mutex))) == NULL)
 231                 return (ENOMEM);
 232         mutex_init_body(pmutex, attr);
 233         *mutex = pmutex;
 234         return (0);
 235 }
 236
 237 static int
 238 init_static(struct pthread *thread, pthread_mutex_t *mutex)
 239 {
 240         int ret;
 241
 242         THR_LOCK_ACQUIRE(thread, &_mutex_static_lock);
 243
 244         if (*mutex == THR_MUTEX_INITIALIZER)
 245                 ret = mutex_init(mutex, &_pthread_mutexattr_default, calloc);
 246         else if (*mutex == THR_ADAPTIVE_MUTEX_INITIALIZER)
 247                 ret = mutex_init(mutex, &_pthread_mutexattr_adaptive_default,
 248                     calloc);
 249         else
 250                 ret = 0;
 251         THR_LOCK_RELEASE(thread, &_mutex_static_lock);
 252
 253         return (ret);
 254 }
 255
 256 static void
 257 set_inherited_priority(struct pthread *curthread, struct pthread_mutex *m)
 258 {
 259         struct pthread_mutex *m2;
 260
 261         m2 = TAILQ_LAST(&curthread->mq[TMQ_NORM_PP], mutex_queue);
 262         if (m2 != NULL)
 263                 m->m_lock.m_ceilings[1] = m2->m_lock.m_ceilings[0];
 264         else
 265                 m->m_lock.m_ceilings[1] = -1;
 266 }
 267
 268 static void
 269 shared_mutex_init(struct pthread_mutex *pmtx, const struct
 270     pthread_mutex_attr *mutex_attr)
 271 {
 272         static const struct pthread_mutex_attr foobar_mutex_attr = {
 273                 .m_type = PTHREAD_MUTEX_DEFAULT,
 274                 .m_protocol = PTHREAD_PRIO_NONE,
 275                 .m_ceiling = 0,
 276                 .m_pshared = PTHREAD_PROCESS_SHARED
 277         };
 278         bool done;
 279
 280         /*
 281          * Hack to allow multiple pthread_mutex_init() calls on the
 282          * same process-shared mutex.  We rely on kernel allocating
 283          * zeroed offpage for the mutex, i.e. the
 284          * PMUTEX_INITSTAGE_ALLOC value must be zero.
 285          */
 286         for (done = false; !done;) {
 287                 switch (pmtx->m_ps) {
 288                 case PMUTEX_INITSTAGE_DONE:
 289                         atomic_thread_fence_acq();
 290                         done = true;
 291                         break;
 292                 case PMUTEX_INITSTAGE_ALLOC:
 293                         if (atomic_cmpset_int(&pmtx->m_ps,
 294                             PMUTEX_INITSTAGE_ALLOC, PMUTEX_INITSTAGE_BUSY)) {
 295                                 if (mutex_attr == NULL)
 296                                         mutex_attr = &foobar_mutex_attr;
 297                                 mutex_init_body(pmtx, mutex_attr);
 298                                 atomic_store_rel_int(&pmtx->m_ps,
 299                                     PMUTEX_INITSTAGE_DONE);
 300                                 done = true;
 301                         }
 302                         break;
 303                 case PMUTEX_INITSTAGE_BUSY:
 304                         _pthread_yield();
 305                         break;
 306                 default:
 307                         PANIC("corrupted offpage");
 308                         break;
 309                 }
 310         }
 311 }
 312
 313 int
 314 __pthread_mutex_init(pthread_mutex_t *mutex,
 315     const pthread_mutexattr_t *mutex_attr)
 316 {
 317         struct pthread_mutex *pmtx;
 318         int ret;
 319
 320         if (mutex_attr != NULL) {
 321                 ret = mutex_check_attr(*mutex_attr);
 322                 if (ret != 0)
 323                         return (ret);
 324         }
 325         if (mutex_attr == NULL ||
 326             (*mutex_attr)->m_pshared == PTHREAD_PROCESS_PRIVATE) {
 327                 return (mutex_init(mutex, mutex_attr ? *mutex_attr : NULL,
 328                    calloc));
 329         }
 330         pmtx = __thr_pshared_offpage(mutex, 1);
 331         if (pmtx == NULL)
 332                 return (EFAULT);
 333         *mutex = THR_PSHARED_PTR;
 334         shared_mutex_init(pmtx, *mutex_attr);
 335         return (0);
 336 }
 337
 338 /* This function is used internally by malloc. */
 339 int
 340 _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
 341     void *(calloc_cb)(size_t, size_t))
 342 {
 343         static const struct pthread_mutex_attr attr = {
 344                 .m_type = PTHREAD_MUTEX_NORMAL,
 345                 .m_protocol = PTHREAD_PRIO_NONE,
 346                 .m_ceiling = 0,
 347                 .m_pshared = PTHREAD_PROCESS_PRIVATE,
 348         };
 349         int ret;
 350
 351         ret = mutex_init(mutex, &attr, calloc_cb);
 352         if (ret == 0)
 353                 (*mutex)->m_flags |= PMUTEX_FLAG_PRIVATE;
 354         return (ret);
 355 }
 356
 357 /*
 358  * Fix mutex ownership for child process.
 359  *
 360  * Process private mutex ownership is transmitted from the forking
 361  * thread to the child process.
 362  *
 363  * Process shared mutex should not be inherited because owner is
 364  * forking thread which is in parent process, they are removed from
 365  * the owned mutex list.
 366  */
 367 static void
 368 queue_fork(struct pthread *curthread, struct mutex_queue *q,
 369     struct mutex_queue *qp, uint bit)
 370 {
 371         struct pthread_mutex *m;
 372
 373         TAILQ_INIT(q);
 374         TAILQ_FOREACH(m, qp, m_pqe) {
 375                 TAILQ_INSERT_TAIL(q, m, m_qe);
 376                 m->m_lock.m_owner = TID(curthread) | bit;
 377                 m->m_owner = TID(curthread);
 378         }
 379 }
 380
 381 void
 382 _mutex_fork(struct pthread *curthread)
 383 {
 384
 385         queue_fork(curthread, &curthread->mq[TMQ_NORM],
 386             &curthread->mq[TMQ_NORM_PRIV], 0);
 387         queue_fork(curthread, &curthread->mq[TMQ_NORM_PP],
 388             &curthread->mq[TMQ_NORM_PP_PRIV], UMUTEX_CONTESTED);
 389 }
 390
 391 int
 392 _pthread_mutex_destroy(pthread_mutex_t *mutex)
 393 {
 394         pthread_mutex_t m, m1;
 395         int ret;
 396
 397         m = *mutex;
 398         if (m < THR_MUTEX_DESTROYED) {
 399                 ret = 0;
 400         } else if (m == THR_MUTEX_DESTROYED) {
 401                 ret = EINVAL;
 402         } else {
 403                 if (m == THR_PSHARED_PTR) {
 404                         m1 = __thr_pshared_offpage(mutex, 0);
 405                         if (m1 != NULL) {
 406                                 mutex_assert_not_owned(m1);
 407                                 __thr_pshared_destroy(mutex);
 408                         }
 409                         *mutex = THR_MUTEX_DESTROYED;
 410                         return (0);
 411                 }
 412                 if (m->m_owner != 0) {
 413                         ret = EBUSY;
 414                 } else {
 415                         *mutex = THR_MUTEX_DESTROYED;
 416                         mutex_assert_not_owned(m);
 417                         free(m);
 418                         ret = 0;
 419                 }
 420         }
 421
 422         return (ret);
 423 }
 424
 425 static int
 426 mutex_qidx(struct pthread_mutex *m)
 427 {
 428
 429         if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)
 430                 return (TMQ_NORM);
 431         return (TMQ_NORM_PP);
 432 }
 433
 434 static void
 435 enqueue_mutex(struct pthread *curthread, struct pthread_mutex *m)
 436 {
 437         int qidx;
 438
 439         m->m_owner = TID(curthread);
 440         /* Add to the list of owned mutexes: */
 441         mutex_assert_not_owned(m);
 442         qidx = mutex_qidx(m);
 443         TAILQ_INSERT_TAIL(&curthread->mq[qidx], m, m_qe);
 444         if (!is_pshared_mutex(m))
 445                 TAILQ_INSERT_TAIL(&curthread->mq[qidx + 1], m, m_pqe);
 446 }
 447
 448 static void
 449 dequeue_mutex(struct pthread *curthread, struct pthread_mutex *m)
 450 {
 451         int qidx;
 452
 453         m->m_owner = 0;
 454         mutex_assert_is_owned(m);
 455         qidx = mutex_qidx(m);
 456         TAILQ_REMOVE(&curthread->mq[qidx], m, m_qe);
 457         if (!is_pshared_mutex(m))
 458                 TAILQ_REMOVE(&curthread->mq[qidx + 1], m, m_pqe);
 459         if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) != 0)
 460                 set_inherited_priority(curthread, m);
 461         mutex_init_link(m);
 462 }
 463
 464 static int
 465 check_and_init_mutex(pthread_mutex_t *mutex, struct pthread_mutex **m)
 466 {
 467         int ret;
 468
 469         *m = *mutex;
 470         ret = 0;
 471         if (*m == THR_PSHARED_PTR) {
 472                 *m = __thr_pshared_offpage(mutex, 0);
 473                 if (*m == NULL)
 474                         ret = EINVAL;
 475                 shared_mutex_init(*m, NULL);
 476         } else if (__predict_false(*m <= THR_MUTEX_DESTROYED)) {
 477                 if (*m == THR_MUTEX_DESTROYED) {
 478                         ret = EINVAL;
 479                 } else {
 480                         ret = init_static(_get_curthread(), mutex);
 481                         if (ret == 0)
 482                                 *m = *mutex;
 483                 }
 484         }
 485         return (ret);
 486 }
 487
 488 int
 489 __pthread_mutex_trylock(pthread_mutex_t *mutex)
 490 {
 491         struct pthread *curthread;
 492         struct pthread_mutex *m;
 493         uint32_t id;
 494         int ret;
 495
 496         ret = check_and_init_mutex(mutex, &m);
 497         if (ret != 0)
 498                 return (ret);
 499         curthread = _get_curthread();
 500         id = TID(curthread);
 501         if (m->m_flags & PMUTEX_FLAG_PRIVATE)
 502                 THR_CRITICAL_ENTER(curthread);
 503         ret = _thr_umutex_trylock(&m->m_lock, id);
 504         if (__predict_true(ret == 0)) {
 505                 enqueue_mutex(curthread, m);
 506         } else if (m->m_owner == id) {
 507                 ret = mutex_self_trylock(m);
 508         } /* else {} */
 509         if (ret && (m->m_flags & PMUTEX_FLAG_PRIVATE))
 510                 THR_CRITICAL_LEAVE(curthread);
 511         return (ret);
 512 }
 513
 514 static int
 515 mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m,
 516         const struct timespec *abstime)
 517 {
 518         uint32_t        id, owner;
 519         int     count;
 520         int     ret;
 521
 522         id = TID(curthread);
 523         if (m->m_owner == id)
 524                 return (mutex_self_lock(m, abstime));
 525
 526         /*
 527          * For adaptive mutexes, spin for a bit in the expectation
 528          * that if the application requests this mutex type then
 529          * the lock is likely to be released quickly and it is
 530          * faster than entering the kernel
 531          */
 532         if (__predict_false(
 533                 (m->m_lock.m_flags &
 534                  (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) != 0))
 535                         goto sleep_in_kernel;
 536
 537         if (!_thr_is_smp)
 538                 goto yield_loop;
 539
 540         count = m->m_spinloops;
 541         while (count--) {
 542                 owner = m->m_lock.m_owner;
 543                 if ((owner & ~UMUTEX_CONTESTED) == 0) {
 544                         if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner, id|owner)) {
 545                                 ret = 0;
 546                                 goto done;
 547                         }
 548                 }
 549                 CPU_SPINWAIT;
 550         }
 551
 552 yield_loop:
 553         count = m->m_yieldloops;
 554         while (count--) {
 555                 _sched_yield();
 556                 owner = m->m_lock.m_owner;
 557                 if ((owner & ~UMUTEX_CONTESTED) == 0) {
 558                         if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner, id|owner)) {
 559                                 ret = 0;
 560                                 goto done;
 561                         }
 562                 }
 563         }
 564
 565 sleep_in_kernel:
 566         if (abstime == NULL) {
 567                 ret = __thr_umutex_lock(&m->m_lock, id);
 568         } else if (__predict_false(
 569                    abstime->tv_nsec < 0 ||
 570                    abstime->tv_nsec >= 1000000000)) {
 571                 ret = EINVAL;
 572         } else {
 573                 ret = __thr_umutex_timedlock(&m->m_lock, id, abstime);
 574         }
 575 done:
 576         if (ret == 0)
 577                 enqueue_mutex(curthread, m);
 578
 579         return (ret);
 580 }
 581
 582 static inline int
 583 mutex_lock_common(struct pthread_mutex *m,
 584         const struct timespec *abstime, int cvattach)
 585 {
 586         struct pthread *curthread  = _get_curthread();
 587         int ret;
 588
 589         if (!cvattach && m->m_flags & PMUTEX_FLAG_PRIVATE)
 590                 THR_CRITICAL_ENTER(curthread);
 591         if (_thr_umutex_trylock2(&m->m_lock, TID(curthread)) == 0) {
 592                 enqueue_mutex(curthread, m);
 593                 ret = 0;
 594         } else {
 595                 ret = mutex_lock_sleep(curthread, m, abstime);
 596         }
 597         if (ret && (m->m_flags & PMUTEX_FLAG_PRIVATE) && !cvattach)
 598                 THR_CRITICAL_LEAVE(curthread);
 599         return (ret);
 600 }
 601
 602 int
 603 __pthread_mutex_lock(pthread_mutex_t *mutex)
 604 {
 605         struct pthread_mutex *m;
 606         int ret;
 607
 608         _thr_check_init();
 609         ret = check_and_init_mutex(mutex, &m);
 610         if (ret == 0)
 611                 ret = mutex_lock_common(m, NULL, 0);
 612         return (ret);
 613 }
 614
 615 int
 616 __pthread_mutex_timedlock(pthread_mutex_t *mutex,
 617     const struct timespec *abstime)
 618 {
 619         struct pthread_mutex *m;
 620         int ret;
 621
 622         _thr_check_init();
 623         ret = check_and_init_mutex(mutex, &m);
 624         if (ret == 0)
 625                 ret = mutex_lock_common(m, abstime, 0);
 626         return (ret);
 627 }
 628
 629 int
 630 _pthread_mutex_unlock(pthread_mutex_t *mutex)
 631 {
 632         struct pthread_mutex *mp;
 633
 634         if (*mutex == THR_PSHARED_PTR) {
 635                 mp = __thr_pshared_offpage(mutex, 0);
 636                 if (mp == NULL)
 637                         return (EINVAL);
 638                 shared_mutex_init(mp, NULL);
 639         } else {
 640                 mp = *mutex;
 641         }
 642         return (mutex_unlock_common(mp, 0, NULL));
 643 }
 644
 645 int
 646 _mutex_cv_lock(struct pthread_mutex *m, int count)
 647 {
 648         int     error;
 649
 650         error = mutex_lock_common(m, NULL, 1);
 651         if (error == 0)
 652                 m->m_count = count;
 653         return (error);
 654 }
 655
 656 int
 657 _mutex_cv_unlock(struct pthread_mutex *m, int *count, int *defer)
 658 {
 659
 660         /*
 661          * Clear the count in case this is a recursive mutex.
 662          */
 663         *count = m->m_count;
 664         m->m_count = 0;
 665         (void)mutex_unlock_common(m, 1, defer);
 666         return (0);
 667 }
 668
 669 int
 670 _mutex_cv_attach(struct pthread_mutex *m, int count)
 671 {
 672         struct pthread *curthread = _get_curthread();
 673
 674         enqueue_mutex(curthread, m);
 675         m->m_count = count;
 676         return (0);
 677 }
 678
 679 int
 680 _mutex_cv_detach(struct pthread_mutex *mp, int *recurse)
 681 {
 682         struct pthread *curthread = _get_curthread();
 683         int     defered;
 684         int     error;
 685
 686         if ((error = _mutex_owned(curthread, mp)) != 0)
 687                 return (error);
 688
 689         /*
 690          * Clear the count in case this is a recursive mutex.
 691          */
 692         *recurse = mp->m_count;
 693         mp->m_count = 0;
 694         dequeue_mutex(curthread, mp);
 695
 696         /* Will this happen in real-world ? */
 697         if ((mp->m_flags & PMUTEX_FLAG_DEFERED) != 0) {
 698                 defered = 1;
 699                 mp->m_flags &= ~PMUTEX_FLAG_DEFERED;
 700         } else
 701                 defered = 0;
 702
 703         if (defered)  {
 704                 _thr_wake_all(curthread->defer_waiters,
 705                                 curthread->nwaiter_defer);
 706                 curthread->nwaiter_defer = 0;
 707         }
 708         return (0);
 709 }
 710
 711 static int
 712 mutex_self_trylock(struct pthread_mutex *m)
 713 {
 714         int     ret;
 715
 716         switch (PMUTEX_TYPE(m->m_flags)) {
 717         case PTHREAD_MUTEX_ERRORCHECK:
 718         case PTHREAD_MUTEX_NORMAL:
 719         case PTHREAD_MUTEX_ADAPTIVE_NP:
 720                 ret = EBUSY;
 721                 break;
 722
 723         case PTHREAD_MUTEX_RECURSIVE:
 724                 /* Increment the lock count: */
 725                 if (m->m_count + 1 > 0) {
 726                         m->m_count++;
 727                         ret = 0;
 728                 } else
 729                         ret = EAGAIN;
 730                 break;
 731
 732         default:
 733                 /* Trap invalid mutex types; */
 734                 ret = EINVAL;
 735         }
 736
 737         return (ret);
 738 }
 739
 740 static int
 741 mutex_self_lock(struct pthread_mutex *m, const struct timespec *abstime)
 742 {
 743         struct timespec ts1, ts2;
 744         int     ret;
 745
 746         switch (PMUTEX_TYPE(m->m_flags)) {
 747         case PTHREAD_MUTEX_ERRORCHECK:
 748         case PTHREAD_MUTEX_ADAPTIVE_NP:
 749                 if (abstime) {
 750                         if (abstime->tv_sec < 0 || abstime->tv_nsec < 0 ||
 751                             abstime->tv_nsec >= 1000000000) {
 752                                 ret = EINVAL;
 753                         } else {
 754                                 clock_gettime(CLOCK_REALTIME, &ts1);
 755                                 TIMESPEC_SUB(&ts2, abstime, &ts1);
 756                                 __sys_nanosleep(&ts2, NULL);
 757                                 ret = ETIMEDOUT;
 758                         }
 759                 } else {
 760                         /*
 761                          * POSIX specifies that mutexes should return
 762                          * EDEADLK if a recursive lock is detected.
 763                          */
 764                         ret = EDEADLK;
 765                 }
 766                 break;
 767
 768         case PTHREAD_MUTEX_NORMAL:
 769                 /*
 770                  * What SS2 define as a 'normal' mutex.  Intentionally
 771                  * deadlock on attempts to get a lock you already own.
 772                  */
 773                 ret = 0;
 774                 if (abstime) {
 775                         if (abstime->tv_sec < 0 || abstime->tv_nsec < 0 ||
 776                             abstime->tv_nsec >= 1000000000) {
 777                                 ret = EINVAL;
 778                         } else {
 779                                 clock_gettime(CLOCK_REALTIME, &ts1);
 780                                 TIMESPEC_SUB(&ts2, abstime, &ts1);
 781                                 __sys_nanosleep(&ts2, NULL);
 782                                 ret = ETIMEDOUT;
 783                         }
 784                 } else {
 785                         ts1.tv_sec = 30;
 786                         ts1.tv_nsec = 0;
 787                         for (;;)
 788                                 __sys_nanosleep(&ts1, NULL);
 789                 }
 790                 break;
 791
 792         case PTHREAD_MUTEX_RECURSIVE:
 793                 /* Increment the lock count: */
 794                 if (m->m_count + 1 > 0) {
 795                         m->m_count++;
 796                         ret = 0;
 797                 } else
 798                         ret = EAGAIN;
 799                 break;
 800
 801         default:
 802                 /* Trap invalid mutex types; */
 803                 ret = EINVAL;
 804         }
 805
 806         return (ret);
 807 }
 808
 809 static int
 810 mutex_unlock_common(struct pthread_mutex *m, int cv, int *mtx_defer)
 811 {
 812         struct pthread *curthread = _get_curthread();
 813         uint32_t id;
 814         int defered, error;
 815
 816         if (__predict_false(m <= THR_MUTEX_DESTROYED)) {
 817                 if (m == THR_MUTEX_DESTROYED)
 818                         return (EINVAL);
 819                 return (EPERM);
 820         }
 821
 822         id = TID(curthread);
 823
 824         /*
 825          * Check if the running thread is not the owner of the mutex.
 826          */
 827         if (__predict_false(m->m_owner != id))
 828                 return (EPERM);
 829
 830         error = 0;
 831         if (__predict_false(
 832                 PMUTEX_TYPE(m->m_flags) == PTHREAD_MUTEX_RECURSIVE &&
 833                 m->m_count > 0)) {
 834                 m->m_count--;
 835         } else {
 836                 if ((m->m_flags & PMUTEX_FLAG_DEFERED) != 0) {
 837                         defered = 1;
 838                         m->m_flags &= ~PMUTEX_FLAG_DEFERED;
 839                 } else
 840                         defered = 0;
 841
 842                 dequeue_mutex(curthread, m);
 843                 error = _thr_umutex_unlock2(&m->m_lock, id, mtx_defer);
 844
 845                 if (mtx_defer == NULL && defered)  {
 846                         _thr_wake_all(curthread->defer_waiters,
 847                                 curthread->nwaiter_defer);
 848                         curthread->nwaiter_defer = 0;
 849                 }
 850         }
 851         if (!cv && m->m_flags & PMUTEX_FLAG_PRIVATE)
 852                 THR_CRITICAL_LEAVE(curthread);
 853         return (error);
 854 }
 855
 856 int
 857 _pthread_mutex_getprioceiling(pthread_mutex_t *mutex,
 858     int *prioceiling)
 859 {
 860         struct pthread_mutex *m;
 861
 862         if (*mutex == THR_PSHARED_PTR) {
 863                 m = __thr_pshared_offpage(mutex, 0);
 864                 if (m == NULL)
 865                         return (EINVAL);
 866                 shared_mutex_init(m, NULL);
 867         } else {
 868                 m = *mutex;
 869                 if (m <= THR_MUTEX_DESTROYED)
 870                         return (EINVAL);
 871         }
 872         if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)
 873                 return (EINVAL);
 874         *prioceiling = m->m_lock.m_ceilings[0];
 875         return (0);
 876 }
 877
 878 int
 879 _pthread_mutex_setprioceiling(pthread_mutex_t *mutex,
 880     int ceiling, int *old_ceiling)
 881 {
 882         struct pthread *curthread;
 883         struct pthread_mutex *m, *m1, *m2;
 884         struct mutex_queue *q, *qp;
 885         int ret;
 886
 887         if (*mutex == THR_PSHARED_PTR) {
 888                 m = __thr_pshared_offpage(mutex, 0);
 889                 if (m == NULL)
 890                         return (EINVAL);
 891                 shared_mutex_init(m, NULL);
 892         } else {
 893                 m = *mutex;
 894                 if (m <= THR_MUTEX_DESTROYED)
 895                         return (EINVAL);
 896         }
 897         if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)
 898                 return (EINVAL);
 899
 900         ret = __thr_umutex_set_ceiling(&m->m_lock, ceiling, old_ceiling);
 901         if (ret != 0)
 902                 return (ret);
 903
 904         curthread = _get_curthread();
 905         if (m->m_owner == TID(curthread)) {
 906                 mutex_assert_is_owned(m);
 907                 m1 = TAILQ_PREV(m, mutex_queue, m_qe);
 908                 m2 = TAILQ_NEXT(m, m_qe);
 909                 if ((m1 != NULL && m1->m_lock.m_ceilings[0] > (u_int)ceiling) ||
 910                     (m2 != NULL && m2->m_lock.m_ceilings[0] < (u_int)ceiling)) {
 911                         q = &curthread->mq[TMQ_NORM_PP];
 912                         qp = &curthread->mq[TMQ_NORM_PP_PRIV];
 913                         TAILQ_REMOVE(q, m, m_qe);
 914                         if (!is_pshared_mutex(m))
 915                                 TAILQ_REMOVE(qp, m, m_pqe);
 916                         TAILQ_FOREACH(m2, q, m_qe) {
 917                                 if (m2->m_lock.m_ceilings[0] > (u_int)ceiling) {
 918                                         TAILQ_INSERT_BEFORE(m2, m, m_qe);
 919                                         if (!is_pshared_mutex(m)) {
 920                                                 while (m2 != NULL &&
 921                                                     is_pshared_mutex(m2)) {
 922                                                         m2 = TAILQ_PREV(m2,
 923                                                             mutex_queue, m_qe);
 924                                                 }
 925                                                 if (m2 == NULL) {
 926                                                         TAILQ_INSERT_HEAD(qp,
 927                                                             m, m_pqe);
 928                                                 } else {
 929                                                         TAILQ_INSERT_BEFORE(m2,
 930                                                             m, m_pqe);
 931                                                 }
 932                                         }
 933                                         return (0);
 934                                 }
 935                         }
 936                         TAILQ_INSERT_TAIL(q, m, m_qe);
 937                         if (!is_pshared_mutex(m))
 938                                 TAILQ_INSERT_TAIL(qp, m, m_pqe);
 939                 }
 940         }
 941         return (0);
 942 }
 943
 944 int
 945 _pthread_mutex_getspinloops_np(pthread_mutex_t *mutex, int *count)
 946 {
 947         struct pthread_mutex *m;
 948         int ret;
 949
 950         ret = check_and_init_mutex(mutex, &m);
 951         if (ret == 0)
 952                 *count = m->m_spinloops;
 953         return (ret);
 954 }
 955
 956 int
 957 __pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count)
 958 {
 959         struct pthread_mutex *m;
 960         int ret;
 961
 962         ret = check_and_init_mutex(mutex, &m);
 963         if (ret == 0)
 964                 m->m_spinloops = count;
 965         return (ret);
 966 }
 967
 968 int
 969 _pthread_mutex_getyieldloops_np(pthread_mutex_t *mutex, int *count)
 970 {
 971         struct pthread_mutex *m;
 972         int ret;
 973
 974         ret = check_and_init_mutex(mutex, &m);
 975         if (ret == 0)
 976                 *count = m->m_yieldloops;
 977         return (ret);
 978 }
 979
 980 int
 981 __pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count)
 982 {
 983         struct pthread_mutex *m;
 984         int ret;
 985
 986         ret = check_and_init_mutex(mutex, &m);
 987         if (ret == 0)
 988                 m->m_yieldloops = count;
 989         return (0);
 990 }
 991
 992 int
 993 _pthread_mutex_isowned_np(pthread_mutex_t *mutex)
 994 {
 995         struct pthread_mutex *m;
 996
 997         if (*mutex == THR_PSHARED_PTR) {
 998                 m = __thr_pshared_offpage(mutex, 0);
 999                 if (m == NULL)
1000                         return (0);
1001                 shared_mutex_init(m, NULL);
1002         } else {
1003                 m = *mutex;
1004                 if (m <= THR_MUTEX_DESTROYED)
1005                         return (0);
1006         }
1007         return (m->m_owner == TID(_get_curthread()));
1008 }
1009
1010 int
1011 _mutex_owned(struct pthread *curthread, const struct pthread_mutex *mp)
1012 {
1013         if (__predict_false(mp <= THR_MUTEX_DESTROYED)) {
1014                 if (mp == THR_MUTEX_DESTROYED)
1015                         return (EINVAL);
1016                 return (EPERM);
1017         }
1018         if (mp->m_owner != TID(curthread))
1019                 return (EPERM);
1020         return (0);
1021 }