lib/libthr/thread/thr_mutex.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-4-Clause
   3  *
   4  * Copyright (c) 1995 John Birrell <jb@cimlogic.com.au>.
   5  * Copyright (c) 2006 David Xu <davidxu@freebsd.org>.
   6  * Copyright (c) 2015, 2016 The FreeBSD Foundation
   7  *
   8  * All rights reserved.
   9  *
  10  * Portions of this software were developed by Konstantin Belousov
  11  * under sponsorship from the FreeBSD Foundation.
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  * 1. Redistributions of source code must retain the above copyright
  17  *    notice, this list of conditions and the following disclaimer.
  18  * 2. Redistributions in binary form must reproduce the above copyright
  19  *    notice, this list of conditions and the following disclaimer in the
  20  *    documentation and/or other materials provided with the distribution.
  21  * 3. All advertising materials mentioning features or use of this software
  22  *    must display the following acknowledgement:
  23  *      This product includes software developed by John Birrell.
  24  * 4. Neither the name of the author nor the names of any co-contributors
  25  *    may be used to endorse or promote products derived from this software
  26  *    without specific prior written permission.
  27  *
  28  * THIS SOFTWARE IS PROVIDED BY JOHN BIRRELL AND CONTRIBUTORS ``AS IS'' AND
  29  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  30  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  31  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  32  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  36  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  37  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  38  * SUCH DAMAGE.
  39  */
  40
  41 #include <sys/cdefs.h>
  42 __FBSDID("$FreeBSD$");
  43
  44 #include "namespace.h"
  45 #include <stdlib.h>
  46 #include <errno.h>
  47 #include <string.h>
  48 #include <sys/param.h>
  49 #include <sys/queue.h>
  50 #include <pthread.h>
  51 #include <pthread_np.h>
  52 #include "un-namespace.h"
  53
  54 #include "thr_private.h"
  55
  56 _Static_assert(sizeof(struct pthread_mutex) <= PAGE_SIZE,
  57     "pthread_mutex is too large for off-page");
  58
  59 /*
  60  * For adaptive mutexes, how many times to spin doing trylock2
  61  * before entering the kernel to block
  62  */
  63 #define MUTEX_ADAPTIVE_SPINS    2000
  64
  65 /*
  66  * Prototypes
  67  */
  68 int     __pthread_mutex_consistent(pthread_mutex_t *mutex);
  69 int     __pthread_mutex_init(pthread_mutex_t *mutex,
  70                 const pthread_mutexattr_t *mutex_attr);
  71 int     __pthread_mutex_trylock(pthread_mutex_t *mutex);
  72 int     __pthread_mutex_lock(pthread_mutex_t *mutex);
  73 int     __pthread_mutex_timedlock(pthread_mutex_t *mutex,
  74                 const struct timespec *abstime);
  75 int     _pthread_mutex_getspinloops_np(pthread_mutex_t *mutex, int *count);
  76 int     _pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count);
  77 int     __pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count);
  78 int     _pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count);
  79 int     _pthread_mutex_getyieldloops_np(pthread_mutex_t *mutex, int *count);
  80 int     __pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count);
  81
  82 static int      mutex_self_trylock(pthread_mutex_t);
  83 static int      mutex_self_lock(pthread_mutex_t,
  84                                 const struct timespec *abstime);
  85 static int      mutex_unlock_common(struct pthread_mutex *, bool, int *);
  86 static int      mutex_lock_sleep(struct pthread *, pthread_mutex_t,
  87                                 const struct timespec *);
  88 static void     mutex_init_robust(struct pthread *curthread);
  89 static int      mutex_qidx(struct pthread_mutex *m);
  90 static bool     is_robust_mutex(struct pthread_mutex *m);
  91 static bool     is_pshared_mutex(struct pthread_mutex *m);
  92
  93 __weak_reference(__pthread_mutex_init, pthread_mutex_init);
  94 __strong_reference(__pthread_mutex_init, _pthread_mutex_init);
  95 __weak_reference(__pthread_mutex_lock, pthread_mutex_lock);
  96 __strong_reference(__pthread_mutex_lock, _pthread_mutex_lock);
  97 __weak_reference(__pthread_mutex_timedlock, pthread_mutex_timedlock);
  98 __strong_reference(__pthread_mutex_timedlock, _pthread_mutex_timedlock);
  99 __weak_reference(__pthread_mutex_trylock, pthread_mutex_trylock);
 100 __strong_reference(__pthread_mutex_trylock, _pthread_mutex_trylock);
 101 __weak_reference(_pthread_mutex_consistent, pthread_mutex_consistent);
 102 __strong_reference(_pthread_mutex_consistent, __pthread_mutex_consistent);
 103
 104 /* Single underscore versions provided for libc internal usage: */
 105 /* No difference between libc and application usage of these: */
 106 __weak_reference(_pthread_mutex_destroy, pthread_mutex_destroy);
 107 __weak_reference(_pthread_mutex_unlock, pthread_mutex_unlock);
 108
 109 __weak_reference(_pthread_mutex_getprioceiling, pthread_mutex_getprioceiling);
 110 __weak_reference(_pthread_mutex_setprioceiling, pthread_mutex_setprioceiling);
 111
 112 __weak_reference(__pthread_mutex_setspinloops_np, pthread_mutex_setspinloops_np);
 113 __strong_reference(__pthread_mutex_setspinloops_np, _pthread_mutex_setspinloops_np);
 114 __weak_reference(_pthread_mutex_getspinloops_np, pthread_mutex_getspinloops_np);
 115
 116 __weak_reference(__pthread_mutex_setyieldloops_np, pthread_mutex_setyieldloops_np);
 117 __strong_reference(__pthread_mutex_setyieldloops_np, _pthread_mutex_setyieldloops_np);
 118 __weak_reference(_pthread_mutex_getyieldloops_np, pthread_mutex_getyieldloops_np);
 119 __weak_reference(_pthread_mutex_isowned_np, pthread_mutex_isowned_np);
 120
 121 static void
 122 mutex_init_link(struct pthread_mutex *m)
 123 {
 124
 125 #if defined(_PTHREADS_INVARIANTS)
 126         m->m_qe.tqe_prev = NULL;
 127         m->m_qe.tqe_next = NULL;
 128         m->m_pqe.tqe_prev = NULL;
 129         m->m_pqe.tqe_next = NULL;
 130 #endif
 131 }
 132
 133 static void
 134 mutex_assert_is_owned(struct pthread_mutex *m __unused)
 135 {
 136
 137 #if defined(_PTHREADS_INVARIANTS)
 138         if (__predict_false(m->m_qe.tqe_prev == NULL))
 139                 PANIC("mutex %p own %#x is not on list %p %p",
 140                     m, m->m_lock.m_owner, m->m_qe.tqe_prev, m->m_qe.tqe_next);
 141 #endif
 142 }
 143
 144 static void
 145 mutex_assert_not_owned(struct pthread *curthread __unused,
 146     struct pthread_mutex *m __unused)
 147 {
 148
 149 #if defined(_PTHREADS_INVARIANTS)
 150         if (__predict_false(m->m_qe.tqe_prev != NULL ||
 151             m->m_qe.tqe_next != NULL))
 152                 PANIC("mutex %p own %#x is on list %p %p",
 153                     m, m->m_lock.m_owner, m->m_qe.tqe_prev, m->m_qe.tqe_next);
 154         if (__predict_false(is_robust_mutex(m) &&
 155             (m->m_lock.m_rb_lnk != 0 || m->m_rb_prev != NULL ||
 156             (is_pshared_mutex(m) && curthread->robust_list ==
 157             (uintptr_t)&m->m_lock) ||
 158             (!is_pshared_mutex(m) && curthread->priv_robust_list ==
 159             (uintptr_t)&m->m_lock))))
 160                 PANIC(
 161     "mutex %p own %#x is on robust linkage %p %p head %p phead %p",
 162                     m, m->m_lock.m_owner, (void *)m->m_lock.m_rb_lnk,
 163                     m->m_rb_prev, (void *)curthread->robust_list,
 164                     (void *)curthread->priv_robust_list);
 165 #endif
 166 }
 167
 168 static bool
 169 is_pshared_mutex(struct pthread_mutex *m)
 170 {
 171
 172         return ((m->m_lock.m_flags & USYNC_PROCESS_SHARED) != 0);
 173 }
 174
 175 static bool
 176 is_robust_mutex(struct pthread_mutex *m)
 177 {
 178
 179         return ((m->m_lock.m_flags & UMUTEX_ROBUST) != 0);
 180 }
 181
 182 int
 183 _mutex_enter_robust(struct pthread *curthread, struct pthread_mutex *m)
 184 {
 185
 186 #if defined(_PTHREADS_INVARIANTS)
 187         if (__predict_false(curthread->inact_mtx != 0))
 188                 PANIC("inact_mtx enter");
 189 #endif
 190         if (!is_robust_mutex(m))
 191                 return (0);
 192
 193         mutex_init_robust(curthread);
 194         curthread->inact_mtx = (uintptr_t)&m->m_lock;
 195         return (1);
 196 }
 197
 198 void
 199 _mutex_leave_robust(struct pthread *curthread, struct pthread_mutex *m __unused)
 200 {
 201
 202 #if defined(_PTHREADS_INVARIANTS)
 203         if (__predict_false(curthread->inact_mtx != (uintptr_t)&m->m_lock))
 204                 PANIC("inact_mtx leave");
 205 #endif
 206         curthread->inact_mtx = 0;
 207 }
 208
 209 static int
 210 mutex_check_attr(const struct pthread_mutex_attr *attr)
 211 {
 212
 213         if (attr->m_type < PTHREAD_MUTEX_ERRORCHECK ||
 214             attr->m_type >= PTHREAD_MUTEX_TYPE_MAX)
 215                 return (EINVAL);
 216         if (attr->m_protocol < PTHREAD_PRIO_NONE ||
 217             attr->m_protocol > PTHREAD_PRIO_PROTECT)
 218                 return (EINVAL);
 219         return (0);
 220 }
 221
 222 static void
 223 mutex_init_robust(struct pthread *curthread)
 224 {
 225         struct umtx_robust_lists_params rb;
 226
 227         if (curthread == NULL)
 228                 curthread = _get_curthread();
 229         if (curthread->robust_inited)
 230                 return;
 231         rb.robust_list_offset = (uintptr_t)&curthread->robust_list;
 232         rb.robust_priv_list_offset = (uintptr_t)&curthread->priv_robust_list;
 233         rb.robust_inact_offset = (uintptr_t)&curthread->inact_mtx;
 234         _umtx_op(NULL, UMTX_OP_ROBUST_LISTS, sizeof(rb), &rb, NULL);
 235         curthread->robust_inited = 1;
 236 }
 237
 238 static void
 239 mutex_init_body(struct pthread_mutex *pmutex,
 240     const struct pthread_mutex_attr *attr)
 241 {
 242
 243         pmutex->m_flags = attr->m_type;
 244         pmutex->m_count = 0;
 245         pmutex->m_spinloops = 0;
 246         pmutex->m_yieldloops = 0;
 247         mutex_init_link(pmutex);
 248         switch (attr->m_protocol) {
 249         case PTHREAD_PRIO_NONE:
 250                 pmutex->m_lock.m_owner = UMUTEX_UNOWNED;
 251                 pmutex->m_lock.m_flags = 0;
 252                 break;
 253         case PTHREAD_PRIO_INHERIT:
 254                 pmutex->m_lock.m_owner = UMUTEX_UNOWNED;
 255                 pmutex->m_lock.m_flags = UMUTEX_PRIO_INHERIT;
 256                 break;
 257         case PTHREAD_PRIO_PROTECT:
 258                 pmutex->m_lock.m_owner = UMUTEX_CONTESTED;
 259                 pmutex->m_lock.m_flags = UMUTEX_PRIO_PROTECT;
 260                 pmutex->m_lock.m_ceilings[0] = attr->m_ceiling;
 261                 break;
 262         }
 263         if (attr->m_pshared == PTHREAD_PROCESS_SHARED)
 264                 pmutex->m_lock.m_flags |= USYNC_PROCESS_SHARED;
 265         if (attr->m_robust == PTHREAD_MUTEX_ROBUST) {
 266                 mutex_init_robust(NULL);
 267                 pmutex->m_lock.m_flags |= UMUTEX_ROBUST;
 268         }
 269         if (PMUTEX_TYPE(pmutex->m_flags) == PTHREAD_MUTEX_ADAPTIVE_NP) {
 270                 pmutex->m_spinloops =
 271                     _thr_spinloops ? _thr_spinloops: MUTEX_ADAPTIVE_SPINS;
 272                 pmutex->m_yieldloops = _thr_yieldloops;
 273         }
 274 }
 275
 276 static int
 277 mutex_init(pthread_mutex_t *mutex,
 278     const struct pthread_mutex_attr *mutex_attr,
 279     void *(calloc_cb)(size_t, size_t))
 280 {
 281         const struct pthread_mutex_attr *attr;
 282         struct pthread_mutex *pmutex;
 283         int error;
 284
 285         if (mutex_attr == NULL) {
 286                 attr = &_pthread_mutexattr_default;
 287         } else {
 288                 attr = mutex_attr;
 289                 error = mutex_check_attr(attr);
 290                 if (error != 0)
 291                         return (error);
 292         }
 293         if ((pmutex = (pthread_mutex_t)
 294                 calloc_cb(1, sizeof(struct pthread_mutex))) == NULL)
 295                 return (ENOMEM);
 296         mutex_init_body(pmutex, attr);
 297         *mutex = pmutex;
 298         return (0);
 299 }
 300
 301 static int
 302 init_static(struct pthread *thread, pthread_mutex_t *mutex)
 303 {
 304         int ret;
 305
 306         THR_LOCK_ACQUIRE(thread, &_mutex_static_lock);
 307
 308         if (*mutex == THR_MUTEX_INITIALIZER)
 309                 ret = mutex_init(mutex, &_pthread_mutexattr_default, calloc);
 310         else if (*mutex == THR_ADAPTIVE_MUTEX_INITIALIZER)
 311                 ret = mutex_init(mutex, &_pthread_mutexattr_adaptive_default,
 312                     calloc);
 313         else
 314                 ret = 0;
 315         THR_LOCK_RELEASE(thread, &_mutex_static_lock);
 316
 317         return (ret);
 318 }
 319
 320 static void
 321 set_inherited_priority(struct pthread *curthread, struct pthread_mutex *m)
 322 {
 323         struct pthread_mutex *m2;
 324
 325         m2 = TAILQ_LAST(&curthread->mq[mutex_qidx(m)], mutex_queue);
 326         if (m2 != NULL)
 327                 m->m_lock.m_ceilings[1] = m2->m_lock.m_ceilings[0];
 328         else
 329                 m->m_lock.m_ceilings[1] = -1;
 330 }
 331
 332 static void
 333 shared_mutex_init(struct pthread_mutex *pmtx, const struct
 334     pthread_mutex_attr *mutex_attr)
 335 {
 336         static const struct pthread_mutex_attr foobar_mutex_attr = {
 337                 .m_type = PTHREAD_MUTEX_DEFAULT,
 338                 .m_protocol = PTHREAD_PRIO_NONE,
 339                 .m_ceiling = 0,
 340                 .m_pshared = PTHREAD_PROCESS_SHARED,
 341                 .m_robust = PTHREAD_MUTEX_STALLED,
 342         };
 343         bool done;
 344
 345         /*
 346          * Hack to allow multiple pthread_mutex_init() calls on the
 347          * same process-shared mutex.  We rely on kernel allocating
 348          * zeroed offpage for the mutex, i.e. the
 349          * PMUTEX_INITSTAGE_ALLOC value must be zero.
 350          */
 351         for (done = false; !done;) {
 352                 switch (pmtx->m_ps) {
 353                 case PMUTEX_INITSTAGE_DONE:
 354                         atomic_thread_fence_acq();
 355                         done = true;
 356                         break;
 357                 case PMUTEX_INITSTAGE_ALLOC:
 358                         if (atomic_cmpset_int(&pmtx->m_ps,
 359                             PMUTEX_INITSTAGE_ALLOC, PMUTEX_INITSTAGE_BUSY)) {
 360                                 if (mutex_attr == NULL)
 361                                         mutex_attr = &foobar_mutex_attr;
 362                                 mutex_init_body(pmtx, mutex_attr);
 363                                 atomic_store_rel_int(&pmtx->m_ps,
 364                                     PMUTEX_INITSTAGE_DONE);
 365                                 done = true;
 366                         }
 367                         break;
 368                 case PMUTEX_INITSTAGE_BUSY:
 369                         _pthread_yield();
 370                         break;
 371                 default:
 372                         PANIC("corrupted offpage");
 373                         break;
 374                 }
 375         }
 376 }
 377
 378 int
 379 __pthread_mutex_init(pthread_mutex_t *mutex,
 380     const pthread_mutexattr_t *mutex_attr)
 381 {
 382         struct pthread_mutex *pmtx;
 383         int ret;
 384
 385         if (mutex_attr != NULL) {
 386                 ret = mutex_check_attr(*mutex_attr);
 387                 if (ret != 0)
 388                         return (ret);
 389         }
 390         if (mutex_attr == NULL ||
 391             (*mutex_attr)->m_pshared == PTHREAD_PROCESS_PRIVATE) {
 392                 return (mutex_init(mutex, mutex_attr ? *mutex_attr : NULL,
 393                     calloc));
 394         }
 395         pmtx = __thr_pshared_offpage(mutex, 1);
 396         if (pmtx == NULL)
 397                 return (EFAULT);
 398         *mutex = THR_PSHARED_PTR;
 399         shared_mutex_init(pmtx, *mutex_attr);
 400         return (0);
 401 }
 402
 403 /* This function is used internally by malloc. */
 404 int
 405 _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
 406     void *(calloc_cb)(size_t, size_t))
 407 {
 408         static const struct pthread_mutex_attr attr = {
 409                 .m_type = PTHREAD_MUTEX_NORMAL,
 410                 .m_protocol = PTHREAD_PRIO_NONE,
 411                 .m_ceiling = 0,
 412                 .m_pshared = PTHREAD_PROCESS_PRIVATE,
 413                 .m_robust = PTHREAD_MUTEX_STALLED,
 414         };
 415         int ret;
 416
 417         ret = mutex_init(mutex, &attr, calloc_cb);
 418         if (ret == 0)
 419                 (*mutex)->m_flags |= PMUTEX_FLAG_PRIVATE;
 420         return (ret);
 421 }
 422
 423 /*
 424  * Fix mutex ownership for child process.
 425  *
 426  * Process private mutex ownership is transmitted from the forking
 427  * thread to the child process.
 428  *
 429  * Process shared mutex should not be inherited because owner is
 430  * forking thread which is in parent process, they are removed from
 431  * the owned mutex list.
 432  */
 433 static void
 434 queue_fork(struct pthread *curthread, struct mutex_queue *q,
 435     struct mutex_queue *qp, uint bit)
 436 {
 437         struct pthread_mutex *m;
 438
 439         TAILQ_INIT(q);
 440         TAILQ_FOREACH(m, qp, m_pqe) {
 441                 TAILQ_INSERT_TAIL(q, m, m_qe);
 442                 m->m_lock.m_owner = TID(curthread) | bit;
 443         }
 444 }
 445
 446 void
 447 _mutex_fork(struct pthread *curthread)
 448 {
 449
 450         queue_fork(curthread, &curthread->mq[TMQ_NORM],
 451             &curthread->mq[TMQ_NORM_PRIV], 0);
 452         queue_fork(curthread, &curthread->mq[TMQ_NORM_PP],
 453             &curthread->mq[TMQ_NORM_PP_PRIV], UMUTEX_CONTESTED);
 454         queue_fork(curthread, &curthread->mq[TMQ_ROBUST_PP],
 455             &curthread->mq[TMQ_ROBUST_PP_PRIV], UMUTEX_CONTESTED);
 456         curthread->robust_list = 0;
 457 }
 458
 459 int
 460 _pthread_mutex_destroy(pthread_mutex_t *mutex)
 461 {
 462         pthread_mutex_t m, m1;
 463         int ret;
 464
 465         m = *mutex;
 466         if (m < THR_MUTEX_DESTROYED) {
 467                 ret = 0;
 468         } else if (m == THR_MUTEX_DESTROYED) {
 469                 ret = EINVAL;
 470         } else {
 471                 if (m == THR_PSHARED_PTR) {
 472                         m1 = __thr_pshared_offpage(mutex, 0);
 473                         if (m1 != NULL) {
 474                                 mutex_assert_not_owned(_get_curthread(), m1);
 475                                 __thr_pshared_destroy(mutex);
 476                         }
 477                         *mutex = THR_MUTEX_DESTROYED;
 478                         return (0);
 479                 }
 480                 if (PMUTEX_OWNER_ID(m) != 0 &&
 481                     (uint32_t)m->m_lock.m_owner != UMUTEX_RB_NOTRECOV) {
 482                         ret = EBUSY;
 483                 } else {
 484                         *mutex = THR_MUTEX_DESTROYED;
 485                         mutex_assert_not_owned(_get_curthread(), m);
 486                         free(m);
 487                         ret = 0;
 488                 }
 489         }
 490
 491         return (ret);
 492 }
 493
 494 static int
 495 mutex_qidx(struct pthread_mutex *m)
 496 {
 497
 498         if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)
 499                 return (TMQ_NORM);
 500         return (is_robust_mutex(m) ? TMQ_ROBUST_PP : TMQ_NORM_PP);
 501 }
 502
 503 /*
 504  * Both enqueue_mutex() and dequeue_mutex() operate on the
 505  * thread-private linkage of the locked mutexes and on the robust
 506  * linkage.
 507  *
 508  * Robust list, as seen by kernel, must be consistent even in the case
 509  * of thread termination at arbitrary moment.  Since either enqueue or
 510  * dequeue for list walked by kernel consists of rewriting a single
 511  * forward pointer, it is safe.  On the other hand, rewrite of the
 512  * back pointer is not atomic WRT the forward one, but kernel does not
 513  * care.
 514  */
 515 static void
 516 enqueue_mutex(struct pthread *curthread, struct pthread_mutex *m,
 517     int error)
 518 {
 519         struct pthread_mutex *m1;
 520         uintptr_t *rl;
 521         int qidx;
 522
 523         /* Add to the list of owned mutexes: */
 524         if (error != EOWNERDEAD)
 525                 mutex_assert_not_owned(curthread, m);
 526         qidx = mutex_qidx(m);
 527         TAILQ_INSERT_TAIL(&curthread->mq[qidx], m, m_qe);
 528         if (!is_pshared_mutex(m))
 529                 TAILQ_INSERT_TAIL(&curthread->mq[qidx + 1], m, m_pqe);
 530         if (is_robust_mutex(m)) {
 531                 rl = is_pshared_mutex(m) ? &curthread->robust_list :
 532                     &curthread->priv_robust_list;
 533                 m->m_rb_prev = NULL;
 534                 if (*rl != 0) {
 535                         m1 = __containerof((void *)*rl,
 536                             struct pthread_mutex, m_lock);
 537                         m->m_lock.m_rb_lnk = (uintptr_t)&m1->m_lock;
 538                         m1->m_rb_prev = m;
 539                 } else {
 540                         m1 = NULL;
 541                         m->m_lock.m_rb_lnk = 0;
 542                 }
 543                 *rl = (uintptr_t)&m->m_lock;
 544         }
 545 }
 546
 547 static void
 548 dequeue_mutex(struct pthread *curthread, struct pthread_mutex *m)
 549 {
 550         struct pthread_mutex *mp, *mn;
 551         int qidx;
 552
 553         mutex_assert_is_owned(m);
 554         qidx = mutex_qidx(m);
 555         if (is_robust_mutex(m)) {
 556                 mp = m->m_rb_prev;
 557                 if (mp == NULL) {
 558                         if (is_pshared_mutex(m)) {
 559                                 curthread->robust_list = m->m_lock.m_rb_lnk;
 560                         } else {
 561                                 curthread->priv_robust_list =
 562                                     m->m_lock.m_rb_lnk;
 563                         }
 564                 } else {
 565                         mp->m_lock.m_rb_lnk = m->m_lock.m_rb_lnk;
 566                 }
 567                 if (m->m_lock.m_rb_lnk != 0) {
 568                         mn = __containerof((void *)m->m_lock.m_rb_lnk,
 569                             struct pthread_mutex, m_lock);
 570                         mn->m_rb_prev = m->m_rb_prev;
 571                 }
 572                 m->m_lock.m_rb_lnk = 0;
 573                 m->m_rb_prev = NULL;
 574         }
 575         TAILQ_REMOVE(&curthread->mq[qidx], m, m_qe);
 576         if (!is_pshared_mutex(m))
 577                 TAILQ_REMOVE(&curthread->mq[qidx + 1], m, m_pqe);
 578         if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) != 0)
 579                 set_inherited_priority(curthread, m);
 580         mutex_init_link(m);
 581 }
 582
 583 static int
 584 check_and_init_mutex(pthread_mutex_t *mutex, struct pthread_mutex **m)
 585 {
 586         int ret;
 587
 588         *m = *mutex;
 589         ret = 0;
 590         if (*m == THR_PSHARED_PTR) {
 591                 *m = __thr_pshared_offpage(mutex, 0);
 592                 if (*m == NULL)
 593                         ret = EINVAL;
 594                 else
 595                         shared_mutex_init(*m, NULL);
 596         } else if (__predict_false(*m <= THR_MUTEX_DESTROYED)) {
 597                 if (*m == THR_MUTEX_DESTROYED) {
 598                         ret = EINVAL;
 599                 } else {
 600                         ret = init_static(_get_curthread(), mutex);
 601                         if (ret == 0)
 602                                 *m = *mutex;
 603                 }
 604         }
 605         return (ret);
 606 }
 607
 608 int
 609 __pthread_mutex_trylock(pthread_mutex_t *mutex)
 610 {
 611         struct pthread *curthread;
 612         struct pthread_mutex *m;
 613         uint32_t id;
 614         int ret, robust;
 615
 616         ret = check_and_init_mutex(mutex, &m);
 617         if (ret != 0)
 618                 return (ret);
 619         curthread = _get_curthread();
 620         id = TID(curthread);
 621         if (m->m_flags & PMUTEX_FLAG_PRIVATE)
 622                 THR_CRITICAL_ENTER(curthread);
 623         robust = _mutex_enter_robust(curthread, m);
 624         ret = _thr_umutex_trylock(&m->m_lock, id);
 625         if (__predict_true(ret == 0) || ret == EOWNERDEAD) {
 626                 enqueue_mutex(curthread, m, ret);
 627                 if (ret == EOWNERDEAD)
 628                         m->m_lock.m_flags |= UMUTEX_NONCONSISTENT;
 629         } else if (PMUTEX_OWNER_ID(m) == id) {
 630                 ret = mutex_self_trylock(m);
 631         } /* else {} */
 632         if (robust)
 633                 _mutex_leave_robust(curthread, m);
 634         if (ret != 0 && ret != EOWNERDEAD &&
 635             (m->m_flags & PMUTEX_FLAG_PRIVATE) != 0)
 636                 THR_CRITICAL_LEAVE(curthread);
 637         return (ret);
 638 }
 639
 640 static int
 641 mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m,
 642     const struct timespec *abstime)
 643 {
 644         uint32_t id, owner;
 645         int count, ret;
 646
 647         id = TID(curthread);
 648         if (PMUTEX_OWNER_ID(m) == id)
 649                 return (mutex_self_lock(m, abstime));
 650
 651         /*
 652          * For adaptive mutexes, spin for a bit in the expectation
 653          * that if the application requests this mutex type then
 654          * the lock is likely to be released quickly and it is
 655          * faster than entering the kernel
 656          */
 657         if (__predict_false((m->m_lock.m_flags & (UMUTEX_PRIO_PROTECT |
 658             UMUTEX_PRIO_INHERIT | UMUTEX_ROBUST | UMUTEX_NONCONSISTENT)) != 0))
 659                 goto sleep_in_kernel;
 660
 661         if (!_thr_is_smp)
 662                 goto yield_loop;
 663
 664         count = m->m_spinloops;
 665         while (count--) {
 666                 owner = m->m_lock.m_owner;
 667                 if ((owner & ~UMUTEX_CONTESTED) == 0) {
 668                         if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner,
 669                             id | owner)) {
 670                                 ret = 0;
 671                                 goto done;
 672                         }
 673                 }
 674                 CPU_SPINWAIT;
 675         }
 676
 677 yield_loop:
 678         count = m->m_yieldloops;
 679         while (count--) {
 680                 _sched_yield();
 681                 owner = m->m_lock.m_owner;
 682                 if ((owner & ~UMUTEX_CONTESTED) == 0) {
 683                         if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner,
 684                             id | owner)) {
 685                                 ret = 0;
 686                                 goto done;
 687                         }
 688                 }
 689         }
 690
 691 sleep_in_kernel:
 692         if (abstime == NULL)
 693                 ret = __thr_umutex_lock(&m->m_lock, id);
 694         else if (__predict_false(abstime->tv_nsec < 0 ||
 695             abstime->tv_nsec >= 1000000000))
 696                 ret = EINVAL;
 697         else
 698                 ret = __thr_umutex_timedlock(&m->m_lock, id, abstime);
 699 done:
 700         if (ret == 0 || ret == EOWNERDEAD) {
 701                 enqueue_mutex(curthread, m, ret);
 702                 if (ret == EOWNERDEAD)
 703                         m->m_lock.m_flags |= UMUTEX_NONCONSISTENT;
 704         }
 705         return (ret);
 706 }
 707
 708 static inline int
 709 mutex_lock_common(struct pthread_mutex *m, const struct timespec *abstime,
 710     bool cvattach, bool rb_onlist)
 711 {
 712         struct pthread *curthread;
 713         int ret, robust;
 714
 715         robust = 0;  /* pacify gcc */
 716         curthread  = _get_curthread();
 717         if (!cvattach && m->m_flags & PMUTEX_FLAG_PRIVATE)
 718                 THR_CRITICAL_ENTER(curthread);
 719         if (!rb_onlist)
 720                 robust = _mutex_enter_robust(curthread, m);
 721         ret = _thr_umutex_trylock2(&m->m_lock, TID(curthread));
 722         if (ret == 0 || ret == EOWNERDEAD) {
 723                 enqueue_mutex(curthread, m, ret);
 724                 if (ret == EOWNERDEAD)
 725                         m->m_lock.m_flags |= UMUTEX_NONCONSISTENT;
 726         } else {
 727                 ret = mutex_lock_sleep(curthread, m, abstime);
 728         }
 729         if (!rb_onlist && robust)
 730                 _mutex_leave_robust(curthread, m);
 731         if (ret != 0 && ret != EOWNERDEAD &&
 732             (m->m_flags & PMUTEX_FLAG_PRIVATE) != 0 && !cvattach)
 733                 THR_CRITICAL_LEAVE(curthread);
 734         return (ret);
 735 }
 736
 737 int
 738 __pthread_mutex_lock(pthread_mutex_t *mutex)
 739 {
 740         struct pthread_mutex *m;
 741         int ret;
 742
 743         _thr_check_init();
 744         ret = check_and_init_mutex(mutex, &m);
 745         if (ret == 0)
 746                 ret = mutex_lock_common(m, NULL, false, false);
 747         return (ret);
 748 }
 749
 750 int
 751 __pthread_mutex_timedlock(pthread_mutex_t *mutex,
 752     const struct timespec *abstime)
 753 {
 754         struct pthread_mutex *m;
 755         int ret;
 756
 757         _thr_check_init();
 758         ret = check_and_init_mutex(mutex, &m);
 759         if (ret == 0)
 760                 ret = mutex_lock_common(m, abstime, false, false);
 761         return (ret);
 762 }
 763
 764 int
 765 _pthread_mutex_unlock(pthread_mutex_t *mutex)
 766 {
 767         struct pthread_mutex *mp;
 768
 769         if (*mutex == THR_PSHARED_PTR) {
 770                 mp = __thr_pshared_offpage(mutex, 0);
 771                 if (mp == NULL)
 772                         return (EINVAL);
 773                 shared_mutex_init(mp, NULL);
 774         } else {
 775                 mp = *mutex;
 776         }
 777         return (mutex_unlock_common(mp, false, NULL));
 778 }
 779
 780 int
 781 _mutex_cv_lock(struct pthread_mutex *m, int count, bool rb_onlist)
 782 {
 783         int error;
 784
 785         error = mutex_lock_common(m, NULL, true, rb_onlist);
 786         if (error == 0 || error == EOWNERDEAD)
 787                 m->m_count = count;
 788         return (error);
 789 }
 790
 791 int
 792 _mutex_cv_unlock(struct pthread_mutex *m, int *count, int *defer)
 793 {
 794
 795         /*
 796          * Clear the count in case this is a recursive mutex.
 797          */
 798         *count = m->m_count;
 799         m->m_count = 0;
 800         (void)mutex_unlock_common(m, true, defer);
 801         return (0);
 802 }
 803
 804 int
 805 _mutex_cv_attach(struct pthread_mutex *m, int count)
 806 {
 807         struct pthread *curthread;
 808
 809         curthread = _get_curthread();
 810         enqueue_mutex(curthread, m, 0);
 811         m->m_count = count;
 812         return (0);
 813 }
 814
 815 int
 816 _mutex_cv_detach(struct pthread_mutex *mp, int *recurse)
 817 {
 818         struct pthread *curthread;
 819         int deferred, error;
 820
 821         curthread = _get_curthread();
 822         if ((error = _mutex_owned(curthread, mp)) != 0)
 823                 return (error);
 824
 825         /*
 826          * Clear the count in case this is a recursive mutex.
 827          */
 828         *recurse = mp->m_count;
 829         mp->m_count = 0;
 830         dequeue_mutex(curthread, mp);
 831
 832         /* Will this happen in real-world ? */
 833         if ((mp->m_flags & PMUTEX_FLAG_DEFERRED) != 0) {
 834                 deferred = 1;
 835                 mp->m_flags &= ~PMUTEX_FLAG_DEFERRED;
 836         } else
 837                 deferred = 0;
 838
 839         if (deferred)  {
 840                 _thr_wake_all(curthread->defer_waiters,
 841                     curthread->nwaiter_defer);
 842                 curthread->nwaiter_defer = 0;
 843         }
 844         return (0);
 845 }
 846
 847 static int
 848 mutex_self_trylock(struct pthread_mutex *m)
 849 {
 850         int ret;
 851
 852         switch (PMUTEX_TYPE(m->m_flags)) {
 853         case PTHREAD_MUTEX_ERRORCHECK:
 854         case PTHREAD_MUTEX_NORMAL:
 855         case PTHREAD_MUTEX_ADAPTIVE_NP:
 856                 ret = EBUSY;
 857                 break;
 858
 859         case PTHREAD_MUTEX_RECURSIVE:
 860                 /* Increment the lock count: */
 861                 if (m->m_count + 1 > 0) {
 862                         m->m_count++;
 863                         ret = 0;
 864                 } else
 865                         ret = EAGAIN;
 866                 break;
 867
 868         default:
 869                 /* Trap invalid mutex types; */
 870                 ret = EINVAL;
 871         }
 872
 873         return (ret);
 874 }
 875
 876 static int
 877 mutex_self_lock(struct pthread_mutex *m, const struct timespec *abstime)
 878 {
 879         struct timespec ts1, ts2;
 880         int ret;
 881
 882         switch (PMUTEX_TYPE(m->m_flags)) {
 883         case PTHREAD_MUTEX_ERRORCHECK:
 884         case PTHREAD_MUTEX_ADAPTIVE_NP:
 885                 if (abstime) {
 886                         if (abstime->tv_sec < 0 || abstime->tv_nsec < 0 ||
 887                             abstime->tv_nsec >= 1000000000) {
 888                                 ret = EINVAL;
 889                         } else {
 890                                 clock_gettime(CLOCK_REALTIME, &ts1);
 891                                 TIMESPEC_SUB(&ts2, abstime, &ts1);
 892                                 __sys_nanosleep(&ts2, NULL);
 893                                 ret = ETIMEDOUT;
 894                         }
 895                 } else {
 896                         /*
 897                          * POSIX specifies that mutexes should return
 898                          * EDEADLK if a recursive lock is detected.
 899                          */
 900                         ret = EDEADLK;
 901                 }
 902                 break;
 903
 904         case PTHREAD_MUTEX_NORMAL:
 905                 /*
 906                  * What SS2 define as a 'normal' mutex.  Intentionally
 907                  * deadlock on attempts to get a lock you already own.
 908                  */
 909                 ret = 0;
 910                 if (abstime) {
 911                         if (abstime->tv_sec < 0 || abstime->tv_nsec < 0 ||
 912                             abstime->tv_nsec >= 1000000000) {
 913                                 ret = EINVAL;
 914                         } else {
 915                                 clock_gettime(CLOCK_REALTIME, &ts1);
 916                                 TIMESPEC_SUB(&ts2, abstime, &ts1);
 917                                 __sys_nanosleep(&ts2, NULL);
 918                                 ret = ETIMEDOUT;
 919                         }
 920                 } else {
 921                         ts1.tv_sec = 30;
 922                         ts1.tv_nsec = 0;
 923                         for (;;)
 924                                 __sys_nanosleep(&ts1, NULL);
 925                 }
 926                 break;
 927
 928         case PTHREAD_MUTEX_RECURSIVE:
 929                 /* Increment the lock count: */
 930                 if (m->m_count + 1 > 0) {
 931                         m->m_count++;
 932                         ret = 0;
 933                 } else
 934                         ret = EAGAIN;
 935                 break;
 936
 937         default:
 938                 /* Trap invalid mutex types; */
 939                 ret = EINVAL;
 940         }
 941
 942         return (ret);
 943 }
 944
 945 static int
 946 mutex_unlock_common(struct pthread_mutex *m, bool cv, int *mtx_defer)
 947 {
 948         struct pthread *curthread;
 949         uint32_t id;
 950         int deferred, error, robust;
 951
 952         if (__predict_false(m <= THR_MUTEX_DESTROYED)) {
 953                 if (m == THR_MUTEX_DESTROYED)
 954                         return (EINVAL);
 955                 return (EPERM);
 956         }
 957
 958         curthread = _get_curthread();
 959         id = TID(curthread);
 960
 961         /*
 962          * Check if the running thread is not the owner of the mutex.
 963          */
 964         if (__predict_false(PMUTEX_OWNER_ID(m) != id))
 965                 return (EPERM);
 966
 967         error = 0;
 968         if (__predict_false(PMUTEX_TYPE(m->m_flags) ==
 969             PTHREAD_MUTEX_RECURSIVE && m->m_count > 0)) {
 970                 m->m_count--;
 971         } else {
 972                 if ((m->m_flags & PMUTEX_FLAG_DEFERRED) != 0) {
 973                         deferred = 1;
 974                         m->m_flags &= ~PMUTEX_FLAG_DEFERRED;
 975                 } else
 976                         deferred = 0;
 977
 978                 robust = _mutex_enter_robust(curthread, m);
 979                 dequeue_mutex(curthread, m);
 980                 error = _thr_umutex_unlock2(&m->m_lock, id, mtx_defer);
 981                 if (deferred)  {
 982                         if (mtx_defer == NULL) {
 983                                 _thr_wake_all(curthread->defer_waiters,
 984                                     curthread->nwaiter_defer);
 985                                 curthread->nwaiter_defer = 0;
 986                         } else
 987                                 *mtx_defer = 1;
 988                 }
 989                 if (robust)
 990                         _mutex_leave_robust(curthread, m);
 991         }
 992         if (!cv && m->m_flags & PMUTEX_FLAG_PRIVATE)
 993                 THR_CRITICAL_LEAVE(curthread);
 994         return (error);
 995 }
 996
 997 int
 998 _pthread_mutex_getprioceiling(pthread_mutex_t *mutex,
 999     int *prioceiling)
1000 {
1001         struct pthread_mutex *m;
1002
1003         if (*mutex == THR_PSHARED_PTR) {
1004                 m = __thr_pshared_offpage(mutex, 0);
1005                 if (m == NULL)
1006                         return (EINVAL);
1007                 shared_mutex_init(m, NULL);
1008         } else {
1009                 m = *mutex;
1010                 if (m <= THR_MUTEX_DESTROYED)
1011                         return (EINVAL);
1012         }
1013         if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)
1014                 return (EINVAL);
1015         *prioceiling = m->m_lock.m_ceilings[0];
1016         return (0);
1017 }
1018
1019 int
1020 _pthread_mutex_setprioceiling(pthread_mutex_t *mutex,
1021     int ceiling, int *old_ceiling)
1022 {
1023         struct pthread *curthread;
1024         struct pthread_mutex *m, *m1, *m2;
1025         struct mutex_queue *q, *qp;
1026         int qidx, ret;
1027
1028         if (*mutex == THR_PSHARED_PTR) {
1029                 m = __thr_pshared_offpage(mutex, 0);
1030                 if (m == NULL)
1031                         return (EINVAL);
1032                 shared_mutex_init(m, NULL);
1033         } else {
1034                 m = *mutex;
1035                 if (m <= THR_MUTEX_DESTROYED)
1036                         return (EINVAL);
1037         }
1038         if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)
1039                 return (EINVAL);
1040
1041         ret = __thr_umutex_set_ceiling(&m->m_lock, ceiling, old_ceiling);
1042         if (ret != 0)
1043                 return (ret);
1044
1045         curthread = _get_curthread();
1046         if (PMUTEX_OWNER_ID(m) == TID(curthread)) {
1047                 mutex_assert_is_owned(m);
1048                 m1 = TAILQ_PREV(m, mutex_queue, m_qe);
1049                 m2 = TAILQ_NEXT(m, m_qe);
1050                 if ((m1 != NULL && m1->m_lock.m_ceilings[0] > (u_int)ceiling) ||
1051                     (m2 != NULL && m2->m_lock.m_ceilings[0] < (u_int)ceiling)) {
1052                         qidx = mutex_qidx(m);
1053                         q = &curthread->mq[qidx];
1054                         qp = &curthread->mq[qidx + 1];
1055                         TAILQ_REMOVE(q, m, m_qe);
1056                         if (!is_pshared_mutex(m))
1057                                 TAILQ_REMOVE(qp, m, m_pqe);
1058                         TAILQ_FOREACH(m2, q, m_qe) {
1059                                 if (m2->m_lock.m_ceilings[0] > (u_int)ceiling) {
1060                                         TAILQ_INSERT_BEFORE(m2, m, m_qe);
1061                                         if (!is_pshared_mutex(m)) {
1062                                                 while (m2 != NULL &&
1063                                                     is_pshared_mutex(m2)) {
1064                                                         m2 = TAILQ_PREV(m2,
1065                                                             mutex_queue, m_qe);
1066                                                 }
1067                                                 if (m2 == NULL) {
1068                                                         TAILQ_INSERT_HEAD(qp,
1069                                                             m, m_pqe);
1070                                                 } else {
1071                                                         TAILQ_INSERT_BEFORE(m2,
1072                                                             m, m_pqe);
1073                                                 }
1074                                         }
1075                                         return (0);
1076                                 }
1077                         }
1078                         TAILQ_INSERT_TAIL(q, m, m_qe);
1079                         if (!is_pshared_mutex(m))
1080                                 TAILQ_INSERT_TAIL(qp, m, m_pqe);
1081                 }
1082         }
1083         return (0);
1084 }
1085
1086 int
1087 _pthread_mutex_getspinloops_np(pthread_mutex_t *mutex, int *count)
1088 {
1089         struct pthread_mutex *m;
1090         int ret;
1091
1092         ret = check_and_init_mutex(mutex, &m);
1093         if (ret == 0)
1094                 *count = m->m_spinloops;
1095         return (ret);
1096 }
1097
1098 int
1099 __pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count)
1100 {
1101         struct pthread_mutex *m;
1102         int ret;
1103
1104         ret = check_and_init_mutex(mutex, &m);
1105         if (ret == 0)
1106                 m->m_spinloops = count;
1107         return (ret);
1108 }
1109
1110 int
1111 _pthread_mutex_getyieldloops_np(pthread_mutex_t *mutex, int *count)
1112 {
1113         struct pthread_mutex *m;
1114         int ret;
1115
1116         ret = check_and_init_mutex(mutex, &m);
1117         if (ret == 0)
1118                 *count = m->m_yieldloops;
1119         return (ret);
1120 }
1121
1122 int
1123 __pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count)
1124 {
1125         struct pthread_mutex *m;
1126         int ret;
1127
1128         ret = check_and_init_mutex(mutex, &m);
1129         if (ret == 0)
1130                 m->m_yieldloops = count;
1131         return (0);
1132 }
1133
1134 int
1135 _pthread_mutex_isowned_np(pthread_mutex_t *mutex)
1136 {
1137         struct pthread_mutex *m;
1138
1139         if (*mutex == THR_PSHARED_PTR) {
1140                 m = __thr_pshared_offpage(mutex, 0);
1141                 if (m == NULL)
1142                         return (0);
1143                 shared_mutex_init(m, NULL);
1144         } else {
1145                 m = *mutex;
1146                 if (m <= THR_MUTEX_DESTROYED)
1147                         return (0);
1148         }
1149         return (PMUTEX_OWNER_ID(m) == TID(_get_curthread()));
1150 }
1151
1152 int
1153 _mutex_owned(struct pthread *curthread, const struct pthread_mutex *mp)
1154 {
1155
1156         if (__predict_false(mp <= THR_MUTEX_DESTROYED)) {
1157                 if (mp == THR_MUTEX_DESTROYED)
1158                         return (EINVAL);
1159                 return (EPERM);
1160         }
1161         if (PMUTEX_OWNER_ID(mp) != TID(curthread))
1162                 return (EPERM);
1163         return (0);
1164 }
1165
1166 int
1167 _pthread_mutex_consistent(pthread_mutex_t *mutex)
1168 {
1169         struct pthread_mutex *m;
1170         struct pthread *curthread;
1171
1172         if (*mutex == THR_PSHARED_PTR) {
1173                 m = __thr_pshared_offpage(mutex, 0);
1174                 if (m == NULL)
1175                         return (EINVAL);
1176                 shared_mutex_init(m, NULL);
1177         } else {
1178                 m = *mutex;
1179                 if (m <= THR_MUTEX_DESTROYED)
1180                         return (EINVAL);
1181         }
1182         curthread = _get_curthread();
1183         if ((m->m_lock.m_flags & (UMUTEX_ROBUST | UMUTEX_NONCONSISTENT)) !=
1184             (UMUTEX_ROBUST | UMUTEX_NONCONSISTENT))
1185                 return (EINVAL);
1186         if (PMUTEX_OWNER_ID(m) != TID(curthread))
1187                 return (EPERM);
1188         m->m_lock.m_flags &= ~UMUTEX_NONCONSISTENT;
1189         return (0);
1190 }