sys/kern/kern_sx.c

   1 /*-
   2  * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org>
   3  * Copyright (c) 2001 Jason Evans <jasone@freebsd.org>
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice(s), this list of conditions and the following disclaimer as
  11  *    the first lines of this file unmodified other than the possible
  12  *    addition of one or more copyright notices.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice(s), this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
  18  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  19  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  20  * DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
  21  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  22  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  23  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  24  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
  27  * DAMAGE.
  28  */
  29
  30 /*
  31  * Shared/exclusive locks.  This implementation attempts to ensure
  32  * deterministic lock granting behavior, so that slocks and xlocks are
  33  * interleaved.
  34  *
  35  * Priority propagation will not generally raise the priority of lock holders,
  36  * so should not be relied upon in combination with sx locks.
  37  */
  38
  39 #include "opt_ddb.h"
  40 #include "opt_hwpmc_hooks.h"
  41 #include "opt_no_adaptive_sx.h"
  42
  43 #include <sys/cdefs.h>
  44 __FBSDID("$FreeBSD$");
  45
  46 #include <sys/param.h>
  47 #include <sys/systm.h>
  48 #include <sys/kdb.h>
  49 #include <sys/ktr.h>
  50 #include <sys/lock.h>
  51 #include <sys/mutex.h>
  52 #include <sys/proc.h>
  53 #include <sys/sched.h>
  54 #include <sys/sleepqueue.h>
  55 #include <sys/sx.h>
  56 #include <sys/sysctl.h>
  57
  58 #if defined(SMP) && !defined(NO_ADAPTIVE_SX)
  59 #include <machine/cpu.h>
  60 #endif
  61
  62 #ifdef DDB
  63 #include <ddb/ddb.h>
  64 #endif
  65
  66 #if defined(SMP) && !defined(NO_ADAPTIVE_SX)
  67 #define ADAPTIVE_SX
  68 #endif
  69
  70 CTASSERT((SX_NOADAPTIVE & LO_CLASSFLAGS) == SX_NOADAPTIVE);
  71
  72 #ifdef HWPMC_HOOKS
  73 #include <sys/pmckern.h>
  74 PMC_SOFT_DECLARE( , , lock, failed);
  75 #endif
  76
  77 /* Handy macros for sleep queues. */
  78 #define SQ_EXCLUSIVE_QUEUE      0
  79 #define SQ_SHARED_QUEUE         1
  80
  81 /*
  82  * Variations on DROP_GIANT()/PICKUP_GIANT() for use in this file.  We
  83  * drop Giant anytime we have to sleep or if we adaptively spin.
  84  */
  85 #define GIANT_DECLARE                                                   \
  86         int _giantcnt = 0;                                              \
  87         WITNESS_SAVE_DECL(Giant)                                        \
  88
  89 #define GIANT_SAVE() do {                                               \
  90         if (mtx_owned(&Giant)) {                                        \
  91                 WITNESS_SAVE(&Giant.lock_object, Giant);                \
  92                 while (mtx_owned(&Giant)) {                             \
  93                         _giantcnt++;                                    \
  94                         mtx_unlock(&Giant);                             \
  95                 }                                                       \
  96         }                                                               \
  97 } while (0)
  98
  99 #define GIANT_RESTORE() do {                                            \
 100         if (_giantcnt > 0) {                                            \
 101                 mtx_assert(&Giant, MA_NOTOWNED);                        \
 102                 while (_giantcnt--)                                     \
 103                         mtx_lock(&Giant);                               \
 104                 WITNESS_RESTORE(&Giant.lock_object, Giant);             \
 105         }                                                               \
 106 } while (0)
 107
 108 /*
 109  * Returns true if an exclusive lock is recursed.  It assumes
 110  * curthread currently has an exclusive lock.
 111  */
 112 #define sx_recursed(sx)         ((sx)->sx_recurse != 0)
 113
 114 static void     assert_sx(const struct lock_object *lock, int what);
 115 #ifdef DDB
 116 static void     db_show_sx(const struct lock_object *lock);
 117 #endif
 118 static void     lock_sx(struct lock_object *lock, uintptr_t how);
 119 #ifdef KDTRACE_HOOKS
 120 static int      owner_sx(const struct lock_object *lock, struct thread **owner);
 121 #endif
 122 static uintptr_t unlock_sx(struct lock_object *lock);
 123
 124 struct lock_class lock_class_sx = {
 125         .lc_name = "sx",
 126         .lc_flags = LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE | LC_UPGRADABLE,
 127         .lc_assert = assert_sx,
 128 #ifdef DDB
 129         .lc_ddb_show = db_show_sx,
 130 #endif
 131         .lc_lock = lock_sx,
 132         .lc_unlock = unlock_sx,
 133 #ifdef KDTRACE_HOOKS
 134         .lc_owner = owner_sx,
 135 #endif
 136 };
 137
 138 #ifndef INVARIANTS
 139 #define _sx_assert(sx, what, file, line)
 140 #endif
 141
 142 #ifdef ADAPTIVE_SX
 143 static u_int asx_retries = 10;
 144 static u_int asx_loops = 10000;
 145 static SYSCTL_NODE(_debug, OID_AUTO, sx, CTLFLAG_RD, NULL, "sxlock debugging");
 146 SYSCTL_UINT(_debug_sx, OID_AUTO, retries, CTLFLAG_RW, &asx_retries, 0, "");
 147 SYSCTL_UINT(_debug_sx, OID_AUTO, loops, CTLFLAG_RW, &asx_loops, 0, "");
 148 #endif
 149
 150 void
 151 assert_sx(const struct lock_object *lock, int what)
 152 {
 153
 154         sx_assert((const struct sx *)lock, what);
 155 }
 156
 157 void
 158 lock_sx(struct lock_object *lock, uintptr_t how)
 159 {
 160         struct sx *sx;
 161
 162         sx = (struct sx *)lock;
 163         if (how)
 164                 sx_slock(sx);
 165         else
 166                 sx_xlock(sx);
 167 }
 168
 169 uintptr_t
 170 unlock_sx(struct lock_object *lock)
 171 {
 172         struct sx *sx;
 173
 174         sx = (struct sx *)lock;
 175         sx_assert(sx, SA_LOCKED | SA_NOTRECURSED);
 176         if (sx_xlocked(sx)) {
 177                 sx_xunlock(sx);
 178                 return (0);
 179         } else {
 180                 sx_sunlock(sx);
 181                 return (1);
 182         }
 183 }
 184
 185 #ifdef KDTRACE_HOOKS
 186 int
 187 owner_sx(const struct lock_object *lock, struct thread **owner)
 188 {
 189         const struct sx *sx = (const struct sx *)lock;
 190         uintptr_t x = sx->sx_lock;
 191
 192         *owner = (struct thread *)SX_OWNER(x);
 193         return ((x & SX_LOCK_SHARED) != 0 ? (SX_SHARERS(x) != 0) :
 194             (*owner != NULL));
 195 }
 196 #endif
 197
 198 void
 199 sx_sysinit(void *arg)
 200 {
 201         struct sx_args *sargs = arg;
 202
 203         sx_init_flags(sargs->sa_sx, sargs->sa_desc, sargs->sa_flags);
 204 }
 205
 206 void
 207 sx_init_flags(struct sx *sx, const char *description, int opts)
 208 {
 209         int flags;
 210
 211         MPASS((opts & ~(SX_QUIET | SX_RECURSE | SX_NOWITNESS | SX_DUPOK |
 212             SX_NOPROFILE | SX_NOADAPTIVE | SX_NEW)) == 0);
 213         ASSERT_ATOMIC_LOAD_PTR(sx->sx_lock,
 214             ("%s: sx_lock not aligned for %s: %p", __func__, description,
 215             &sx->sx_lock));
 216
 217         flags = LO_SLEEPABLE | LO_UPGRADABLE;
 218         if (opts & SX_DUPOK)
 219                 flags |= LO_DUPOK;
 220         if (opts & SX_NOPROFILE)
 221                 flags |= LO_NOPROFILE;
 222         if (!(opts & SX_NOWITNESS))
 223                 flags |= LO_WITNESS;
 224         if (opts & SX_RECURSE)
 225                 flags |= LO_RECURSABLE;
 226         if (opts & SX_QUIET)
 227                 flags |= LO_QUIET;
 228         if (opts & SX_NEW)
 229                 flags |= LO_NEW;
 230
 231         flags |= opts & SX_NOADAPTIVE;
 232         lock_init(&sx->lock_object, &lock_class_sx, description, NULL, flags);
 233         sx->sx_lock = SX_LOCK_UNLOCKED;
 234         sx->sx_recurse = 0;
 235 }
 236
 237 void
 238 sx_destroy(struct sx *sx)
 239 {
 240
 241         KASSERT(sx->sx_lock == SX_LOCK_UNLOCKED, ("sx lock still held"));
 242         KASSERT(sx->sx_recurse == 0, ("sx lock still recursed"));
 243         sx->sx_lock = SX_LOCK_DESTROYED;
 244         lock_destroy(&sx->lock_object);
 245 }
 246
 247 int
 248 _sx_slock(struct sx *sx, int opts, const char *file, int line)
 249 {
 250         int error = 0;
 251
 252         if (SCHEDULER_STOPPED())
 253                 return (0);
 254         KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
 255             ("sx_slock() by idle thread %p on sx %s @ %s:%d",
 256             curthread, sx->lock_object.lo_name, file, line));
 257         KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
 258             ("sx_slock() of destroyed sx @ %s:%d", file, line));
 259         WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER, file, line, NULL);
 260         error = __sx_slock(sx, opts, file, line);
 261         if (!error) {
 262                 LOCK_LOG_LOCK("SLOCK", &sx->lock_object, 0, 0, file, line);
 263                 WITNESS_LOCK(&sx->lock_object, 0, file, line);
 264                 curthread->td_locks++;
 265         }
 266
 267         return (error);
 268 }
 269
 270 int
 271 sx_try_slock_(struct sx *sx, const char *file, int line)
 272 {
 273         uintptr_t x;
 274
 275         if (SCHEDULER_STOPPED())
 276                 return (1);
 277
 278         KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
 279             ("sx_try_slock() by idle thread %p on sx %s @ %s:%d",
 280             curthread, sx->lock_object.lo_name, file, line));
 281
 282         for (;;) {
 283                 x = sx->sx_lock;
 284                 KASSERT(x != SX_LOCK_DESTROYED,
 285                     ("sx_try_slock() of destroyed sx @ %s:%d", file, line));
 286                 if (!(x & SX_LOCK_SHARED))
 287                         break;
 288                 if (atomic_cmpset_acq_ptr(&sx->sx_lock, x, x + SX_ONE_SHARER)) {
 289                         LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 1, file, line);
 290                         WITNESS_LOCK(&sx->lock_object, LOP_TRYLOCK, file, line);
 291                         curthread->td_locks++;
 292                         return (1);
 293                 }
 294         }
 295
 296         LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 0, file, line);
 297         return (0);
 298 }
 299
 300 int
 301 _sx_xlock(struct sx *sx, int opts, const char *file, int line)
 302 {
 303         int error = 0;
 304
 305         if (SCHEDULER_STOPPED())
 306                 return (0);
 307         KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
 308             ("sx_xlock() by idle thread %p on sx %s @ %s:%d",
 309             curthread, sx->lock_object.lo_name, file, line));
 310         KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
 311             ("sx_xlock() of destroyed sx @ %s:%d", file, line));
 312         WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
 313             line, NULL);
 314         error = __sx_xlock(sx, curthread, opts, file, line);
 315         if (!error) {
 316                 LOCK_LOG_LOCK("XLOCK", &sx->lock_object, 0, sx->sx_recurse,
 317                     file, line);
 318                 WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line);
 319                 curthread->td_locks++;
 320         }
 321
 322         return (error);
 323 }
 324
 325 int
 326 sx_try_xlock_(struct sx *sx, const char *file, int line)
 327 {
 328         int rval;
 329
 330         if (SCHEDULER_STOPPED())
 331                 return (1);
 332
 333         KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
 334             ("sx_try_xlock() by idle thread %p on sx %s @ %s:%d",
 335             curthread, sx->lock_object.lo_name, file, line));
 336         KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
 337             ("sx_try_xlock() of destroyed sx @ %s:%d", file, line));
 338
 339         if (sx_xlocked(sx) &&
 340             (sx->lock_object.lo_flags & LO_RECURSABLE) != 0) {
 341                 sx->sx_recurse++;
 342                 atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
 343                 rval = 1;
 344         } else
 345                 rval = atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED,
 346                     (uintptr_t)curthread);
 347         LOCK_LOG_TRY("XLOCK", &sx->lock_object, 0, rval, file, line);
 348         if (rval) {
 349                 WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
 350                     file, line);
 351                 curthread->td_locks++;
 352         }
 353
 354         return (rval);
 355 }
 356
 357 void
 358 _sx_sunlock(struct sx *sx, const char *file, int line)
 359 {
 360
 361         if (SCHEDULER_STOPPED())
 362                 return;
 363         KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
 364             ("sx_sunlock() of destroyed sx @ %s:%d", file, line));
 365         _sx_assert(sx, SA_SLOCKED, file, line);
 366         WITNESS_UNLOCK(&sx->lock_object, 0, file, line);
 367         LOCK_LOG_LOCK("SUNLOCK", &sx->lock_object, 0, 0, file, line);
 368         __sx_sunlock(sx, file, line);
 369         curthread->td_locks--;
 370 }
 371
 372 void
 373 _sx_xunlock(struct sx *sx, const char *file, int line)
 374 {
 375
 376         if (SCHEDULER_STOPPED())
 377                 return;
 378         KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
 379             ("sx_xunlock() of destroyed sx @ %s:%d", file, line));
 380         _sx_assert(sx, SA_XLOCKED, file, line);
 381         WITNESS_UNLOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line);
 382         LOCK_LOG_LOCK("XUNLOCK", &sx->lock_object, 0, sx->sx_recurse, file,
 383             line);
 384         __sx_xunlock(sx, curthread, file, line);
 385         curthread->td_locks--;
 386 }
 387
 388 /*
 389  * Try to do a non-blocking upgrade from a shared lock to an exclusive lock.
 390  * This will only succeed if this thread holds a single shared lock.
 391  * Return 1 if if the upgrade succeed, 0 otherwise.
 392  */
 393 int
 394 sx_try_upgrade_(struct sx *sx, const char *file, int line)
 395 {
 396         uintptr_t x;
 397         int success;
 398
 399         if (SCHEDULER_STOPPED())
 400                 return (1);
 401
 402         KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
 403             ("sx_try_upgrade() of destroyed sx @ %s:%d", file, line));
 404         _sx_assert(sx, SA_SLOCKED, file, line);
 405
 406         /*
 407          * Try to switch from one shared lock to an exclusive lock.  We need
 408          * to maintain the SX_LOCK_EXCLUSIVE_WAITERS flag if set so that
 409          * we will wake up the exclusive waiters when we drop the lock.
 410          */
 411         x = sx->sx_lock & SX_LOCK_EXCLUSIVE_WAITERS;
 412         success = atomic_cmpset_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) | x,
 413             (uintptr_t)curthread | x);
 414         LOCK_LOG_TRY("XUPGRADE", &sx->lock_object, 0, success, file, line);
 415         if (success) {
 416                 WITNESS_UPGRADE(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
 417                     file, line);
 418                 LOCKSTAT_RECORD0(LS_SX_TRYUPGRADE_UPGRADE, sx);
 419         }
 420         return (success);
 421 }
 422
 423 /*
 424  * Downgrade an unrecursed exclusive lock into a single shared lock.
 425  */
 426 void
 427 sx_downgrade_(struct sx *sx, const char *file, int line)
 428 {
 429         uintptr_t x;
 430         int wakeup_swapper;
 431
 432         if (SCHEDULER_STOPPED())
 433                 return;
 434
 435         KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
 436             ("sx_downgrade() of destroyed sx @ %s:%d", file, line));
 437         _sx_assert(sx, SA_XLOCKED | SA_NOTRECURSED, file, line);
 438 #ifndef INVARIANTS
 439         if (sx_recursed(sx))
 440                 panic("downgrade of a recursed lock");
 441 #endif
 442
 443         WITNESS_DOWNGRADE(&sx->lock_object, 0, file, line);
 444
 445         /*
 446          * Try to switch from an exclusive lock with no shared waiters
 447          * to one sharer with no shared waiters.  If there are
 448          * exclusive waiters, we don't need to lock the sleep queue so
 449          * long as we preserve the flag.  We do one quick try and if
 450          * that fails we grab the sleepq lock to keep the flags from
 451          * changing and do it the slow way.
 452          *
 453          * We have to lock the sleep queue if there are shared waiters
 454          * so we can wake them up.
 455          */
 456         x = sx->sx_lock;
 457         if (!(x & SX_LOCK_SHARED_WAITERS) &&
 458             atomic_cmpset_rel_ptr(&sx->sx_lock, x, SX_SHARERS_LOCK(1) |
 459             (x & SX_LOCK_EXCLUSIVE_WAITERS))) {
 460                 LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line);
 461                 return;
 462         }
 463
 464         /*
 465          * Lock the sleep queue so we can read the waiters bits
 466          * without any races and wakeup any shared waiters.
 467          */
 468         sleepq_lock(&sx->lock_object);
 469
 470         /*
 471          * Preserve SX_LOCK_EXCLUSIVE_WAITERS while downgraded to a single
 472          * shared lock.  If there are any shared waiters, wake them up.
 473          */
 474         wakeup_swapper = 0;
 475         x = sx->sx_lock;
 476         atomic_store_rel_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) |
 477             (x & SX_LOCK_EXCLUSIVE_WAITERS));
 478         if (x & SX_LOCK_SHARED_WAITERS)
 479                 wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX,
 480                     0, SQ_SHARED_QUEUE);
 481         sleepq_release(&sx->lock_object);
 482
 483         LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line);
 484         LOCKSTAT_RECORD0(LS_SX_DOWNGRADE_DOWNGRADE, sx);
 485
 486         if (wakeup_swapper)
 487                 kick_proc0();
 488 }
 489
 490 /*
 491  * This function represents the so-called 'hard case' for sx_xlock
 492  * operation.  All 'easy case' failures are redirected to this.  Note
 493  * that ideally this would be a static function, but it needs to be
 494  * accessible from at least sx.h.
 495  */
 496 int
 497 _sx_xlock_hard(struct sx *sx, uintptr_t tid, int opts, const char *file,
 498     int line)
 499 {
 500         GIANT_DECLARE;
 501 #ifdef ADAPTIVE_SX
 502         volatile struct thread *owner;
 503         u_int i, spintries = 0;
 504 #endif
 505         uintptr_t x;
 506 #ifdef LOCK_PROFILING
 507         uint64_t waittime = 0;
 508         int contested = 0;
 509 #endif
 510         int error = 0;
 511 #ifdef  KDTRACE_HOOKS
 512         uint64_t spin_cnt = 0;
 513         uint64_t sleep_cnt = 0;
 514         int64_t sleep_time = 0;
 515 #endif
 516
 517         if (SCHEDULER_STOPPED())
 518                 return (0);
 519
 520         /* If we already hold an exclusive lock, then recurse. */
 521         if (sx_xlocked(sx)) {
 522                 KASSERT((sx->lock_object.lo_flags & LO_RECURSABLE) != 0,
 523             ("_sx_xlock_hard: recursed on non-recursive sx %s @ %s:%d\n",
 524                     sx->lock_object.lo_name, file, line));
 525                 sx->sx_recurse++;
 526                 atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
 527                 if (LOCK_LOG_TEST(&sx->lock_object, 0))
 528                         CTR2(KTR_LOCK, "%s: %p recursing", __func__, sx);
 529                 return (0);
 530         }
 531
 532         if (LOCK_LOG_TEST(&sx->lock_object, 0))
 533                 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__,
 534                     sx->lock_object.lo_name, (void *)sx->sx_lock, file, line);
 535
 536         while (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) {
 537 #ifdef KDTRACE_HOOKS
 538                 spin_cnt++;
 539 #endif
 540 #ifdef HWPMC_HOOKS
 541                 PMC_SOFT_CALL( , , lock, failed);
 542 #endif
 543                 lock_profile_obtain_lock_failed(&sx->lock_object, &contested,
 544                     &waittime);
 545 #ifdef ADAPTIVE_SX
 546                 /*
 547                  * If the lock is write locked and the owner is
 548                  * running on another CPU, spin until the owner stops
 549                  * running or the state of the lock changes.
 550                  */
 551                 x = sx->sx_lock;
 552                 if ((sx->lock_object.lo_flags & SX_NOADAPTIVE) == 0) {
 553                         if ((x & SX_LOCK_SHARED) == 0) {
 554                                 x = SX_OWNER(x);
 555                                 owner = (struct thread *)x;
 556                                 if (TD_IS_RUNNING(owner)) {
 557                                         if (LOCK_LOG_TEST(&sx->lock_object, 0))
 558                                                 CTR3(KTR_LOCK,
 559                                             "%s: spinning on %p held by %p",
 560                                                     __func__, sx, owner);
 561                                         KTR_STATE1(KTR_SCHED, "thread",
 562                                             sched_tdname(curthread), "spinning",
 563                                             "lockname:\"%s\"",
 564                                             sx->lock_object.lo_name);
 565                                         GIANT_SAVE();
 566                                         while (SX_OWNER(sx->sx_lock) == x &&
 567                                             TD_IS_RUNNING(owner)) {
 568                                                 cpu_spinwait();
 569 #ifdef KDTRACE_HOOKS
 570                                                 spin_cnt++;
 571 #endif
 572                                         }
 573                                         KTR_STATE0(KTR_SCHED, "thread",
 574                                             sched_tdname(curthread), "running");
 575                                         continue;
 576                                 }
 577                         } else if (SX_SHARERS(x) && spintries < asx_retries) {
 578                                 KTR_STATE1(KTR_SCHED, "thread",
 579                                     sched_tdname(curthread), "spinning",
 580                                     "lockname:\"%s\"", sx->lock_object.lo_name);
 581                                 GIANT_SAVE();
 582                                 spintries++;
 583                                 for (i = 0; i < asx_loops; i++) {
 584                                         if (LOCK_LOG_TEST(&sx->lock_object, 0))
 585                                                 CTR4(KTR_LOCK,
 586                                     "%s: shared spinning on %p with %u and %u",
 587                                                     __func__, sx, spintries, i);
 588                                         x = sx->sx_lock;
 589                                         if ((x & SX_LOCK_SHARED) == 0 ||
 590                                             SX_SHARERS(x) == 0)
 591                                                 break;
 592                                         cpu_spinwait();
 593 #ifdef KDTRACE_HOOKS
 594                                         spin_cnt++;
 595 #endif
 596                                 }
 597                                 KTR_STATE0(KTR_SCHED, "thread",
 598                                     sched_tdname(curthread), "running");
 599                                 if (i != asx_loops)
 600                                         continue;
 601                         }
 602                 }
 603 #endif
 604
 605                 sleepq_lock(&sx->lock_object);
 606                 x = sx->sx_lock;
 607
 608                 /*
 609                  * If the lock was released while spinning on the
 610                  * sleep queue chain lock, try again.
 611                  */
 612                 if (x == SX_LOCK_UNLOCKED) {
 613                         sleepq_release(&sx->lock_object);
 614                         continue;
 615                 }
 616
 617 #ifdef ADAPTIVE_SX
 618                 /*
 619                  * The current lock owner might have started executing
 620                  * on another CPU (or the lock could have changed
 621                  * owners) while we were waiting on the sleep queue
 622                  * chain lock.  If so, drop the sleep queue lock and try
 623                  * again.
 624                  */
 625                 if (!(x & SX_LOCK_SHARED) &&
 626                     (sx->lock_object.lo_flags & SX_NOADAPTIVE) == 0) {
 627                         owner = (struct thread *)SX_OWNER(x);
 628                         if (TD_IS_RUNNING(owner)) {
 629                                 sleepq_release(&sx->lock_object);
 630                                 continue;
 631                         }
 632                 }
 633 #endif
 634
 635                 /*
 636                  * If an exclusive lock was released with both shared
 637                  * and exclusive waiters and a shared waiter hasn't
 638                  * woken up and acquired the lock yet, sx_lock will be
 639                  * set to SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS.
 640                  * If we see that value, try to acquire it once.  Note
 641                  * that we have to preserve SX_LOCK_EXCLUSIVE_WAITERS
 642                  * as there are other exclusive waiters still.  If we
 643                  * fail, restart the loop.
 644                  */
 645                 if (x == (SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS)) {
 646                         if (atomic_cmpset_acq_ptr(&sx->sx_lock,
 647                             SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS,
 648                             tid | SX_LOCK_EXCLUSIVE_WAITERS)) {
 649                                 sleepq_release(&sx->lock_object);
 650                                 CTR2(KTR_LOCK, "%s: %p claimed by new writer",
 651                                     __func__, sx);
 652                                 break;
 653                         }
 654                         sleepq_release(&sx->lock_object);
 655                         continue;
 656                 }
 657
 658                 /*
 659                  * Try to set the SX_LOCK_EXCLUSIVE_WAITERS.  If we fail,
 660                  * than loop back and retry.
 661                  */
 662                 if (!(x & SX_LOCK_EXCLUSIVE_WAITERS)) {
 663                         if (!atomic_cmpset_ptr(&sx->sx_lock, x,
 664                             x | SX_LOCK_EXCLUSIVE_WAITERS)) {
 665                                 sleepq_release(&sx->lock_object);
 666                                 continue;
 667                         }
 668                         if (LOCK_LOG_TEST(&sx->lock_object, 0))
 669                                 CTR2(KTR_LOCK, "%s: %p set excl waiters flag",
 670                                     __func__, sx);
 671                 }
 672
 673                 /*
 674                  * Since we have been unable to acquire the exclusive
 675                  * lock and the exclusive waiters flag is set, we have
 676                  * to sleep.
 677                  */
 678                 if (LOCK_LOG_TEST(&sx->lock_object, 0))
 679                         CTR2(KTR_LOCK, "%s: %p blocking on sleep queue",
 680                             __func__, sx);
 681
 682 #ifdef KDTRACE_HOOKS
 683                 sleep_time -= lockstat_nsecs();
 684 #endif
 685                 GIANT_SAVE();
 686                 sleepq_add(&sx->lock_object, NULL, sx->lock_object.lo_name,
 687                     SLEEPQ_SX | ((opts & SX_INTERRUPTIBLE) ?
 688                     SLEEPQ_INTERRUPTIBLE : 0), SQ_EXCLUSIVE_QUEUE);
 689                 if (!(opts & SX_INTERRUPTIBLE))
 690                         sleepq_wait(&sx->lock_object, 0);
 691                 else
 692                         error = sleepq_wait_sig(&sx->lock_object, 0);
 693 #ifdef KDTRACE_HOOKS
 694                 sleep_time += lockstat_nsecs();
 695                 sleep_cnt++;
 696 #endif
 697                 if (error) {
 698                         if (LOCK_LOG_TEST(&sx->lock_object, 0))
 699                                 CTR2(KTR_LOCK,
 700                         "%s: interruptible sleep by %p suspended by signal",
 701                                     __func__, sx);
 702                         break;
 703                 }
 704                 if (LOCK_LOG_TEST(&sx->lock_object, 0))
 705                         CTR2(KTR_LOCK, "%s: %p resuming from sleep queue",
 706                             __func__, sx);
 707         }
 708
 709         GIANT_RESTORE();
 710         if (!error)
 711                 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_XLOCK_ACQUIRE, sx,
 712                     contested, waittime, file, line);
 713 #ifdef KDTRACE_HOOKS
 714         if (sleep_time)
 715                 LOCKSTAT_RECORD1(LS_SX_XLOCK_BLOCK, sx, sleep_time);
 716         if (spin_cnt > sleep_cnt)
 717                 LOCKSTAT_RECORD1(LS_SX_XLOCK_SPIN, sx, (spin_cnt - sleep_cnt));
 718 #endif
 719         return (error);
 720 }
 721
 722 /*
 723  * This function represents the so-called 'hard case' for sx_xunlock
 724  * operation.  All 'easy case' failures are redirected to this.  Note
 725  * that ideally this would be a static function, but it needs to be
 726  * accessible from at least sx.h.
 727  */
 728 void
 729 _sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line)
 730 {
 731         uintptr_t x;
 732         int queue, wakeup_swapper;
 733
 734         if (SCHEDULER_STOPPED())
 735                 return;
 736
 737         MPASS(!(sx->sx_lock & SX_LOCK_SHARED));
 738
 739         /* If the lock is recursed, then unrecurse one level. */
 740         if (sx_xlocked(sx) && sx_recursed(sx)) {
 741                 if ((--sx->sx_recurse) == 0)
 742                         atomic_clear_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
 743                 if (LOCK_LOG_TEST(&sx->lock_object, 0))
 744                         CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, sx);
 745                 return;
 746         }
 747         MPASS(sx->sx_lock & (SX_LOCK_SHARED_WAITERS |
 748             SX_LOCK_EXCLUSIVE_WAITERS));
 749         if (LOCK_LOG_TEST(&sx->lock_object, 0))
 750                 CTR2(KTR_LOCK, "%s: %p contested", __func__, sx);
 751
 752         sleepq_lock(&sx->lock_object);
 753         x = SX_LOCK_UNLOCKED;
 754
 755         /*
 756          * The wake up algorithm here is quite simple and probably not
 757          * ideal.  It gives precedence to shared waiters if they are
 758          * present.  For this condition, we have to preserve the
 759          * state of the exclusive waiters flag.
 760          * If interruptible sleeps left the shared queue empty avoid a
 761          * starvation for the threads sleeping on the exclusive queue by giving
 762          * them precedence and cleaning up the shared waiters bit anyway.
 763          */
 764         if ((sx->sx_lock & SX_LOCK_SHARED_WAITERS) != 0 &&
 765             sleepq_sleepcnt(&sx->lock_object, SQ_SHARED_QUEUE) != 0) {
 766                 queue = SQ_SHARED_QUEUE;
 767                 x |= (sx->sx_lock & SX_LOCK_EXCLUSIVE_WAITERS);
 768         } else
 769                 queue = SQ_EXCLUSIVE_QUEUE;
 770
 771         /* Wake up all the waiters for the specific queue. */
 772         if (LOCK_LOG_TEST(&sx->lock_object, 0))
 773                 CTR3(KTR_LOCK, "%s: %p waking up all threads on %s queue",
 774                     __func__, sx, queue == SQ_SHARED_QUEUE ? "shared" :
 775                     "exclusive");
 776         atomic_store_rel_ptr(&sx->sx_lock, x);
 777         wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0,
 778             queue);
 779         sleepq_release(&sx->lock_object);
 780         if (wakeup_swapper)
 781                 kick_proc0();
 782 }
 783
 784 /*
 785  * This function represents the so-called 'hard case' for sx_slock
 786  * operation.  All 'easy case' failures are redirected to this.  Note
 787  * that ideally this would be a static function, but it needs to be
 788  * accessible from at least sx.h.
 789  */
 790 int
 791 _sx_slock_hard(struct sx *sx, int opts, const char *file, int line)
 792 {
 793         GIANT_DECLARE;
 794 #ifdef ADAPTIVE_SX
 795         volatile struct thread *owner;
 796 #endif
 797 #ifdef LOCK_PROFILING
 798         uint64_t waittime = 0;
 799         int contested = 0;
 800 #endif
 801         uintptr_t x;
 802         int error = 0;
 803 #ifdef KDTRACE_HOOKS
 804         uint64_t spin_cnt = 0;
 805         uint64_t sleep_cnt = 0;
 806         int64_t sleep_time = 0;
 807 #endif
 808
 809         if (SCHEDULER_STOPPED())
 810                 return (0);
 811
 812         /*
 813          * As with rwlocks, we don't make any attempt to try to block
 814          * shared locks once there is an exclusive waiter.
 815          */
 816         for (;;) {
 817 #ifdef KDTRACE_HOOKS
 818                 spin_cnt++;
 819 #endif
 820                 x = sx->sx_lock;
 821
 822                 /*
 823                  * If no other thread has an exclusive lock then try to bump up
 824                  * the count of sharers.  Since we have to preserve the state
 825                  * of SX_LOCK_EXCLUSIVE_WAITERS, if we fail to acquire the
 826                  * shared lock loop back and retry.
 827                  */
 828                 if (x & SX_LOCK_SHARED) {
 829                         MPASS(!(x & SX_LOCK_SHARED_WAITERS));
 830                         if (atomic_cmpset_acq_ptr(&sx->sx_lock, x,
 831                             x + SX_ONE_SHARER)) {
 832                                 if (LOCK_LOG_TEST(&sx->lock_object, 0))
 833                                         CTR4(KTR_LOCK,
 834                                             "%s: %p succeed %p -> %p", __func__,
 835                                             sx, (void *)x,
 836                                             (void *)(x + SX_ONE_SHARER));
 837                                 break;
 838                         }
 839                         continue;
 840                 }
 841 #ifdef HWPMC_HOOKS
 842                 PMC_SOFT_CALL( , , lock, failed);
 843 #endif
 844                 lock_profile_obtain_lock_failed(&sx->lock_object, &contested,
 845                     &waittime);
 846
 847 #ifdef ADAPTIVE_SX
 848                 /*
 849                  * If the owner is running on another CPU, spin until
 850                  * the owner stops running or the state of the lock
 851                  * changes.
 852                  */
 853                 if ((sx->lock_object.lo_flags & SX_NOADAPTIVE) == 0) {
 854                         x = SX_OWNER(x);
 855                         owner = (struct thread *)x;
 856                         if (TD_IS_RUNNING(owner)) {
 857                                 if (LOCK_LOG_TEST(&sx->lock_object, 0))
 858                                         CTR3(KTR_LOCK,
 859                                             "%s: spinning on %p held by %p",
 860                                             __func__, sx, owner);
 861                                 KTR_STATE1(KTR_SCHED, "thread",
 862                                     sched_tdname(curthread), "spinning",
 863                                     "lockname:\"%s\"", sx->lock_object.lo_name);
 864                                 GIANT_SAVE();
 865                                 while (SX_OWNER(sx->sx_lock) == x &&
 866                                     TD_IS_RUNNING(owner)) {
 867 #ifdef KDTRACE_HOOKS
 868                                         spin_cnt++;
 869 #endif
 870                                         cpu_spinwait();
 871                                 }
 872                                 KTR_STATE0(KTR_SCHED, "thread",
 873                                     sched_tdname(curthread), "running");
 874                                 continue;
 875                         }
 876                 }
 877 #endif
 878
 879                 /*
 880                  * Some other thread already has an exclusive lock, so
 881                  * start the process of blocking.
 882                  */
 883                 sleepq_lock(&sx->lock_object);
 884                 x = sx->sx_lock;
 885
 886                 /*
 887                  * The lock could have been released while we spun.
 888                  * In this case loop back and retry.
 889                  */
 890                 if (x & SX_LOCK_SHARED) {
 891                         sleepq_release(&sx->lock_object);
 892                         continue;
 893                 }
 894
 895 #ifdef ADAPTIVE_SX
 896                 /*
 897                  * If the owner is running on another CPU, spin until
 898                  * the owner stops running or the state of the lock
 899                  * changes.
 900                  */
 901                 if (!(x & SX_LOCK_SHARED) &&
 902                     (sx->lock_object.lo_flags & SX_NOADAPTIVE) == 0) {
 903                         owner = (struct thread *)SX_OWNER(x);
 904                         if (TD_IS_RUNNING(owner)) {
 905                                 sleepq_release(&sx->lock_object);
 906                                 continue;
 907                         }
 908                 }
 909 #endif
 910
 911                 /*
 912                  * Try to set the SX_LOCK_SHARED_WAITERS flag.  If we
 913                  * fail to set it drop the sleep queue lock and loop
 914                  * back.
 915                  */
 916                 if (!(x & SX_LOCK_SHARED_WAITERS)) {
 917                         if (!atomic_cmpset_ptr(&sx->sx_lock, x,
 918                             x | SX_LOCK_SHARED_WAITERS)) {
 919                                 sleepq_release(&sx->lock_object);
 920                                 continue;
 921                         }
 922                         if (LOCK_LOG_TEST(&sx->lock_object, 0))
 923                                 CTR2(KTR_LOCK, "%s: %p set shared waiters flag",
 924                                     __func__, sx);
 925                 }
 926
 927                 /*
 928                  * Since we have been unable to acquire the shared lock,
 929                  * we have to sleep.
 930                  */
 931                 if (LOCK_LOG_TEST(&sx->lock_object, 0))
 932                         CTR2(KTR_LOCK, "%s: %p blocking on sleep queue",
 933                             __func__, sx);
 934
 935 #ifdef KDTRACE_HOOKS
 936                 sleep_time -= lockstat_nsecs();
 937 #endif
 938                 GIANT_SAVE();
 939                 sleepq_add(&sx->lock_object, NULL, sx->lock_object.lo_name,
 940                     SLEEPQ_SX | ((opts & SX_INTERRUPTIBLE) ?
 941                     SLEEPQ_INTERRUPTIBLE : 0), SQ_SHARED_QUEUE);
 942                 if (!(opts & SX_INTERRUPTIBLE))
 943                         sleepq_wait(&sx->lock_object, 0);
 944                 else
 945                         error = sleepq_wait_sig(&sx->lock_object, 0);
 946 #ifdef KDTRACE_HOOKS
 947                 sleep_time += lockstat_nsecs();
 948                 sleep_cnt++;
 949 #endif
 950                 if (error) {
 951                         if (LOCK_LOG_TEST(&sx->lock_object, 0))
 952                                 CTR2(KTR_LOCK,
 953                         "%s: interruptible sleep by %p suspended by signal",
 954                                     __func__, sx);
 955                         break;
 956                 }
 957                 if (LOCK_LOG_TEST(&sx->lock_object, 0))
 958                         CTR2(KTR_LOCK, "%s: %p resuming from sleep queue",
 959                             __func__, sx);
 960         }
 961         if (error == 0)
 962                 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_SLOCK_ACQUIRE, sx,
 963                     contested, waittime, file, line);
 964 #ifdef KDTRACE_HOOKS
 965         if (sleep_time)
 966                 LOCKSTAT_RECORD1(LS_SX_XLOCK_BLOCK, sx, sleep_time);
 967         if (spin_cnt > sleep_cnt)
 968                 LOCKSTAT_RECORD1(LS_SX_XLOCK_SPIN, sx, (spin_cnt - sleep_cnt));
 969 #endif
 970         GIANT_RESTORE();
 971         return (error);
 972 }
 973
 974 /*
 975  * This function represents the so-called 'hard case' for sx_sunlock
 976  * operation.  All 'easy case' failures are redirected to this.  Note
 977  * that ideally this would be a static function, but it needs to be
 978  * accessible from at least sx.h.
 979  */
 980 void
 981 _sx_sunlock_hard(struct sx *sx, const char *file, int line)
 982 {
 983         uintptr_t x;
 984         int wakeup_swapper;
 985
 986         if (SCHEDULER_STOPPED())
 987                 return;
 988
 989         for (;;) {
 990                 x = sx->sx_lock;
 991
 992                 /*
 993                  * We should never have sharers while at least one thread
 994                  * holds a shared lock.
 995                  */
 996                 KASSERT(!(x & SX_LOCK_SHARED_WAITERS),
 997                     ("%s: waiting sharers", __func__));
 998
 999                 /*
1000                  * See if there is more than one shared lock held.  If
1001                  * so, just drop one and return.
1002                  */
1003                 if (SX_SHARERS(x) > 1) {
1004                         if (atomic_cmpset_rel_ptr(&sx->sx_lock, x,
1005                             x - SX_ONE_SHARER)) {
1006                                 if (LOCK_LOG_TEST(&sx->lock_object, 0))
1007                                         CTR4(KTR_LOCK,
1008                                             "%s: %p succeeded %p -> %p",
1009                                             __func__, sx, (void *)x,
1010                                             (void *)(x - SX_ONE_SHARER));
1011                                 break;
1012                         }
1013                         continue;
1014                 }
1015
1016                 /*
1017                  * If there aren't any waiters for an exclusive lock,
1018                  * then try to drop it quickly.
1019                  */
1020                 if (!(x & SX_LOCK_EXCLUSIVE_WAITERS)) {
1021                         MPASS(x == SX_SHARERS_LOCK(1));
1022                         if (atomic_cmpset_rel_ptr(&sx->sx_lock,
1023                             SX_SHARERS_LOCK(1), SX_LOCK_UNLOCKED)) {
1024                                 if (LOCK_LOG_TEST(&sx->lock_object, 0))
1025                                         CTR2(KTR_LOCK, "%s: %p last succeeded",
1026                                             __func__, sx);
1027                                 break;
1028                         }
1029                         continue;
1030                 }
1031
1032                 /*
1033                  * At this point, there should just be one sharer with
1034                  * exclusive waiters.
1035                  */
1036                 MPASS(x == (SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS));
1037
1038                 sleepq_lock(&sx->lock_object);
1039
1040                 /*
1041                  * Wake up semantic here is quite simple:
1042                  * Just wake up all the exclusive waiters.
1043                  * Note that the state of the lock could have changed,
1044                  * so if it fails loop back and retry.
1045                  */
1046                 if (!atomic_cmpset_rel_ptr(&sx->sx_lock,
1047                     SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS,
1048                     SX_LOCK_UNLOCKED)) {
1049                         sleepq_release(&sx->lock_object);
1050                         continue;
1051                 }
1052                 if (LOCK_LOG_TEST(&sx->lock_object, 0))
1053                         CTR2(KTR_LOCK, "%s: %p waking up all thread on"
1054                             "exclusive queue", __func__, sx);
1055                 wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX,
1056                     0, SQ_EXCLUSIVE_QUEUE);
1057                 sleepq_release(&sx->lock_object);
1058                 if (wakeup_swapper)
1059                         kick_proc0();
1060                 break;
1061         }
1062 }
1063
1064 #ifdef INVARIANT_SUPPORT
1065 #ifndef INVARIANTS
1066 #undef  _sx_assert
1067 #endif
1068
1069 /*
1070  * In the non-WITNESS case, sx_assert() can only detect that at least
1071  * *some* thread owns an slock, but it cannot guarantee that *this*
1072  * thread owns an slock.
1073  */
1074 void
1075 _sx_assert(const struct sx *sx, int what, const char *file, int line)
1076 {
1077 #ifndef WITNESS
1078         int slocked = 0;
1079 #endif
1080
1081         if (panicstr != NULL)
1082                 return;
1083         switch (what) {
1084         case SA_SLOCKED:
1085         case SA_SLOCKED | SA_NOTRECURSED:
1086         case SA_SLOCKED | SA_RECURSED:
1087 #ifndef WITNESS
1088                 slocked = 1;
1089                 /* FALLTHROUGH */
1090 #endif
1091         case SA_LOCKED:
1092         case SA_LOCKED | SA_NOTRECURSED:
1093         case SA_LOCKED | SA_RECURSED:
1094 #ifdef WITNESS
1095                 witness_assert(&sx->lock_object, what, file, line);
1096 #else
1097                 /*
1098                  * If some other thread has an exclusive lock or we
1099                  * have one and are asserting a shared lock, fail.
1100                  * Also, if no one has a lock at all, fail.
1101                  */
1102                 if (sx->sx_lock == SX_LOCK_UNLOCKED ||
1103                     (!(sx->sx_lock & SX_LOCK_SHARED) && (slocked ||
1104                     sx_xholder(sx) != curthread)))
1105                         panic("Lock %s not %slocked @ %s:%d\n",
1106                             sx->lock_object.lo_name, slocked ? "share " : "",
1107                             file, line);
1108
1109                 if (!(sx->sx_lock & SX_LOCK_SHARED)) {
1110                         if (sx_recursed(sx)) {
1111                                 if (what & SA_NOTRECURSED)
1112                                         panic("Lock %s recursed @ %s:%d\n",
1113                                             sx->lock_object.lo_name, file,
1114                                             line);
1115                         } else if (what & SA_RECURSED)
1116                                 panic("Lock %s not recursed @ %s:%d\n",
1117                                     sx->lock_object.lo_name, file, line);
1118                 }
1119 #endif
1120                 break;
1121         case SA_XLOCKED:
1122         case SA_XLOCKED | SA_NOTRECURSED:
1123         case SA_XLOCKED | SA_RECURSED:
1124                 if (sx_xholder(sx) != curthread)
1125                         panic("Lock %s not exclusively locked @ %s:%d\n",
1126                             sx->lock_object.lo_name, file, line);
1127                 if (sx_recursed(sx)) {
1128                         if (what & SA_NOTRECURSED)
1129                                 panic("Lock %s recursed @ %s:%d\n",
1130                                     sx->lock_object.lo_name, file, line);
1131                 } else if (what & SA_RECURSED)
1132                         panic("Lock %s not recursed @ %s:%d\n",
1133                             sx->lock_object.lo_name, file, line);
1134                 break;
1135         case SA_UNLOCKED:
1136 #ifdef WITNESS
1137                 witness_assert(&sx->lock_object, what, file, line);
1138 #else
1139                 /*
1140                  * If we hold an exclusve lock fail.  We can't
1141                  * reliably check to see if we hold a shared lock or
1142                  * not.
1143                  */
1144                 if (sx_xholder(sx) == curthread)
1145                         panic("Lock %s exclusively locked @ %s:%d\n",
1146                             sx->lock_object.lo_name, file, line);
1147 #endif
1148                 break;
1149         default:
1150                 panic("Unknown sx lock assertion: %d @ %s:%d", what, file,
1151                     line);
1152         }
1153 }
1154 #endif  /* INVARIANT_SUPPORT */
1155
1156 #ifdef DDB
1157 static void
1158 db_show_sx(const struct lock_object *lock)
1159 {
1160         struct thread *td;
1161         const struct sx *sx;
1162
1163         sx = (const struct sx *)lock;
1164
1165         db_printf(" state: ");
1166         if (sx->sx_lock == SX_LOCK_UNLOCKED)
1167                 db_printf("UNLOCKED\n");
1168         else if (sx->sx_lock == SX_LOCK_DESTROYED) {
1169                 db_printf("DESTROYED\n");
1170                 return;
1171         } else if (sx->sx_lock & SX_LOCK_SHARED)
1172                 db_printf("SLOCK: %ju\n", (uintmax_t)SX_SHARERS(sx->sx_lock));
1173         else {
1174                 td = sx_xholder(sx);
1175                 db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
1176                     td->td_tid, td->td_proc->p_pid, td->td_name);
1177                 if (sx_recursed(sx))
1178                         db_printf(" recursed: %d\n", sx->sx_recurse);
1179         }
1180
1181         db_printf(" waiters: ");
1182         switch(sx->sx_lock &
1183             (SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS)) {
1184         case SX_LOCK_SHARED_WAITERS:
1185                 db_printf("shared\n");
1186                 break;
1187         case SX_LOCK_EXCLUSIVE_WAITERS:
1188                 db_printf("exclusive\n");
1189                 break;
1190         case SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS:
1191                 db_printf("exclusive and shared\n");
1192                 break;
1193         default:
1194                 db_printf("none\n");
1195         }
1196 }
1197
1198 /*
1199  * Check to see if a thread that is blocked on a sleep queue is actually
1200  * blocked on an sx lock.  If so, output some details and return true.
1201  * If the lock has an exclusive owner, return that in *ownerp.
1202  */
1203 int
1204 sx_chain(struct thread *td, struct thread **ownerp)
1205 {
1206         struct sx *sx;
1207
1208         /*
1209          * Check to see if this thread is blocked on an sx lock.
1210          * First, we check the lock class.  If that is ok, then we
1211          * compare the lock name against the wait message.
1212          */
1213         sx = td->td_wchan;
1214         if (LOCK_CLASS(&sx->lock_object) != &lock_class_sx ||
1215             sx->lock_object.lo_name != td->td_wmesg)
1216                 return (0);
1217
1218         /* We think we have an sx lock, so output some details. */
1219         db_printf("blocked on sx \"%s\" ", td->td_wmesg);
1220         *ownerp = sx_xholder(sx);
1221         if (sx->sx_lock & SX_LOCK_SHARED)
1222                 db_printf("SLOCK (count %ju)\n",
1223                     (uintmax_t)SX_SHARERS(sx->sx_lock));
1224         else
1225                 db_printf("XLOCK\n");
1226         return (1);
1227 }
1228 #endif