sys/kern/kern_lock.c

   1 /*-
   2  * Copyright (c) 2008 Attilio Rao <attilio@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice(s), this list of conditions and the following disclaimer as
  10  *    the first lines of this file unmodified other than the possible
  11  *    addition of one or more copyright notices.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice(s), this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
  17  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19  * DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
  20  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  21  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  22  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  23  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
  26  * DAMAGE.
  27  */
  28
  29 #include "opt_adaptive_lockmgrs.h"
  30 #include "opt_ddb.h"
  31 #include "opt_kdtrace.h"
  32
  33 #include <sys/cdefs.h>
  34 __FBSDID("$FreeBSD$");
  35
  36 #include <sys/param.h>
  37 #include <sys/ktr.h>
  38 #include <sys/linker_set.h>
  39 #include <sys/lock.h>
  40 #include <sys/lock_profile.h>
  41 #include <sys/lockmgr.h>
  42 #include <sys/mutex.h>
  43 #include <sys/proc.h>
  44 #include <sys/sleepqueue.h>
  45 #ifdef DEBUG_LOCKS
  46 #include <sys/stack.h>
  47 #endif
  48 #include <sys/sysctl.h>
  49 #include <sys/systm.h>
  50
  51 #include <machine/cpu.h>
  52
  53 #ifdef DDB
  54 #include <ddb/ddb.h>
  55 #endif
  56
  57 CTASSERT(((LK_ADAPTIVE | LK_EXSLPFAIL | LK_NOSHARE) & LO_CLASSFLAGS) ==
  58     (LK_ADAPTIVE | LK_EXSLPFAIL | LK_NOSHARE));
  59 CTASSERT(LK_UNLOCKED == (LK_UNLOCKED &
  60     ~(LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS)));
  61
  62 #define SQ_EXCLUSIVE_QUEUE      0
  63 #define SQ_SHARED_QUEUE         1
  64
  65 #ifdef ADAPTIVE_LOCKMGRS
  66 #define ALK_RETRIES             10
  67 #define ALK_LOOPS               10000
  68 #endif
  69
  70 #ifndef INVARIANTS
  71 #define _lockmgr_assert(lk, what, file, line)
  72 #define TD_LOCKS_INC(td)
  73 #define TD_LOCKS_DEC(td)
  74 #else
  75 #define TD_LOCKS_INC(td)        ((td)->td_locks++)
  76 #define TD_LOCKS_DEC(td)        ((td)->td_locks--)
  77 #endif
  78 #define TD_SLOCKS_INC(td)       ((td)->td_lk_slocks++)
  79 #define TD_SLOCKS_DEC(td)       ((td)->td_lk_slocks--)
  80
  81 #ifndef DEBUG_LOCKS
  82 #define STACK_PRINT(lk)
  83 #define STACK_SAVE(lk)
  84 #define STACK_ZERO(lk)
  85 #else
  86 #define STACK_PRINT(lk) stack_print_ddb(&(lk)->lk_stack)
  87 #define STACK_SAVE(lk)  stack_save(&(lk)->lk_stack)
  88 #define STACK_ZERO(lk)  stack_zero(&(lk)->lk_stack)
  89 #endif
  90
  91 #define LOCK_LOG2(lk, string, arg1, arg2)                               \
  92         if (LOCK_LOG_TEST(&(lk)->lock_object, 0))                       \
  93                 CTR2(KTR_LOCK, (string), (arg1), (arg2))
  94 #define LOCK_LOG3(lk, string, arg1, arg2, arg3)                         \
  95         if (LOCK_LOG_TEST(&(lk)->lock_object, 0))                       \
  96                 CTR3(KTR_LOCK, (string), (arg1), (arg2), (arg3))
  97
  98 #define GIANT_DECLARE                                                   \
  99         int _i = 0;                                                     \
 100         WITNESS_SAVE_DECL(Giant)
 101 #define GIANT_RESTORE() do {                                            \
 102         if (_i > 0) {                                                   \
 103                 while (_i--)                                            \
 104                         mtx_lock(&Giant);                               \
 105                 WITNESS_RESTORE(&Giant.lock_object, Giant);             \
 106         }                                                               \
 107 } while (0)
 108 #define GIANT_SAVE() do {                                               \
 109         if (mtx_owned(&Giant)) {                                        \
 110                 WITNESS_SAVE(&Giant.lock_object, Giant);                \
 111                 while (mtx_owned(&Giant)) {                             \
 112                         _i++;                                           \
 113                         mtx_unlock(&Giant);                             \
 114                 }                                                       \
 115         }                                                               \
 116 } while (0)
 117
 118 #define LK_CAN_SHARE(x)                                                 \
 119         (((x) & LK_SHARE) && (((x) & LK_EXCLUSIVE_WAITERS) == 0 ||      \
 120         ((x) & LK_EXCLUSIVE_SPINNERS) == 0 ||                           \
 121         curthread->td_lk_slocks || (curthread->td_pflags & TDP_DEADLKTREAT)))
 122 #define LK_TRYOP(x)                                                     \
 123         ((x) & LK_NOWAIT)
 124
 125 #define LK_CAN_WITNESS(x)                                               \
 126         (((x) & LK_NOWITNESS) == 0 && !LK_TRYOP(x))
 127 #define LK_TRYWIT(x)                                                    \
 128         (LK_TRYOP(x) ? LOP_TRYLOCK : 0)
 129
 130 #define LK_CAN_ADAPT(lk, f)                                             \
 131         (((lk)->lock_object.lo_flags & LK_ADAPTIVE) != 0 &&             \
 132         ((f) & LK_SLEEPFAIL) == 0)
 133
 134 #define lockmgr_disowned(lk)                                            \
 135         (((lk)->lk_lock & ~(LK_FLAGMASK & ~LK_SHARE)) == LK_KERNPROC)
 136
 137 #define lockmgr_xlocked(lk)                                             \
 138         (((lk)->lk_lock & ~(LK_FLAGMASK & ~LK_SHARE)) == (uintptr_t)curthread)
 139
 140 static void      assert_lockmgr(struct lock_object *lock, int how);
 141 #ifdef DDB
 142 static void      db_show_lockmgr(struct lock_object *lock);
 143 #endif
 144 static void      lock_lockmgr(struct lock_object *lock, int how);
 145 #ifdef KDTRACE_HOOKS
 146 static int       owner_lockmgr(struct lock_object *lock, struct thread **owner);
 147 #endif
 148 static int       unlock_lockmgr(struct lock_object *lock);
 149
 150 struct lock_class lock_class_lockmgr = {
 151         .lc_name = "lockmgr",
 152         .lc_flags = LC_RECURSABLE | LC_SLEEPABLE | LC_SLEEPLOCK | LC_UPGRADABLE,
 153         .lc_assert = assert_lockmgr,
 154 #ifdef DDB
 155         .lc_ddb_show = db_show_lockmgr,
 156 #endif
 157         .lc_lock = lock_lockmgr,
 158         .lc_unlock = unlock_lockmgr,
 159 #ifdef KDTRACE_HOOKS
 160         .lc_owner = owner_lockmgr,
 161 #endif
 162 };
 163
 164 static __inline struct thread *
 165 lockmgr_xholder(struct lock *lk)
 166 {
 167         uintptr_t x;
 168
 169         x = lk->lk_lock;
 170         return ((x & LK_SHARE) ? NULL : (struct thread *)LK_HOLDER(x));
 171 }
 172
 173 /*
 174  * It assumes sleepq_lock held and returns with this one unheld.
 175  * It also assumes the generic interlock is sane and previously checked.
 176  * If LK_INTERLOCK is specified the interlock is not reacquired after the
 177  * sleep.
 178  */
 179 static __inline int
 180 sleeplk(struct lock *lk, u_int flags, struct lock_object *ilk,
 181     const char *wmesg, int pri, int timo, int queue)
 182 {
 183         GIANT_DECLARE;
 184         struct lock_class *class;
 185         int catch, error;
 186
 187         class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL;
 188         catch = pri & PCATCH;
 189         pri &= PRIMASK;
 190         error = 0;
 191
 192         LOCK_LOG3(lk, "%s: %p blocking on the %s sleepqueue", __func__, lk,
 193             (queue == SQ_EXCLUSIVE_QUEUE) ? "exclusive" : "shared");
 194
 195         if (flags & LK_INTERLOCK)
 196                 class->lc_unlock(ilk);
 197
 198         /*
 199          * LK_EXSLPFAIL is not invariant during the lock pattern but it is
 200          * always protected by the sleepqueue spinlock, thus it is safe to
 201          * handle within the lo_flags.
 202          */
 203         if (queue == SQ_EXCLUSIVE_QUEUE && (flags & LK_SLEEPFAIL) != 0)
 204                 lk->lock_object.lo_flags |= LK_EXSLPFAIL;
 205         GIANT_SAVE();
 206         sleepq_add(&lk->lock_object, NULL, wmesg, SLEEPQ_LK | (catch ?
 207             SLEEPQ_INTERRUPTIBLE : 0), queue);
 208         if ((flags & LK_TIMELOCK) && timo)
 209                 sleepq_set_timeout(&lk->lock_object, timo);
 210
 211         /*
 212          * Decisional switch for real sleeping.
 213          */
 214         if ((flags & LK_TIMELOCK) && timo && catch)
 215                 error = sleepq_timedwait_sig(&lk->lock_object, pri);
 216         else if ((flags & LK_TIMELOCK) && timo)
 217                 error = sleepq_timedwait(&lk->lock_object, pri);
 218         else if (catch)
 219                 error = sleepq_wait_sig(&lk->lock_object, pri);
 220         else
 221                 sleepq_wait(&lk->lock_object, pri);
 222         GIANT_RESTORE();
 223         if ((flags & LK_SLEEPFAIL) && error == 0)
 224                 error = ENOLCK;
 225
 226         return (error);
 227 }
 228
 229 static __inline int
 230 wakeupshlk(struct lock *lk, const char *file, int line)
 231 {
 232         uintptr_t v, x;
 233         u_int realexslp;
 234         int queue, wakeup_swapper;
 235
 236         TD_LOCKS_DEC(curthread);
 237         TD_SLOCKS_DEC(curthread);
 238         WITNESS_UNLOCK(&lk->lock_object, 0, file, line);
 239         LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, file, line);
 240
 241         wakeup_swapper = 0;
 242         for (;;) {
 243                 x = lk->lk_lock;
 244
 245                 /*
 246                  * If there is more than one shared lock held, just drop one
 247                  * and return.
 248                  */
 249                 if (LK_SHARERS(x) > 1) {
 250                         if (atomic_cmpset_rel_ptr(&lk->lk_lock, x,
 251                             x - LK_ONE_SHARER))
 252                                 break;
 253                         continue;
 254                 }
 255
 256                 /*
 257                  * If there are not waiters on the exclusive queue, drop the
 258                  * lock quickly.
 259                  */
 260                 if ((x & LK_ALL_WAITERS) == 0) {
 261                         MPASS((x & ~LK_EXCLUSIVE_SPINNERS) ==
 262                             LK_SHARERS_LOCK(1));
 263                         if (atomic_cmpset_rel_ptr(&lk->lk_lock, x, LK_UNLOCKED))
 264                                 break;
 265                         continue;
 266                 }
 267
 268                 /*
 269                  * We should have a sharer with waiters, so enter the hard
 270                  * path in order to handle wakeups correctly.
 271                  */
 272                 sleepq_lock(&lk->lock_object);
 273                 x = lk->lk_lock & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
 274                 v = LK_UNLOCKED;
 275
 276                 /*
 277                  * If the lock has exclusive waiters, give them preference in
 278                  * order to avoid deadlock with shared runners up.
 279                  * If interruptible sleeps left the exclusive queue empty
 280                  * avoid a starvation for the threads sleeping on the shared
 281                  * queue by giving them precedence and cleaning up the
 282                  * exclusive waiters bit anyway.
 283                  * Please note that the LK_EXSLPFAIL flag may be lying about
 284                  * the real presence of waiters with the LK_SLEEPFAIL flag on
 285                  * because they may be used in conjuction with interruptible
 286                  * sleeps.
 287                  */
 288                 realexslp = sleepq_sleepcnt(&lk->lock_object,
 289                     SQ_EXCLUSIVE_QUEUE);
 290                 if ((x & LK_EXCLUSIVE_WAITERS) != 0 && realexslp != 0) {
 291                         if ((lk->lock_object.lo_flags & LK_EXSLPFAIL) == 0) {
 292                                 lk->lock_object.lo_flags &= ~LK_EXSLPFAIL;
 293                                 queue = SQ_EXCLUSIVE_QUEUE;
 294                                 v |= (x & LK_SHARED_WAITERS);
 295                         } else {
 296                                 lk->lock_object.lo_flags &= ~LK_EXSLPFAIL;
 297                                 LOCK_LOG2(lk,
 298                                     "%s: %p has only LK_SLEEPFAIL sleepers",
 299                                     __func__, lk);
 300                                 LOCK_LOG2(lk,
 301                             "%s: %p waking up threads on the exclusive queue",
 302                                     __func__, lk);
 303                                 wakeup_swapper =
 304                                     sleepq_broadcast(&lk->lock_object,
 305                                     SLEEPQ_LK, 0, SQ_EXCLUSIVE_QUEUE);
 306                                 queue = SQ_SHARED_QUEUE;
 307                         }
 308
 309                 } else {
 310
 311                         /*
 312                          * Exclusive waiters sleeping with LK_SLEEPFAIL on
 313                          * and using interruptible sleeps/timeout may have
 314                          * left spourious LK_EXSLPFAIL flag on, so clean
 315                          * it up anyway.
 316                          */
 317                         lk->lock_object.lo_flags &= ~LK_EXSLPFAIL;
 318                         queue = SQ_SHARED_QUEUE;
 319                 }
 320
 321                 if (!atomic_cmpset_rel_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x,
 322                     v)) {
 323                         sleepq_release(&lk->lock_object);
 324                         continue;
 325                 }
 326                 LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue",
 327                     __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
 328                     "exclusive");
 329                 wakeup_swapper |= sleepq_broadcast(&lk->lock_object, SLEEPQ_LK,
 330                     0, queue);
 331                 sleepq_release(&lk->lock_object);
 332                 break;
 333         }
 334
 335         lock_profile_release_lock(&lk->lock_object);
 336         return (wakeup_swapper);
 337 }
 338
 339 static void
 340 assert_lockmgr(struct lock_object *lock, int what)
 341 {
 342
 343         panic("lockmgr locks do not support assertions");
 344 }
 345
 346 static void
 347 lock_lockmgr(struct lock_object *lock, int how)
 348 {
 349
 350         panic("lockmgr locks do not support sleep interlocking");
 351 }
 352
 353 static int
 354 unlock_lockmgr(struct lock_object *lock)
 355 {
 356
 357         panic("lockmgr locks do not support sleep interlocking");
 358 }
 359
 360 #ifdef KDTRACE_HOOKS
 361 static int
 362 owner_lockmgr(struct lock_object *lock, struct thread **owner)
 363 {
 364
 365         panic("lockmgr locks do not support owner inquiring");
 366 }
 367 #endif
 368
 369 void
 370 lockinit(struct lock *lk, int pri, const char *wmesg, int timo, int flags)
 371 {
 372         int iflags;
 373
 374         MPASS((flags & ~LK_INIT_MASK) == 0);
 375         ASSERT_ATOMIC_LOAD_PTR(lk->lk_lock,
 376             ("%s: lockmgr not aligned for %s: %p", __func__, wmesg,
 377             &lk->lk_lock));
 378
 379         iflags = LO_SLEEPABLE | LO_UPGRADABLE;
 380         if (flags & LK_CANRECURSE)
 381                 iflags |= LO_RECURSABLE;
 382         if ((flags & LK_NODUP) == 0)
 383                 iflags |= LO_DUPOK;
 384         if (flags & LK_NOPROFILE)
 385                 iflags |= LO_NOPROFILE;
 386         if ((flags & LK_NOWITNESS) == 0)
 387                 iflags |= LO_WITNESS;
 388         if (flags & LK_QUIET)
 389                 iflags |= LO_QUIET;
 390         iflags |= flags & (LK_ADAPTIVE | LK_NOSHARE);
 391
 392         lk->lk_lock = LK_UNLOCKED;
 393         lk->lk_recurse = 0;
 394         lk->lk_timo = timo;
 395         lk->lk_pri = pri;
 396         lock_init(&lk->lock_object, &lock_class_lockmgr, wmesg, NULL, iflags);
 397         STACK_ZERO(lk);
 398 }
 399
 400 void
 401 lockdestroy(struct lock *lk)
 402 {
 403
 404         KASSERT(lk->lk_lock == LK_UNLOCKED, ("lockmgr still held"));
 405         KASSERT(lk->lk_recurse == 0, ("lockmgr still recursed"));
 406         KASSERT((lk->lock_object.lo_flags & LK_EXSLPFAIL) == 0,
 407             ("lockmgr still exclusive waiters"));
 408         lock_destroy(&lk->lock_object);
 409 }
 410
 411 int
 412 __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
 413     const char *wmesg, int pri, int timo, const char *file, int line)
 414 {
 415         GIANT_DECLARE;
 416         struct lock_class *class;
 417         const char *iwmesg;
 418         uintptr_t tid, v, x;
 419         u_int op, realexslp;
 420         int error, ipri, itimo, queue, wakeup_swapper;
 421 #ifdef LOCK_PROFILING
 422         uint64_t waittime = 0;
 423         int contested = 0;
 424 #endif
 425 #ifdef ADAPTIVE_LOCKMGRS
 426         volatile struct thread *owner;
 427         u_int i, spintries = 0;
 428 #endif
 429
 430         error = 0;
 431         tid = (uintptr_t)curthread;
 432         op = (flags & LK_TYPE_MASK);
 433         iwmesg = (wmesg == LK_WMESG_DEFAULT) ? lk->lock_object.lo_name : wmesg;
 434         ipri = (pri == LK_PRIO_DEFAULT) ? lk->lk_pri : pri;
 435         itimo = (timo == LK_TIMO_DEFAULT) ? lk->lk_timo : timo;
 436
 437         MPASS((flags & ~LK_TOTAL_MASK) == 0);
 438         KASSERT((op & (op - 1)) == 0,
 439             ("%s: Invalid requested operation @ %s:%d", __func__, file, line));
 440         KASSERT((flags & (LK_NOWAIT | LK_SLEEPFAIL)) == 0 ||
 441             (op != LK_DOWNGRADE && op != LK_RELEASE),
 442             ("%s: Invalid flags in regard of the operation desired @ %s:%d",
 443             __func__, file, line));
 444         KASSERT((flags & LK_INTERLOCK) == 0 || ilk != NULL,
 445             ("%s: LK_INTERLOCK passed without valid interlock @ %s:%d",
 446             __func__, file, line));
 447
 448         class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL;
 449         if (panicstr != NULL) {
 450                 if (flags & LK_INTERLOCK)
 451                         class->lc_unlock(ilk);
 452                 return (0);
 453         }
 454
 455         if (op == LK_SHARED && (lk->lock_object.lo_flags & LK_NOSHARE))
 456                 op = LK_EXCLUSIVE;
 457
 458         wakeup_swapper = 0;
 459         switch (op) {
 460         case LK_SHARED:
 461                 if (LK_CAN_WITNESS(flags))
 462                         WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
 463                             file, line, ilk);
 464                 for (;;) {
 465                         x = lk->lk_lock;
 466
 467                         /*
 468                          * If no other thread has an exclusive lock, or
 469                          * no exclusive waiter is present, bump the count of
 470                          * sharers.  Since we have to preserve the state of
 471                          * waiters, if we fail to acquire the shared lock
 472                          * loop back and retry.
 473                          */
 474                         if (LK_CAN_SHARE(x)) {
 475                                 if (atomic_cmpset_acq_ptr(&lk->lk_lock, x,
 476                                     x + LK_ONE_SHARER))
 477                                         break;
 478                                 continue;
 479                         }
 480                         lock_profile_obtain_lock_failed(&lk->lock_object,
 481                             &contested, &waittime);
 482
 483                         /*
 484                          * If the lock is already held by curthread in
 485                          * exclusive way avoid a deadlock.
 486                          */
 487                         if (LK_HOLDER(x) == tid) {
 488                                 LOCK_LOG2(lk,
 489                                     "%s: %p already held in exclusive mode",
 490                                     __func__, lk);
 491                                 error = EDEADLK;
 492                                 break;
 493                         }
 494
 495                         /*
 496                          * If the lock is expected to not sleep just give up
 497                          * and return.
 498                          */
 499                         if (LK_TRYOP(flags)) {
 500                                 LOCK_LOG2(lk, "%s: %p fails the try operation",
 501                                     __func__, lk);
 502                                 error = EBUSY;
 503                                 break;
 504                         }
 505
 506 #ifdef ADAPTIVE_LOCKMGRS
 507                         /*
 508                          * If the owner is running on another CPU, spin until
 509                          * the owner stops running or the state of the lock
 510                          * changes.  We need a double-state handle here
 511                          * because for a failed acquisition the lock can be
 512                          * either held in exclusive mode or shared mode
 513                          * (for the writer starvation avoidance technique).
 514                          */
 515                         if (LK_CAN_ADAPT(lk, flags) && (x & LK_SHARE) == 0 &&
 516                             LK_HOLDER(x) != LK_KERNPROC) {
 517                                 owner = (struct thread *)LK_HOLDER(x);
 518                                 if (LOCK_LOG_TEST(&lk->lock_object, 0))
 519                                         CTR3(KTR_LOCK,
 520                                             "%s: spinning on %p held by %p",
 521                                             __func__, lk, owner);
 522
 523                                 /*
 524                                  * If we are holding also an interlock drop it
 525                                  * in order to avoid a deadlock if the lockmgr
 526                                  * owner is adaptively spinning on the
 527                                  * interlock itself.
 528                                  */
 529                                 if (flags & LK_INTERLOCK) {
 530                                         class->lc_unlock(ilk);
 531                                         flags &= ~LK_INTERLOCK;
 532                                 }
 533                                 GIANT_SAVE();
 534                                 while (LK_HOLDER(lk->lk_lock) ==
 535                                     (uintptr_t)owner && TD_IS_RUNNING(owner))
 536                                         cpu_spinwait();
 537                                 GIANT_RESTORE();
 538                                 continue;
 539                         } else if (LK_CAN_ADAPT(lk, flags) &&
 540                             (x & LK_SHARE) != 0 && LK_SHARERS(x) &&
 541                             spintries < ALK_RETRIES) {
 542                                 if (flags & LK_INTERLOCK) {
 543                                         class->lc_unlock(ilk);
 544                                         flags &= ~LK_INTERLOCK;
 545                                 }
 546                                 GIANT_SAVE();
 547                                 spintries++;
 548                                 for (i = 0; i < ALK_LOOPS; i++) {
 549                                         if (LOCK_LOG_TEST(&lk->lock_object, 0))
 550                                                 CTR4(KTR_LOCK,
 551                                     "%s: shared spinning on %p with %u and %u",
 552                                                     __func__, lk, spintries, i);
 553                                         x = lk->lk_lock;
 554                                         if ((x & LK_SHARE) == 0 ||
 555                                             LK_CAN_SHARE(x) != 0)
 556                                                 break;
 557                                         cpu_spinwait();
 558                                 }
 559                                 GIANT_RESTORE();
 560                                 if (i != ALK_LOOPS)
 561                                         continue;
 562                         }
 563 #endif
 564
 565                         /*
 566                          * Acquire the sleepqueue chain lock because we
 567                          * probabilly will need to manipulate waiters flags.
 568                          */
 569                         sleepq_lock(&lk->lock_object);
 570                         x = lk->lk_lock;
 571
 572                         /*
 573                          * if the lock can be acquired in shared mode, try
 574                          * again.
 575                          */
 576                         if (LK_CAN_SHARE(x)) {
 577                                 sleepq_release(&lk->lock_object);
 578                                 continue;
 579                         }
 580
 581 #ifdef ADAPTIVE_LOCKMGRS
 582                         /*
 583                          * The current lock owner might have started executing
 584                          * on another CPU (or the lock could have changed
 585                          * owner) while we were waiting on the turnstile
 586                          * chain lock.  If so, drop the turnstile lock and try
 587                          * again.
 588                          */
 589                         if (LK_CAN_ADAPT(lk, flags) && (x & LK_SHARE) == 0 &&
 590                             LK_HOLDER(x) != LK_KERNPROC) {
 591                                 owner = (struct thread *)LK_HOLDER(x);
 592                                 if (TD_IS_RUNNING(owner)) {
 593                                         sleepq_release(&lk->lock_object);
 594                                         continue;
 595                                 }
 596                         }
 597 #endif
 598
 599                         /*
 600                          * Try to set the LK_SHARED_WAITERS flag.  If we fail,
 601                          * loop back and retry.
 602                          */
 603                         if ((x & LK_SHARED_WAITERS) == 0) {
 604                                 if (!atomic_cmpset_acq_ptr(&lk->lk_lock, x,
 605                                     x | LK_SHARED_WAITERS)) {
 606                                         sleepq_release(&lk->lock_object);
 607                                         continue;
 608                                 }
 609                                 LOCK_LOG2(lk, "%s: %p set shared waiters flag",
 610                                     __func__, lk);
 611                         }
 612
 613                         /*
 614                          * As far as we have been unable to acquire the
 615                          * shared lock and the shared waiters flag is set,
 616                          * we will sleep.
 617                          */
 618                         error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,
 619                             SQ_SHARED_QUEUE);
 620                         flags &= ~LK_INTERLOCK;
 621                         if (error) {
 622                                 LOCK_LOG3(lk,
 623                                     "%s: interrupted sleep for %p with %d",
 624                                     __func__, lk, error);
 625                                 break;
 626                         }
 627                         LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
 628                             __func__, lk);
 629                 }
 630                 if (error == 0) {
 631                         lock_profile_obtain_lock_success(&lk->lock_object,
 632                             contested, waittime, file, line);
 633                         LOCK_LOG_LOCK("SLOCK", &lk->lock_object, 0, 0, file,
 634                             line);
 635                         WITNESS_LOCK(&lk->lock_object, LK_TRYWIT(flags), file,
 636                             line);
 637                         TD_LOCKS_INC(curthread);
 638                         TD_SLOCKS_INC(curthread);
 639                         STACK_SAVE(lk);
 640                 }
 641                 break;
 642         case LK_UPGRADE:
 643                 _lockmgr_assert(lk, KA_SLOCKED, file, line);
 644                 v = lk->lk_lock;
 645                 x = v & LK_ALL_WAITERS;
 646                 v &= LK_EXCLUSIVE_SPINNERS;
 647
 648                 /*
 649                  * Try to switch from one shared lock to an exclusive one.
 650                  * We need to preserve waiters flags during the operation.
 651                  */
 652                 if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x | v,
 653                     tid | x)) {
 654                         LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file,
 655                             line);
 656                         WITNESS_UPGRADE(&lk->lock_object, LOP_EXCLUSIVE |
 657                             LK_TRYWIT(flags), file, line);
 658                         TD_SLOCKS_DEC(curthread);
 659                         break;
 660                 }
 661
 662                 /*
 663                  * We have been unable to succeed in upgrading, so just
 664                  * give up the shared lock.
 665                  */
 666                 wakeup_swapper |= wakeupshlk(lk, file, line);
 667
 668                 /* FALLTHROUGH */
 669         case LK_EXCLUSIVE:
 670                 if (LK_CAN_WITNESS(flags))
 671                         WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
 672                             LOP_EXCLUSIVE, file, line, ilk);
 673
 674                 /*
 675                  * If curthread already holds the lock and this one is
 676                  * allowed to recurse, simply recurse on it.
 677                  */
 678                 if (lockmgr_xlocked(lk)) {
 679                         if ((flags & LK_CANRECURSE) == 0 &&
 680                             (lk->lock_object.lo_flags & LO_RECURSABLE) == 0) {
 681
 682                                 /*
 683                                  * If the lock is expected to not panic just
 684                                  * give up and return.
 685                                  */
 686                                 if (LK_TRYOP(flags)) {
 687                                         LOCK_LOG2(lk,
 688                                             "%s: %p fails the try operation",
 689                                             __func__, lk);
 690                                         error = EBUSY;
 691                                         break;
 692                                 }
 693                                 if (flags & LK_INTERLOCK)
 694                                         class->lc_unlock(ilk);
 695                 panic("%s: recursing on non recursive lockmgr %s @ %s:%d\n",
 696                                     __func__, iwmesg, file, line);
 697                         }
 698                         lk->lk_recurse++;
 699                         LOCK_LOG2(lk, "%s: %p recursing", __func__, lk);
 700                         LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0,
 701                             lk->lk_recurse, file, line);
 702                         WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
 703                             LK_TRYWIT(flags), file, line);
 704                         TD_LOCKS_INC(curthread);
 705                         break;
 706                 }
 707
 708                 while (!atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED,
 709                     tid)) {
 710                         lock_profile_obtain_lock_failed(&lk->lock_object,
 711                             &contested, &waittime);
 712
 713                         /*
 714                          * If the lock is expected to not sleep just give up
 715                          * and return.
 716                          */
 717                         if (LK_TRYOP(flags)) {
 718                                 LOCK_LOG2(lk, "%s: %p fails the try operation",
 719                                     __func__, lk);
 720                                 error = EBUSY;
 721                                 break;
 722                         }
 723
 724 #ifdef ADAPTIVE_LOCKMGRS
 725                         /*
 726                          * If the owner is running on another CPU, spin until
 727                          * the owner stops running or the state of the lock
 728                          * changes.
 729                          */
 730                         x = lk->lk_lock;
 731                         if (LK_CAN_ADAPT(lk, flags) && (x & LK_SHARE) == 0 &&
 732                             LK_HOLDER(x) != LK_KERNPROC) {
 733                                 owner = (struct thread *)LK_HOLDER(x);
 734                                 if (LOCK_LOG_TEST(&lk->lock_object, 0))
 735                                         CTR3(KTR_LOCK,
 736                                             "%s: spinning on %p held by %p",
 737                                             __func__, lk, owner);
 738
 739                                 /*
 740                                  * If we are holding also an interlock drop it
 741                                  * in order to avoid a deadlock if the lockmgr
 742                                  * owner is adaptively spinning on the
 743                                  * interlock itself.
 744                                  */
 745                                 if (flags & LK_INTERLOCK) {
 746                                         class->lc_unlock(ilk);
 747                                         flags &= ~LK_INTERLOCK;
 748                                 }
 749                                 GIANT_SAVE();
 750                                 while (LK_HOLDER(lk->lk_lock) ==
 751                                     (uintptr_t)owner && TD_IS_RUNNING(owner))
 752                                         cpu_spinwait();
 753                                 GIANT_RESTORE();
 754                                 continue;
 755                         } else if (LK_CAN_ADAPT(lk, flags) &&
 756                             (x & LK_SHARE) != 0 && LK_SHARERS(x) &&
 757                             spintries < ALK_RETRIES) {
 758                                 if ((x & LK_EXCLUSIVE_SPINNERS) == 0 &&
 759                                     !atomic_cmpset_ptr(&lk->lk_lock, x,
 760                                     x | LK_EXCLUSIVE_SPINNERS))
 761                                         continue;
 762                                 if (flags & LK_INTERLOCK) {
 763                                         class->lc_unlock(ilk);
 764                                         flags &= ~LK_INTERLOCK;
 765                                 }
 766                                 GIANT_SAVE();
 767                                 spintries++;
 768                                 for (i = 0; i < ALK_LOOPS; i++) {
 769                                         if (LOCK_LOG_TEST(&lk->lock_object, 0))
 770                                                 CTR4(KTR_LOCK,
 771                                     "%s: shared spinning on %p with %u and %u",
 772                                                     __func__, lk, spintries, i);
 773                                         if ((lk->lk_lock &
 774                                             LK_EXCLUSIVE_SPINNERS) == 0)
 775                                                 break;
 776                                         cpu_spinwait();
 777                                 }
 778                                 GIANT_RESTORE();
 779                                 if (i != ALK_LOOPS)
 780                                         continue;
 781                         }
 782 #endif
 783
 784                         /*
 785                          * Acquire the sleepqueue chain lock because we
 786                          * probabilly will need to manipulate waiters flags.
 787                          */
 788                         sleepq_lock(&lk->lock_object);
 789                         x = lk->lk_lock;
 790
 791                         /*
 792                          * if the lock has been released while we spun on
 793                          * the sleepqueue chain lock just try again.
 794                          */
 795                         if (x == LK_UNLOCKED) {
 796                                 sleepq_release(&lk->lock_object);
 797                                 continue;
 798                         }
 799
 800 #ifdef ADAPTIVE_LOCKMGRS
 801                         /*
 802                          * The current lock owner might have started executing
 803                          * on another CPU (or the lock could have changed
 804                          * owner) while we were waiting on the turnstile
 805                          * chain lock.  If so, drop the turnstile lock and try
 806                          * again.
 807                          */
 808                         if (LK_CAN_ADAPT(lk, flags) && (x & LK_SHARE) == 0 &&
 809                             LK_HOLDER(x) != LK_KERNPROC) {
 810                                 owner = (struct thread *)LK_HOLDER(x);
 811                                 if (TD_IS_RUNNING(owner)) {
 812                                         sleepq_release(&lk->lock_object);
 813                                         continue;
 814                                 }
 815                         }
 816 #endif
 817
 818                         /*
 819                          * The lock can be in the state where there is a
 820                          * pending queue of waiters, but still no owner.
 821                          * This happens when the lock is contested and an
 822                          * owner is going to claim the lock.
 823                          * If curthread is the one successfully acquiring it
 824                          * claim lock ownership and return, preserving waiters
 825                          * flags.
 826                          */
 827                         v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
 828                         if ((x & ~v) == LK_UNLOCKED) {
 829                                 v &= ~LK_EXCLUSIVE_SPINNERS;
 830                                 if (atomic_cmpset_acq_ptr(&lk->lk_lock, x,
 831                                     tid | v)) {
 832                                         sleepq_release(&lk->lock_object);
 833                                         LOCK_LOG2(lk,
 834                                             "%s: %p claimed by a new writer",
 835                                             __func__, lk);
 836                                         break;
 837                                 }
 838                                 sleepq_release(&lk->lock_object);
 839                                 continue;
 840                         }
 841
 842                         /*
 843                          * Try to set the LK_EXCLUSIVE_WAITERS flag.  If we
 844                          * fail, loop back and retry.
 845                          */
 846                         if ((x & LK_EXCLUSIVE_WAITERS) == 0) {
 847                                 if (!atomic_cmpset_ptr(&lk->lk_lock, x,
 848                                     x | LK_EXCLUSIVE_WAITERS)) {
 849                                         sleepq_release(&lk->lock_object);
 850                                         continue;
 851                                 }
 852                                 LOCK_LOG2(lk, "%s: %p set excl waiters flag",
 853                                     __func__, lk);
 854                         }
 855
 856                         /*
 857                          * As far as we have been unable to acquire the
 858                          * exclusive lock and the exclusive waiters flag
 859                          * is set, we will sleep.
 860                          */
 861                         error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,
 862                             SQ_EXCLUSIVE_QUEUE);
 863                         flags &= ~LK_INTERLOCK;
 864                         if (error) {
 865                                 LOCK_LOG3(lk,
 866                                     "%s: interrupted sleep for %p with %d",
 867                                     __func__, lk, error);
 868                                 break;
 869                         }
 870                         LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
 871                             __func__, lk);
 872                 }
 873                 if (error == 0) {
 874                         lock_profile_obtain_lock_success(&lk->lock_object,
 875                             contested, waittime, file, line);
 876                         LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0,
 877                             lk->lk_recurse, file, line);
 878                         WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
 879                             LK_TRYWIT(flags), file, line);
 880                         TD_LOCKS_INC(curthread);
 881                         STACK_SAVE(lk);
 882                 }
 883                 break;
 884         case LK_DOWNGRADE:
 885                 _lockmgr_assert(lk, KA_XLOCKED | KA_NOTRECURSED, file, line);
 886                 LOCK_LOG_LOCK("XDOWNGRADE", &lk->lock_object, 0, 0, file, line);
 887                 WITNESS_DOWNGRADE(&lk->lock_object, 0, file, line);
 888                 TD_SLOCKS_INC(curthread);
 889
 890                 /*
 891                  * In order to preserve waiters flags, just spin.
 892                  */
 893                 for (;;) {
 894                         x = lk->lk_lock;
 895                         MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
 896                         x &= LK_ALL_WAITERS;
 897                         if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x,
 898                             LK_SHARERS_LOCK(1) | x))
 899                                 break;
 900                         cpu_spinwait();
 901                 }
 902                 break;
 903         case LK_RELEASE:
 904                 _lockmgr_assert(lk, KA_LOCKED, file, line);
 905                 x = lk->lk_lock;
 906
 907                 if ((x & LK_SHARE) == 0) {
 908
 909                         /*
 910                          * As first option, treact the lock as if it has not
 911                          * any waiter.
 912                          * Fix-up the tid var if the lock has been disowned.
 913                          */
 914                         if (LK_HOLDER(x) == LK_KERNPROC)
 915                                 tid = LK_KERNPROC;
 916                         else {
 917                                 WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE,
 918                                     file, line);
 919                                 TD_LOCKS_DEC(curthread);
 920                         }
 921                         LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0,
 922                             lk->lk_recurse, file, line);
 923
 924                         /*
 925                          * The lock is held in exclusive mode.
 926                          * If the lock is recursed also, then unrecurse it.
 927                          */
 928                         if (lockmgr_xlocked(lk) && lockmgr_recursed(lk)) {
 929                                 LOCK_LOG2(lk, "%s: %p unrecursing", __func__,
 930                                     lk);
 931                                 lk->lk_recurse--;
 932                                 break;
 933                         }
 934                         if (tid != LK_KERNPROC)
 935                                 lock_profile_release_lock(&lk->lock_object);
 936
 937                         if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid,
 938                             LK_UNLOCKED))
 939                                 break;
 940
 941                         sleepq_lock(&lk->lock_object);
 942                         x = lk->lk_lock;
 943                         v = LK_UNLOCKED;
 944
 945                         /*
 946                          * If the lock has exclusive waiters, give them
 947                          * preference in order to avoid deadlock with
 948                          * shared runners up.
 949                          * If interruptible sleeps left the exclusive queue
 950                          * empty avoid a starvation for the threads sleeping
 951                          * on the shared queue by giving them precedence
 952                          * and cleaning up the exclusive waiters bit anyway.
 953                          * Please note that the LK_EXSLPFAIL flag may be lying
 954                          * about the real presence of waiters with the
 955                          * LK_SLEEPFAIL flag on because they may be used in
 956                          * conjuction with interruptible sleeps.
 957                          */
 958                         MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
 959                         realexslp = sleepq_sleepcnt(&lk->lock_object,
 960                             SQ_EXCLUSIVE_QUEUE);
 961                         if ((x & LK_EXCLUSIVE_WAITERS) != 0 && realexslp != 0) {
 962                                 if ((lk->lock_object.lo_flags &
 963                                     LK_EXSLPFAIL) == 0) {
 964                                         lk->lock_object.lo_flags &=
 965                                             ~LK_EXSLPFAIL;
 966                                         queue = SQ_EXCLUSIVE_QUEUE;
 967                                         v |= (x & LK_SHARED_WAITERS);
 968                                 } else {
 969                                         lk->lock_object.lo_flags &=
 970                                             ~LK_EXSLPFAIL;
 971                                         LOCK_LOG2(lk,
 972                                         "%s: %p has only LK_SLEEPFAIL sleepers",
 973                                             __func__, lk);
 974                                         LOCK_LOG2(lk,
 975                         "%s: %p waking up threads on the exclusive queue",
 976                                             __func__, lk);
 977                                         wakeup_swapper =
 978                                             sleepq_broadcast(&lk->lock_object,
 979                                             SLEEPQ_LK, 0, SQ_EXCLUSIVE_QUEUE);
 980                                         queue = SQ_SHARED_QUEUE;
 981                                 }
 982                         } else {
 983
 984                                 /*
 985                                  * Exclusive waiters sleeping with LK_SLEEPFAIL
 986                                  * on and using interruptible sleeps/timeout
 987                                  * may have left spourious LK_EXSLPFAIL flag
 988                                  * on, so clean it up anyway.
 989                                  */
 990                                 lk->lock_object.lo_flags &= ~LK_EXSLPFAIL;
 991                                 queue = SQ_SHARED_QUEUE;
 992                         }
 993
 994                         LOCK_LOG3(lk,
 995                             "%s: %p waking up threads on the %s queue",
 996                             __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
 997                             "exclusive");
 998                         atomic_store_rel_ptr(&lk->lk_lock, v);
 999                         wakeup_swapper |= sleepq_broadcast(&lk->lock_object,
1000                             SLEEPQ_LK, 0, queue);
1001                         sleepq_release(&lk->lock_object);
1002                         break;
1003                 } else
1004                         wakeup_swapper = wakeupshlk(lk, file, line);
1005                 break;
1006         case LK_DRAIN:
1007                 if (LK_CAN_WITNESS(flags))
1008                         WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
1009                             LOP_EXCLUSIVE, file, line, ilk);
1010
1011                 /*
1012                  * Trying to drain a lock we already own will result in a
1013                  * deadlock.
1014                  */
1015                 if (lockmgr_xlocked(lk)) {
1016                         if (flags & LK_INTERLOCK)
1017                                 class->lc_unlock(ilk);
1018                         panic("%s: draining %s with the lock held @ %s:%d\n",
1019                             __func__, iwmesg, file, line);
1020                 }
1021
1022                 while (!atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) {
1023                         lock_profile_obtain_lock_failed(&lk->lock_object,
1024                             &contested, &waittime);
1025
1026                         /*
1027                          * If the lock is expected to not sleep just give up
1028                          * and return.
1029                          */
1030                         if (LK_TRYOP(flags)) {
1031                                 LOCK_LOG2(lk, "%s: %p fails the try operation",
1032                                     __func__, lk);
1033                                 error = EBUSY;
1034                                 break;
1035                         }
1036
1037                         /*
1038                          * Acquire the sleepqueue chain lock because we
1039                          * probabilly will need to manipulate waiters flags.
1040                          */
1041                         sleepq_lock(&lk->lock_object);
1042                         x = lk->lk_lock;
1043
1044                         /*
1045                          * if the lock has been released while we spun on
1046                          * the sleepqueue chain lock just try again.
1047                          */
1048                         if (x == LK_UNLOCKED) {
1049                                 sleepq_release(&lk->lock_object);
1050                                 continue;
1051                         }
1052
1053                         v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
1054                         if ((x & ~v) == LK_UNLOCKED) {
1055                                 v = (x & ~LK_EXCLUSIVE_SPINNERS);
1056
1057                                 /*
1058                                  * If interruptible sleeps left the exclusive
1059                                  * queue empty avoid a starvation for the
1060                                  * threads sleeping on the shared queue by
1061                                  * giving them precedence and cleaning up the
1062                                  * exclusive waiters bit anyway.
1063                                  * Please note that the LK_EXSLPFAIL flag may
1064                                  * be lying about the real presence of waiters
1065                                  * with the LK_SLEEPFAIL flag on because they
1066                                  * may be used in conjuction with interruptible
1067                                  * sleeps.
1068                                  */
1069                                 if (v & LK_EXCLUSIVE_WAITERS) {
1070                                         queue = SQ_EXCLUSIVE_QUEUE;
1071                                         v &= ~LK_EXCLUSIVE_WAITERS;
1072                                 } else {
1073
1074                                         /*
1075                                          * Exclusive waiters sleeping with
1076                                          * LK_SLEEPFAIL on and using
1077                                          * interruptible sleeps/timeout may
1078                                          * have left spourious LK_EXSLPFAIL
1079                                          * flag on, so clean it up anyway.
1080                                          */
1081                                         MPASS(v & LK_SHARED_WAITERS);
1082                                         lk->lock_object.lo_flags &=
1083                                             ~LK_EXSLPFAIL;
1084                                         queue = SQ_SHARED_QUEUE;
1085                                         v &= ~LK_SHARED_WAITERS;
1086                                 }
1087                                 if (queue == SQ_EXCLUSIVE_QUEUE) {
1088                                         realexslp =
1089                                             sleepq_sleepcnt(&lk->lock_object,
1090                                             SQ_EXCLUSIVE_QUEUE);
1091                                         if ((lk->lock_object.lo_flags &
1092                                             LK_EXSLPFAIL) == 0) {
1093                                                 lk->lock_object.lo_flags &=
1094                                                     ~LK_EXSLPFAIL;
1095                                                 queue = SQ_SHARED_QUEUE;
1096                                                 v &= ~LK_SHARED_WAITERS;
1097                                                 if (realexslp != 0) {
1098                                                         LOCK_LOG2(lk,
1099                                         "%s: %p has only LK_SLEEPFAIL sleepers",
1100                                                             __func__, lk);
1101                                                         LOCK_LOG2(lk,
1102                         "%s: %p waking up threads on the exclusive queue",
1103                                                             __func__, lk);
1104                                                         wakeup_swapper =
1105                                                             sleepq_broadcast(
1106                                                             &lk->lock_object,
1107                                                             SLEEPQ_LK, 0,
1108                                                             SQ_EXCLUSIVE_QUEUE);
1109                                                 }
1110                                         } else
1111                                                 lk->lock_object.lo_flags &=
1112                                                     ~LK_EXSLPFAIL;
1113                                 }
1114                                 if (!atomic_cmpset_ptr(&lk->lk_lock, x, v)) {
1115                                         sleepq_release(&lk->lock_object);
1116                                         continue;
1117                                 }
1118                                 LOCK_LOG3(lk,
1119                                 "%s: %p waking up all threads on the %s queue",
1120                                     __func__, lk, queue == SQ_SHARED_QUEUE ?
1121                                     "shared" : "exclusive");
1122                                 wakeup_swapper |= sleepq_broadcast(
1123                                     &lk->lock_object, SLEEPQ_LK, 0, queue);
1124
1125                                 /*
1126                                  * If shared waiters have been woken up we need
1127                                  * to wait for one of them to acquire the lock
1128                                  * before to set the exclusive waiters in
1129                                  * order to avoid a deadlock.
1130                                  */
1131                                 if (queue == SQ_SHARED_QUEUE) {
1132                                         for (v = lk->lk_lock;
1133                                             (v & LK_SHARE) && !LK_SHARERS(v);
1134                                             v = lk->lk_lock)
1135                                                 cpu_spinwait();
1136                                 }
1137                         }
1138
1139                         /*
1140                          * Try to set the LK_EXCLUSIVE_WAITERS flag.  If we
1141                          * fail, loop back and retry.
1142                          */
1143                         if ((x & LK_EXCLUSIVE_WAITERS) == 0) {
1144                                 if (!atomic_cmpset_ptr(&lk->lk_lock, x,
1145                                     x | LK_EXCLUSIVE_WAITERS)) {
1146                                         sleepq_release(&lk->lock_object);
1147                                         continue;
1148                                 }
1149                                 LOCK_LOG2(lk, "%s: %p set drain waiters flag",
1150                                     __func__, lk);
1151                         }
1152
1153                         /*
1154                          * As far as we have been unable to acquire the
1155                          * exclusive lock and the exclusive waiters flag
1156                          * is set, we will sleep.
1157                          */
1158                         if (flags & LK_INTERLOCK) {
1159                                 class->lc_unlock(ilk);
1160                                 flags &= ~LK_INTERLOCK;
1161                         }
1162                         GIANT_SAVE();
1163                         sleepq_add(&lk->lock_object, NULL, iwmesg, SLEEPQ_LK,
1164                             SQ_EXCLUSIVE_QUEUE);
1165                         sleepq_wait(&lk->lock_object, ipri & PRIMASK);
1166                         GIANT_RESTORE();
1167                         LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
1168                             __func__, lk);
1169                 }
1170
1171                 if (error == 0) {
1172                         lock_profile_obtain_lock_success(&lk->lock_object,
1173                             contested, waittime, file, line);
1174                         LOCK_LOG_LOCK("DRAIN", &lk->lock_object, 0,
1175                             lk->lk_recurse, file, line);
1176                         WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
1177                             LK_TRYWIT(flags), file, line);
1178                         TD_LOCKS_INC(curthread);
1179                         STACK_SAVE(lk);
1180                 }
1181                 break;
1182         default:
1183                 if (flags & LK_INTERLOCK)
1184                         class->lc_unlock(ilk);
1185                 panic("%s: unknown lockmgr request 0x%x\n", __func__, op);
1186         }
1187
1188         if (flags & LK_INTERLOCK)
1189                 class->lc_unlock(ilk);
1190         if (wakeup_swapper)
1191                 kick_proc0();
1192
1193         return (error);
1194 }
1195
1196 void
1197 _lockmgr_disown(struct lock *lk, const char *file, int line)
1198 {
1199         uintptr_t tid, x;
1200
1201         tid = (uintptr_t)curthread;
1202         _lockmgr_assert(lk, KA_XLOCKED | KA_NOTRECURSED, file, line);
1203
1204         /*
1205          * If the owner is already LK_KERNPROC just skip the whole operation.
1206          */
1207         if (LK_HOLDER(lk->lk_lock) != tid)
1208                 return;
1209         lock_profile_release_lock(&lk->lock_object);
1210         LOCK_LOG_LOCK("XDISOWN", &lk->lock_object, 0, 0, file, line);
1211         WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
1212         TD_LOCKS_DEC(curthread);
1213         STACK_SAVE(lk);
1214
1215         /*
1216          * In order to preserve waiters flags, just spin.
1217          */
1218         for (;;) {
1219                 x = lk->lk_lock;
1220                 MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
1221                 x &= LK_ALL_WAITERS;
1222                 if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x,
1223                     LK_KERNPROC | x))
1224                         return;
1225                 cpu_spinwait();
1226         }
1227 }
1228
1229 void
1230 lockmgr_printinfo(struct lock *lk)
1231 {
1232         struct thread *td;
1233         uintptr_t x;
1234
1235         if (lk->lk_lock == LK_UNLOCKED)
1236                 printf("lock type %s: UNLOCKED\n", lk->lock_object.lo_name);
1237         else if (lk->lk_lock & LK_SHARE)
1238                 printf("lock type %s: SHARED (count %ju)\n",
1239                     lk->lock_object.lo_name,
1240                     (uintmax_t)LK_SHARERS(lk->lk_lock));
1241         else {
1242                 td = lockmgr_xholder(lk);
1243                 printf("lock type %s: EXCL by thread %p (pid %d)\n",
1244                     lk->lock_object.lo_name, td, td->td_proc->p_pid);
1245         }
1246
1247         x = lk->lk_lock;
1248         if (x & LK_EXCLUSIVE_WAITERS)
1249                 printf(" with exclusive waiters pending\n");
1250         if (x & LK_SHARED_WAITERS)
1251                 printf(" with shared waiters pending\n");
1252         if (x & LK_EXCLUSIVE_SPINNERS)
1253                 printf(" with exclusive spinners pending\n");
1254
1255         STACK_PRINT(lk);
1256 }
1257
1258 int
1259 lockstatus(struct lock *lk)
1260 {
1261         uintptr_t v, x;
1262         int ret;
1263
1264         ret = LK_SHARED;
1265         x = lk->lk_lock;
1266         v = LK_HOLDER(x);
1267
1268         if ((x & LK_SHARE) == 0) {
1269                 if (v == (uintptr_t)curthread || v == LK_KERNPROC)
1270                         ret = LK_EXCLUSIVE;
1271                 else
1272                         ret = LK_EXCLOTHER;
1273         } else if (x == LK_UNLOCKED)
1274                 ret = 0;
1275
1276         return (ret);
1277 }
1278
1279 #ifdef INVARIANT_SUPPORT
1280 #ifndef INVARIANTS
1281 #undef  _lockmgr_assert
1282 #endif
1283
1284 void
1285 _lockmgr_assert(struct lock *lk, int what, const char *file, int line)
1286 {
1287         int slocked = 0;
1288
1289         if (panicstr != NULL)
1290                 return;
1291         switch (what) {
1292         case KA_SLOCKED:
1293         case KA_SLOCKED | KA_NOTRECURSED:
1294         case KA_SLOCKED | KA_RECURSED:
1295                 slocked = 1;
1296         case KA_LOCKED:
1297         case KA_LOCKED | KA_NOTRECURSED:
1298         case KA_LOCKED | KA_RECURSED:
1299 #ifdef WITNESS
1300
1301                 /*
1302                  * We cannot trust WITNESS if the lock is held in exclusive
1303                  * mode and a call to lockmgr_disown() happened.
1304                  * Workaround this skipping the check if the lock is held in
1305                  * exclusive mode even for the KA_LOCKED case.
1306                  */
1307                 if (slocked || (lk->lk_lock & LK_SHARE)) {
1308                         witness_assert(&lk->lock_object, what, file, line);
1309                         break;
1310                 }
1311 #endif
1312                 if (lk->lk_lock == LK_UNLOCKED ||
1313                     ((lk->lk_lock & LK_SHARE) == 0 && (slocked ||
1314                     (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk)))))
1315                         panic("Lock %s not %slocked @ %s:%d\n",
1316                             lk->lock_object.lo_name, slocked ? "share" : "",
1317                             file, line);
1318
1319                 if ((lk->lk_lock & LK_SHARE) == 0) {
1320                         if (lockmgr_recursed(lk)) {
1321                                 if (what & KA_NOTRECURSED)
1322                                         panic("Lock %s recursed @ %s:%d\n",
1323                                             lk->lock_object.lo_name, file,
1324                                             line);
1325                         } else if (what & KA_RECURSED)
1326                                 panic("Lock %s not recursed @ %s:%d\n",
1327                                     lk->lock_object.lo_name, file, line);
1328                 }
1329                 break;
1330         case KA_XLOCKED:
1331         case KA_XLOCKED | KA_NOTRECURSED:
1332         case KA_XLOCKED | KA_RECURSED:
1333                 if (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk))
1334                         panic("Lock %s not exclusively locked @ %s:%d\n",
1335                             lk->lock_object.lo_name, file, line);
1336                 if (lockmgr_recursed(lk)) {
1337                         if (what & KA_NOTRECURSED)
1338                                 panic("Lock %s recursed @ %s:%d\n",
1339                                     lk->lock_object.lo_name, file, line);
1340                 } else if (what & KA_RECURSED)
1341                         panic("Lock %s not recursed @ %s:%d\n",
1342                             lk->lock_object.lo_name, file, line);
1343                 break;
1344         case KA_UNLOCKED:
1345                 if (lockmgr_xlocked(lk) || lockmgr_disowned(lk))
1346                         panic("Lock %s exclusively locked @ %s:%d\n",
1347                             lk->lock_object.lo_name, file, line);
1348                 break;
1349         default:
1350                 panic("Unknown lockmgr assertion: %d @ %s:%d\n", what, file,
1351                     line);
1352         }
1353 }
1354 #endif
1355
1356 #ifdef DDB
1357 int
1358 lockmgr_chain(struct thread *td, struct thread **ownerp)
1359 {
1360         struct lock *lk;
1361
1362         lk = td->td_wchan;
1363
1364         if (LOCK_CLASS(&lk->lock_object) != &lock_class_lockmgr)
1365                 return (0);
1366         db_printf("blocked on lockmgr %s", lk->lock_object.lo_name);
1367         if (lk->lk_lock & LK_SHARE)
1368                 db_printf("SHARED (count %ju)\n",
1369                     (uintmax_t)LK_SHARERS(lk->lk_lock));
1370         else
1371                 db_printf("EXCL\n");
1372         *ownerp = lockmgr_xholder(lk);
1373
1374         return (1);
1375 }
1376
1377 static void
1378 db_show_lockmgr(struct lock_object *lock)
1379 {
1380         struct thread *td;
1381         struct lock *lk;
1382
1383         lk = (struct lock *)lock;
1384
1385         db_printf(" state: ");
1386         if (lk->lk_lock == LK_UNLOCKED)
1387                 db_printf("UNLOCKED\n");
1388         else if (lk->lk_lock & LK_SHARE)
1389                 db_printf("SLOCK: %ju\n", (uintmax_t)LK_SHARERS(lk->lk_lock));
1390         else {
1391                 td = lockmgr_xholder(lk);
1392                 if (td == (struct thread *)LK_KERNPROC)
1393                         db_printf("XLOCK: LK_KERNPROC\n");
1394                 else
1395                         db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
1396                             td->td_tid, td->td_proc->p_pid,
1397                             td->td_proc->p_comm);
1398                 if (lockmgr_recursed(lk))
1399                         db_printf(" recursed: %d\n", lk->lk_recurse);
1400         }
1401         db_printf(" waiters: ");
1402         switch (lk->lk_lock & LK_ALL_WAITERS) {
1403         case LK_SHARED_WAITERS:
1404                 db_printf("shared\n");
1405                 break;
1406         case LK_EXCLUSIVE_WAITERS:
1407                 db_printf("exclusive\n");
1408                 break;
1409         case LK_ALL_WAITERS:
1410                 db_printf("shared and exclusive\n");
1411                 break;
1412         default:
1413                 db_printf("none\n");
1414         }
1415         db_printf(" spinners: ");
1416         if (lk->lk_lock & LK_EXCLUSIVE_SPINNERS)
1417                 db_printf("exclusive\n");
1418         else
1419                 db_printf("none\n");
1420 }
1421 #endif