sys/kern/kern_lock.c

   1 /*-
   2  * Copyright (c) 2008 Attilio Rao <attilio@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice(s), this list of conditions and the following disclaimer as
  10  *    the first lines of this file unmodified other than the possible
  11  *    addition of one or more copyright notices.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice(s), this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
  17  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19  * DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
  20  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  21  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  22  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  23  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
  26  * DAMAGE.
  27  */
  28
  29 #include "opt_ddb.h"
  30 #include "opt_hwpmc_hooks.h"
  31
  32 #include <sys/cdefs.h>
  33 __FBSDID("$FreeBSD$");
  34
  35 #include <sys/param.h>
  36 #include <sys/kdb.h>
  37 #include <sys/ktr.h>
  38 #include <sys/lock.h>
  39 #include <sys/lock_profile.h>
  40 #include <sys/lockmgr.h>
  41 #include <sys/mutex.h>
  42 #include <sys/proc.h>
  43 #include <sys/sleepqueue.h>
  44 #ifdef DEBUG_LOCKS
  45 #include <sys/stack.h>
  46 #endif
  47 #include <sys/sysctl.h>
  48 #include <sys/systm.h>
  49
  50 #include <machine/cpu.h>
  51
  52 #ifdef DDB
  53 #include <ddb/ddb.h>
  54 #endif
  55
  56 #ifdef HWPMC_HOOKS
  57 #include <sys/pmckern.h>
  58 PMC_SOFT_DECLARE( , , lock, failed);
  59 #endif
  60
  61 CTASSERT(((LK_ADAPTIVE | LK_NOSHARE) & LO_CLASSFLAGS) ==
  62     (LK_ADAPTIVE | LK_NOSHARE));
  63 CTASSERT(LK_UNLOCKED == (LK_UNLOCKED &
  64     ~(LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS)));
  65
  66 #define SQ_EXCLUSIVE_QUEUE      0
  67 #define SQ_SHARED_QUEUE         1
  68
  69 #ifndef INVARIANTS
  70 #define _lockmgr_assert(lk, what, file, line)
  71 #endif
  72
  73 #define TD_SLOCKS_INC(td)       ((td)->td_lk_slocks++)
  74 #define TD_SLOCKS_DEC(td)       ((td)->td_lk_slocks--)
  75
  76 #ifndef DEBUG_LOCKS
  77 #define STACK_PRINT(lk)
  78 #define STACK_SAVE(lk)
  79 #define STACK_ZERO(lk)
  80 #else
  81 #define STACK_PRINT(lk) stack_print_ddb(&(lk)->lk_stack)
  82 #define STACK_SAVE(lk)  stack_save(&(lk)->lk_stack)
  83 #define STACK_ZERO(lk)  stack_zero(&(lk)->lk_stack)
  84 #endif
  85
  86 #define LOCK_LOG2(lk, string, arg1, arg2)                               \
  87         if (LOCK_LOG_TEST(&(lk)->lock_object, 0))                       \
  88                 CTR2(KTR_LOCK, (string), (arg1), (arg2))
  89 #define LOCK_LOG3(lk, string, arg1, arg2, arg3)                         \
  90         if (LOCK_LOG_TEST(&(lk)->lock_object, 0))                       \
  91                 CTR3(KTR_LOCK, (string), (arg1), (arg2), (arg3))
  92
  93 #define GIANT_DECLARE                                                   \
  94         int _i = 0;                                                     \
  95         WITNESS_SAVE_DECL(Giant)
  96 #define GIANT_RESTORE() do {                                            \
  97         if (_i > 0) {                                                   \
  98                 while (_i--)                                            \
  99                         mtx_lock(&Giant);                               \
 100                 WITNESS_RESTORE(&Giant.lock_object, Giant);             \
 101         }                                                               \
 102 } while (0)
 103 #define GIANT_SAVE() do {                                               \
 104         if (mtx_owned(&Giant)) {                                        \
 105                 WITNESS_SAVE(&Giant.lock_object, Giant);                \
 106                 while (mtx_owned(&Giant)) {                             \
 107                         _i++;                                           \
 108                         mtx_unlock(&Giant);                             \
 109                 }                                                       \
 110         }                                                               \
 111 } while (0)
 112
 113 #define LK_CAN_SHARE(x, flags)                                          \
 114         (((x) & LK_SHARE) &&                                            \
 115         (((x) & (LK_EXCLUSIVE_WAITERS | LK_EXCLUSIVE_SPINNERS)) == 0 || \
 116         (curthread->td_lk_slocks != 0 && !(flags & LK_NODDLKTREAT)) ||  \
 117         (curthread->td_pflags & TDP_DEADLKTREAT)))
 118 #define LK_TRYOP(x)                                                     \
 119         ((x) & LK_NOWAIT)
 120
 121 #define LK_CAN_WITNESS(x)                                               \
 122         (((x) & LK_NOWITNESS) == 0 && !LK_TRYOP(x))
 123 #define LK_TRYWIT(x)                                                    \
 124         (LK_TRYOP(x) ? LOP_TRYLOCK : 0)
 125
 126 #define LK_CAN_ADAPT(lk, f)                                             \
 127         (((lk)->lock_object.lo_flags & LK_ADAPTIVE) != 0 &&             \
 128         ((f) & LK_SLEEPFAIL) == 0)
 129
 130 #define lockmgr_disowned(lk)                                            \
 131         (((lk)->lk_lock & ~(LK_FLAGMASK & ~LK_SHARE)) == LK_KERNPROC)
 132
 133 #define lockmgr_xlocked(lk)                                             \
 134         (((lk)->lk_lock & ~(LK_FLAGMASK & ~LK_SHARE)) == (uintptr_t)curthread)
 135
 136 static void     assert_lockmgr(const struct lock_object *lock, int how);
 137 #ifdef DDB
 138 static void     db_show_lockmgr(const struct lock_object *lock);
 139 #endif
 140 static void     lock_lockmgr(struct lock_object *lock, uintptr_t how);
 141 #ifdef KDTRACE_HOOKS
 142 static int      owner_lockmgr(const struct lock_object *lock,
 143                     struct thread **owner);
 144 #endif
 145 static uintptr_t unlock_lockmgr(struct lock_object *lock);
 146
 147 struct lock_class lock_class_lockmgr = {
 148         .lc_name = "lockmgr",
 149         .lc_flags = LC_RECURSABLE | LC_SLEEPABLE | LC_SLEEPLOCK | LC_UPGRADABLE,
 150         .lc_assert = assert_lockmgr,
 151 #ifdef DDB
 152         .lc_ddb_show = db_show_lockmgr,
 153 #endif
 154         .lc_lock = lock_lockmgr,
 155         .lc_unlock = unlock_lockmgr,
 156 #ifdef KDTRACE_HOOKS
 157         .lc_owner = owner_lockmgr,
 158 #endif
 159 };
 160
 161 static bool __always_inline lockmgr_slock_try(struct lock *lk, uintptr_t *xp,
 162     int flags);
 163 static bool __always_inline lockmgr_sunlock_try(struct lock *lk, uintptr_t x);
 164
 165 static void
 166 lockmgr_note_shared_acquire(struct lock *lk, int contested,
 167     uint64_t waittime, const char *file, int line, int flags)
 168 {
 169
 170         lock_profile_obtain_lock_success(&lk->lock_object, contested, waittime,
 171             file, line);
 172         LOCK_LOG_LOCK("SLOCK", &lk->lock_object, 0, 0, file, line);
 173         WITNESS_LOCK(&lk->lock_object, LK_TRYWIT(flags), file, line);
 174         TD_LOCKS_INC(curthread);
 175         TD_SLOCKS_INC(curthread);
 176         STACK_SAVE(lk);
 177 }
 178
 179 static void
 180 lockmgr_note_shared_release(struct lock *lk, const char *file, int line)
 181 {
 182
 183         lock_profile_release_lock(&lk->lock_object);
 184         WITNESS_UNLOCK(&lk->lock_object, 0, file, line);
 185         LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, file, line);
 186         TD_LOCKS_DEC(curthread);
 187         TD_SLOCKS_DEC(curthread);
 188 }
 189
 190 static void
 191 lockmgr_note_exclusive_acquire(struct lock *lk, int contested,
 192     uint64_t waittime, const char *file, int line, int flags)
 193 {
 194
 195         lock_profile_obtain_lock_success(&lk->lock_object, contested, waittime,
 196             file, line);
 197         LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0, lk->lk_recurse, file, line);
 198         WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE | LK_TRYWIT(flags), file,
 199             line);
 200         TD_LOCKS_INC(curthread);
 201         STACK_SAVE(lk);
 202 }
 203
 204 static void
 205 lockmgr_note_exclusive_release(struct lock *lk, const char *file, int line)
 206 {
 207
 208         lock_profile_release_lock(&lk->lock_object);
 209         LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0, lk->lk_recurse, file,
 210             line);
 211         WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
 212         TD_LOCKS_DEC(curthread);
 213 }
 214
 215 static void
 216 lockmgr_note_exclusive_upgrade(struct lock *lk, const char *file, int line,
 217     int flags)
 218 {
 219
 220         LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file,
 221             line);
 222         WITNESS_UPGRADE(&lk->lock_object, LOP_EXCLUSIVE |
 223             LK_TRYWIT(flags), file, line);
 224         TD_SLOCKS_DEC(curthread);
 225 }
 226
 227 static __inline struct thread *
 228 lockmgr_xholder(const struct lock *lk)
 229 {
 230         uintptr_t x;
 231
 232         x = lk->lk_lock;
 233         return ((x & LK_SHARE) ? NULL : (struct thread *)LK_HOLDER(x));
 234 }
 235
 236 /*
 237  * It assumes sleepq_lock held and returns with this one unheld.
 238  * It also assumes the generic interlock is sane and previously checked.
 239  * If LK_INTERLOCK is specified the interlock is not reacquired after the
 240  * sleep.
 241  */
 242 static __inline int
 243 sleeplk(struct lock *lk, u_int flags, struct lock_object *ilk,
 244     const char *wmesg, int pri, int timo, int queue)
 245 {
 246         GIANT_DECLARE;
 247         struct lock_class *class;
 248         int catch, error;
 249
 250         class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL;
 251         catch = pri & PCATCH;
 252         pri &= PRIMASK;
 253         error = 0;
 254
 255         LOCK_LOG3(lk, "%s: %p blocking on the %s sleepqueue", __func__, lk,
 256             (queue == SQ_EXCLUSIVE_QUEUE) ? "exclusive" : "shared");
 257
 258         if (flags & LK_INTERLOCK)
 259                 class->lc_unlock(ilk);
 260         if (queue == SQ_EXCLUSIVE_QUEUE && (flags & LK_SLEEPFAIL) != 0)
 261                 lk->lk_exslpfail++;
 262         GIANT_SAVE();
 263         sleepq_add(&lk->lock_object, NULL, wmesg, SLEEPQ_LK | (catch ?
 264             SLEEPQ_INTERRUPTIBLE : 0), queue);
 265         if ((flags & LK_TIMELOCK) && timo)
 266                 sleepq_set_timeout(&lk->lock_object, timo);
 267
 268         /*
 269          * Decisional switch for real sleeping.
 270          */
 271         if ((flags & LK_TIMELOCK) && timo && catch)
 272                 error = sleepq_timedwait_sig(&lk->lock_object, pri);
 273         else if ((flags & LK_TIMELOCK) && timo)
 274                 error = sleepq_timedwait(&lk->lock_object, pri);
 275         else if (catch)
 276                 error = sleepq_wait_sig(&lk->lock_object, pri);
 277         else
 278                 sleepq_wait(&lk->lock_object, pri);
 279         GIANT_RESTORE();
 280         if ((flags & LK_SLEEPFAIL) && error == 0)
 281                 error = ENOLCK;
 282
 283         return (error);
 284 }
 285
 286 static __inline int
 287 wakeupshlk(struct lock *lk, const char *file, int line)
 288 {
 289         uintptr_t v, x;
 290         u_int realexslp;
 291         int queue, wakeup_swapper;
 292
 293         wakeup_swapper = 0;
 294         for (;;) {
 295                 x = lk->lk_lock;
 296                 if (lockmgr_sunlock_try(lk, x))
 297                         break;
 298
 299                 /*
 300                  * We should have a sharer with waiters, so enter the hard
 301                  * path in order to handle wakeups correctly.
 302                  */
 303                 sleepq_lock(&lk->lock_object);
 304                 x = lk->lk_lock & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
 305                 v = LK_UNLOCKED;
 306
 307                 /*
 308                  * If the lock has exclusive waiters, give them preference in
 309                  * order to avoid deadlock with shared runners up.
 310                  * If interruptible sleeps left the exclusive queue empty
 311                  * avoid a starvation for the threads sleeping on the shared
 312                  * queue by giving them precedence and cleaning up the
 313                  * exclusive waiters bit anyway.
 314                  * Please note that lk_exslpfail count may be lying about
 315                  * the real number of waiters with the LK_SLEEPFAIL flag on
 316                  * because they may be used in conjunction with interruptible
 317                  * sleeps so lk_exslpfail might be considered an 'upper limit'
 318                  * bound, including the edge cases.
 319                  */
 320                 realexslp = sleepq_sleepcnt(&lk->lock_object,
 321                     SQ_EXCLUSIVE_QUEUE);
 322                 if ((x & LK_EXCLUSIVE_WAITERS) != 0 && realexslp != 0) {
 323                         if (lk->lk_exslpfail < realexslp) {
 324                                 lk->lk_exslpfail = 0;
 325                                 queue = SQ_EXCLUSIVE_QUEUE;
 326                                 v |= (x & LK_SHARED_WAITERS);
 327                         } else {
 328                                 lk->lk_exslpfail = 0;
 329                                 LOCK_LOG2(lk,
 330                                     "%s: %p has only LK_SLEEPFAIL sleepers",
 331                                     __func__, lk);
 332                                 LOCK_LOG2(lk,
 333                             "%s: %p waking up threads on the exclusive queue",
 334                                     __func__, lk);
 335                                 wakeup_swapper =
 336                                     sleepq_broadcast(&lk->lock_object,
 337                                     SLEEPQ_LK, 0, SQ_EXCLUSIVE_QUEUE);
 338                                 queue = SQ_SHARED_QUEUE;
 339                         }
 340
 341                 } else {
 342
 343                         /*
 344                          * Exclusive waiters sleeping with LK_SLEEPFAIL on
 345                          * and using interruptible sleeps/timeout may have
 346                          * left spourious lk_exslpfail counts on, so clean
 347                          * it up anyway.
 348                          */
 349                         lk->lk_exslpfail = 0;
 350                         queue = SQ_SHARED_QUEUE;
 351                 }
 352
 353                 if (!atomic_cmpset_rel_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x,
 354                     v)) {
 355                         sleepq_release(&lk->lock_object);
 356                         continue;
 357                 }
 358                 LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue",
 359                     __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
 360                     "exclusive");
 361                 wakeup_swapper |= sleepq_broadcast(&lk->lock_object, SLEEPQ_LK,
 362                     0, queue);
 363                 sleepq_release(&lk->lock_object);
 364                 break;
 365         }
 366
 367         lockmgr_note_shared_release(lk, file, line);
 368         return (wakeup_swapper);
 369 }
 370
 371 static void
 372 assert_lockmgr(const struct lock_object *lock, int what)
 373 {
 374
 375         panic("lockmgr locks do not support assertions");
 376 }
 377
 378 static void
 379 lock_lockmgr(struct lock_object *lock, uintptr_t how)
 380 {
 381
 382         panic("lockmgr locks do not support sleep interlocking");
 383 }
 384
 385 static uintptr_t
 386 unlock_lockmgr(struct lock_object *lock)
 387 {
 388
 389         panic("lockmgr locks do not support sleep interlocking");
 390 }
 391
 392 #ifdef KDTRACE_HOOKS
 393 static int
 394 owner_lockmgr(const struct lock_object *lock, struct thread **owner)
 395 {
 396
 397         panic("lockmgr locks do not support owner inquiring");
 398 }
 399 #endif
 400
 401 void
 402 lockinit(struct lock *lk, int pri, const char *wmesg, int timo, int flags)
 403 {
 404         int iflags;
 405
 406         MPASS((flags & ~LK_INIT_MASK) == 0);
 407         ASSERT_ATOMIC_LOAD_PTR(lk->lk_lock,
 408             ("%s: lockmgr not aligned for %s: %p", __func__, wmesg,
 409             &lk->lk_lock));
 410
 411         iflags = LO_SLEEPABLE | LO_UPGRADABLE;
 412         if (flags & LK_CANRECURSE)
 413                 iflags |= LO_RECURSABLE;
 414         if ((flags & LK_NODUP) == 0)
 415                 iflags |= LO_DUPOK;
 416         if (flags & LK_NOPROFILE)
 417                 iflags |= LO_NOPROFILE;
 418         if ((flags & LK_NOWITNESS) == 0)
 419                 iflags |= LO_WITNESS;
 420         if (flags & LK_QUIET)
 421                 iflags |= LO_QUIET;
 422         if (flags & LK_IS_VNODE)
 423                 iflags |= LO_IS_VNODE;
 424         iflags |= flags & (LK_ADAPTIVE | LK_NOSHARE);
 425
 426         lock_init(&lk->lock_object, &lock_class_lockmgr, wmesg, NULL, iflags);
 427         lk->lk_lock = LK_UNLOCKED;
 428         lk->lk_recurse = 0;
 429         lk->lk_exslpfail = 0;
 430         lk->lk_timo = timo;
 431         lk->lk_pri = pri;
 432         STACK_ZERO(lk);
 433 }
 434
 435 /*
 436  * XXX: Gross hacks to manipulate external lock flags after
 437  * initialization.  Used for certain vnode and buf locks.
 438  */
 439 void
 440 lockallowshare(struct lock *lk)
 441 {
 442
 443         lockmgr_assert(lk, KA_XLOCKED);
 444         lk->lock_object.lo_flags &= ~LK_NOSHARE;
 445 }
 446
 447 void
 448 lockdisableshare(struct lock *lk)
 449 {
 450
 451         lockmgr_assert(lk, KA_XLOCKED);
 452         lk->lock_object.lo_flags |= LK_NOSHARE;
 453 }
 454
 455 void
 456 lockallowrecurse(struct lock *lk)
 457 {
 458
 459         lockmgr_assert(lk, KA_XLOCKED);
 460         lk->lock_object.lo_flags |= LO_RECURSABLE;
 461 }
 462
 463 void
 464 lockdisablerecurse(struct lock *lk)
 465 {
 466
 467         lockmgr_assert(lk, KA_XLOCKED);
 468         lk->lock_object.lo_flags &= ~LO_RECURSABLE;
 469 }
 470
 471 void
 472 lockdestroy(struct lock *lk)
 473 {
 474
 475         KASSERT(lk->lk_lock == LK_UNLOCKED, ("lockmgr still held"));
 476         KASSERT(lk->lk_recurse == 0, ("lockmgr still recursed"));
 477         KASSERT(lk->lk_exslpfail == 0, ("lockmgr still exclusive waiters"));
 478         lock_destroy(&lk->lock_object);
 479 }
 480
 481 static bool __always_inline
 482 lockmgr_slock_try(struct lock *lk, uintptr_t *xp, int flags)
 483 {
 484
 485         /*
 486          * If no other thread has an exclusive lock, or
 487          * no exclusive waiter is present, bump the count of
 488          * sharers.  Since we have to preserve the state of
 489          * waiters, if we fail to acquire the shared lock
 490          * loop back and retry.
 491          */
 492         *xp = lk->lk_lock;
 493         while (LK_CAN_SHARE(*xp, flags)) {
 494                 if (atomic_fcmpset_acq_ptr(&lk->lk_lock, xp,
 495                     *xp + LK_ONE_SHARER)) {
 496                         return (true);
 497                 }
 498         }
 499         return (false);
 500 }
 501
 502 static bool __always_inline
 503 lockmgr_sunlock_try(struct lock *lk, uintptr_t x)
 504 {
 505
 506         for (;;) {
 507                 /*
 508                  * If there is more than one shared lock held, just drop one
 509                  * and return.
 510                  */
 511                 if (LK_SHARERS(x) > 1) {
 512                         if (atomic_fcmpset_rel_ptr(&lk->lk_lock, &x,
 513                             x - LK_ONE_SHARER))
 514                                 return (true);
 515                         continue;
 516                 }
 517
 518                 /*
 519                  * If there are not waiters on the exclusive queue, drop the
 520                  * lock quickly.
 521                  */
 522                 if ((x & LK_ALL_WAITERS) == 0) {
 523                         MPASS((x & ~LK_EXCLUSIVE_SPINNERS) ==
 524                             LK_SHARERS_LOCK(1));
 525                         if (atomic_fcmpset_rel_ptr(&lk->lk_lock, &x,
 526                             LK_UNLOCKED))
 527                                 return (true);
 528                         continue;
 529                 }
 530                 break;
 531         }
 532         return (false);
 533 }
 534
 535 int
 536 lockmgr_lock_fast_path(struct lock *lk, u_int flags, struct lock_object *ilk,
 537     const char *file, int line)
 538 {
 539         struct lock_class *class;
 540         uintptr_t x, v, tid;
 541         u_int op;
 542         bool locked;
 543
 544         op = flags & LK_TYPE_MASK;
 545         locked = false;
 546         switch (op) {
 547         case LK_SHARED:
 548                 if (LK_CAN_WITNESS(flags))
 549                         WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
 550                             file, line, flags & LK_INTERLOCK ? ilk : NULL);
 551                 if (__predict_false(lk->lock_object.lo_flags & LK_NOSHARE))
 552                         break;
 553                 if (lockmgr_slock_try(lk, &x, flags)) {
 554                         lockmgr_note_shared_acquire(lk, 0, 0,
 555                             file, line, flags);
 556                         locked = true;
 557                 }
 558                 break;
 559         case LK_EXCLUSIVE:
 560                 if (LK_CAN_WITNESS(flags))
 561                         WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
 562                             LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ?
 563                             ilk : NULL);
 564                 tid = (uintptr_t)curthread;
 565                 if (lk->lk_lock == LK_UNLOCKED &&
 566                     atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) {
 567                         lockmgr_note_exclusive_acquire(lk, 0, 0, file, line,
 568                             flags);
 569                         locked = true;
 570                 }
 571                 break;
 572         case LK_UPGRADE:
 573         case LK_TRYUPGRADE:
 574                 _lockmgr_assert(lk, KA_SLOCKED, file, line);
 575                 tid = (uintptr_t)curthread;
 576                 v = lk->lk_lock;
 577                 x = v & LK_ALL_WAITERS;
 578                 v &= LK_EXCLUSIVE_SPINNERS;
 579                 if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x | v,
 580                     tid | x)) {
 581                         lockmgr_note_exclusive_upgrade(lk, file, line, flags);
 582                         locked = true;
 583                 }
 584                 break;
 585         default:
 586                 break;
 587         }
 588         if (__predict_true(locked)) {
 589                 if (__predict_false(flags & LK_INTERLOCK)) {
 590                         class = LOCK_CLASS(ilk);
 591                         class->lc_unlock(ilk);
 592                 }
 593                 return (0);
 594         } else {
 595                 return (__lockmgr_args(lk, flags, ilk, LK_WMESG_DEFAULT,
 596                     LK_PRIO_DEFAULT, LK_TIMO_DEFAULT, file, line));
 597         }
 598 }
 599
 600 int
 601 lockmgr_unlock_fast_path(struct lock *lk, u_int flags, struct lock_object *ilk)
 602 {
 603         struct lock_class *class;
 604         uintptr_t x, tid;
 605         bool unlocked;
 606         const char *file;
 607         int line;
 608
 609         file = __FILE__;
 610         line = __LINE__;
 611
 612         _lockmgr_assert(lk, KA_LOCKED, file, line);
 613         unlocked = false;
 614         x = lk->lk_lock;
 615         if (__predict_true(x & LK_SHARE) != 0) {
 616                 if (lockmgr_sunlock_try(lk, x)) {
 617                         lockmgr_note_shared_release(lk, file, line);
 618                         unlocked = true;
 619                 }
 620         } else {
 621                 tid = (uintptr_t)curthread;
 622                 if (!lockmgr_recursed(lk) &&
 623                     atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED)) {
 624                         lockmgr_note_exclusive_release(lk, file, line);
 625                         unlocked = true;
 626                 }
 627         }
 628         if (__predict_true(unlocked)) {
 629                 if (__predict_false(flags & LK_INTERLOCK)) {
 630                         class = LOCK_CLASS(ilk);
 631                         class->lc_unlock(ilk);
 632                 }
 633                 return (0);
 634         } else {
 635                 return (__lockmgr_args(lk, flags | LK_RELEASE, ilk, LK_WMESG_DEFAULT,
 636                     LK_PRIO_DEFAULT, LK_TIMO_DEFAULT, LOCK_FILE, LOCK_LINE));
 637         }
 638 }
 639
 640 int
 641 __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
 642     const char *wmesg, int pri, int timo, const char *file, int line)
 643 {
 644         GIANT_DECLARE;
 645         struct lock_class *class;
 646         const char *iwmesg;
 647         uintptr_t tid, v, x;
 648         u_int op, realexslp;
 649         int error, ipri, itimo, queue, wakeup_swapper;
 650 #ifdef LOCK_PROFILING
 651         uint64_t waittime = 0;
 652         int contested = 0;
 653 #endif
 654
 655         error = 0;
 656         tid = (uintptr_t)curthread;
 657         op = (flags & LK_TYPE_MASK);
 658         iwmesg = (wmesg == LK_WMESG_DEFAULT) ? lk->lock_object.lo_name : wmesg;
 659         ipri = (pri == LK_PRIO_DEFAULT) ? lk->lk_pri : pri;
 660         itimo = (timo == LK_TIMO_DEFAULT) ? lk->lk_timo : timo;
 661
 662         MPASS((flags & ~LK_TOTAL_MASK) == 0);
 663         KASSERT((op & (op - 1)) == 0,
 664             ("%s: Invalid requested operation @ %s:%d", __func__, file, line));
 665         KASSERT((flags & (LK_NOWAIT | LK_SLEEPFAIL)) == 0 ||
 666             (op != LK_DOWNGRADE && op != LK_RELEASE),
 667             ("%s: Invalid flags in regard of the operation desired @ %s:%d",
 668             __func__, file, line));
 669         KASSERT((flags & LK_INTERLOCK) == 0 || ilk != NULL,
 670             ("%s: LK_INTERLOCK passed without valid interlock @ %s:%d",
 671             __func__, file, line));
 672         KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
 673             ("%s: idle thread %p on lockmgr %s @ %s:%d", __func__, curthread,
 674             lk->lock_object.lo_name, file, line));
 675
 676         class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL;
 677         if (panicstr != NULL) {
 678                 if (flags & LK_INTERLOCK)
 679                         class->lc_unlock(ilk);
 680                 return (0);
 681         }
 682
 683         if (lk->lock_object.lo_flags & LK_NOSHARE) {
 684                 switch (op) {
 685                 case LK_SHARED:
 686                         op = LK_EXCLUSIVE;
 687                         break;
 688                 case LK_UPGRADE:
 689                 case LK_TRYUPGRADE:
 690                 case LK_DOWNGRADE:
 691                         _lockmgr_assert(lk, KA_XLOCKED | KA_NOTRECURSED,
 692                             file, line);
 693                         if (flags & LK_INTERLOCK)
 694                                 class->lc_unlock(ilk);
 695                         return (0);
 696                 }
 697         }
 698
 699         wakeup_swapper = 0;
 700         switch (op) {
 701         case LK_SHARED:
 702                 if (LK_CAN_WITNESS(flags))
 703                         WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
 704                             file, line, flags & LK_INTERLOCK ? ilk : NULL);
 705                 for (;;) {
 706                         if (lockmgr_slock_try(lk, &x, flags))
 707                                 break;
 708 #ifdef HWPMC_HOOKS
 709                         PMC_SOFT_CALL( , , lock, failed);
 710 #endif
 711                         lock_profile_obtain_lock_failed(&lk->lock_object,
 712                             &contested, &waittime);
 713
 714                         /*
 715                          * If the lock is already held by curthread in
 716                          * exclusive way avoid a deadlock.
 717                          */
 718                         if (LK_HOLDER(x) == tid) {
 719                                 LOCK_LOG2(lk,
 720                                     "%s: %p already held in exclusive mode",
 721                                     __func__, lk);
 722                                 error = EDEADLK;
 723                                 break;
 724                         }
 725
 726                         /*
 727                          * If the lock is expected to not sleep just give up
 728                          * and return.
 729                          */
 730                         if (LK_TRYOP(flags)) {
 731                                 LOCK_LOG2(lk, "%s: %p fails the try operation",
 732                                     __func__, lk);
 733                                 error = EBUSY;
 734                                 break;
 735                         }
 736
 737                         /*
 738                          * Acquire the sleepqueue chain lock because we
 739                          * probabilly will need to manipulate waiters flags.
 740                          */
 741                         sleepq_lock(&lk->lock_object);
 742                         x = lk->lk_lock;
 743
 744                         /*
 745                          * if the lock can be acquired in shared mode, try
 746                          * again.
 747                          */
 748                         if (LK_CAN_SHARE(x, flags)) {
 749                                 sleepq_release(&lk->lock_object);
 750                                 continue;
 751                         }
 752
 753                         /*
 754                          * Try to set the LK_SHARED_WAITERS flag.  If we fail,
 755                          * loop back and retry.
 756                          */
 757                         if ((x & LK_SHARED_WAITERS) == 0) {
 758                                 if (!atomic_cmpset_acq_ptr(&lk->lk_lock, x,
 759                                     x | LK_SHARED_WAITERS)) {
 760                                         sleepq_release(&lk->lock_object);
 761                                         continue;
 762                                 }
 763                                 LOCK_LOG2(lk, "%s: %p set shared waiters flag",
 764                                     __func__, lk);
 765                         }
 766
 767                         /*
 768                          * As far as we have been unable to acquire the
 769                          * shared lock and the shared waiters flag is set,
 770                          * we will sleep.
 771                          */
 772                         error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,
 773                             SQ_SHARED_QUEUE);
 774                         flags &= ~LK_INTERLOCK;
 775                         if (error) {
 776                                 LOCK_LOG3(lk,
 777                                     "%s: interrupted sleep for %p with %d",
 778                                     __func__, lk, error);
 779                                 break;
 780                         }
 781                         LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
 782                             __func__, lk);
 783                 }
 784                 if (error == 0) {
 785 #ifdef LOCK_PROFILING
 786                         lockmgr_note_shared_acquire(lk, contested, waittime,
 787                             file, line, flags);
 788 #else
 789                         lockmgr_note_shared_acquire(lk, 0, 0, file, line,
 790                             flags);
 791 #endif
 792                 }
 793                 break;
 794         case LK_UPGRADE:
 795         case LK_TRYUPGRADE:
 796                 _lockmgr_assert(lk, KA_SLOCKED, file, line);
 797                 v = lk->lk_lock;
 798                 x = v & LK_ALL_WAITERS;
 799                 v &= LK_EXCLUSIVE_SPINNERS;
 800
 801                 /*
 802                  * Try to switch from one shared lock to an exclusive one.
 803                  * We need to preserve waiters flags during the operation.
 804                  */
 805                 if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x | v,
 806                     tid | x)) {
 807                         LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file,
 808                             line);
 809                         WITNESS_UPGRADE(&lk->lock_object, LOP_EXCLUSIVE |
 810                             LK_TRYWIT(flags), file, line);
 811                         TD_SLOCKS_DEC(curthread);
 812                         break;
 813                 }
 814
 815                 /*
 816                  * In LK_TRYUPGRADE mode, do not drop the lock,
 817                  * returning EBUSY instead.
 818                  */
 819                 if (op == LK_TRYUPGRADE) {
 820                         LOCK_LOG2(lk, "%s: %p failed the nowait upgrade",
 821                             __func__, lk);
 822                         error = EBUSY;
 823                         break;
 824                 }
 825
 826                 /*
 827                  * We have been unable to succeed in upgrading, so just
 828                  * give up the shared lock.
 829                  */
 830                 wakeup_swapper |= wakeupshlk(lk, file, line);
 831
 832                 /* FALLTHROUGH */
 833         case LK_EXCLUSIVE:
 834                 if (LK_CAN_WITNESS(flags))
 835                         WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
 836                             LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ?
 837                             ilk : NULL);
 838
 839                 /*
 840                  * If curthread already holds the lock and this one is
 841                  * allowed to recurse, simply recurse on it.
 842                  */
 843                 if (lockmgr_xlocked(lk)) {
 844                         if ((flags & LK_CANRECURSE) == 0 &&
 845                             (lk->lock_object.lo_flags & LO_RECURSABLE) == 0) {
 846
 847                                 /*
 848                                  * If the lock is expected to not panic just
 849                                  * give up and return.
 850                                  */
 851                                 if (LK_TRYOP(flags)) {
 852                                         LOCK_LOG2(lk,
 853                                             "%s: %p fails the try operation",
 854                                             __func__, lk);
 855                                         error = EBUSY;
 856                                         break;
 857                                 }
 858                                 if (flags & LK_INTERLOCK)
 859                                         class->lc_unlock(ilk);
 860                 panic("%s: recursing on non recursive lockmgr %s @ %s:%d\n",
 861                                     __func__, iwmesg, file, line);
 862                         }
 863                         lk->lk_recurse++;
 864                         LOCK_LOG2(lk, "%s: %p recursing", __func__, lk);
 865                         LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0,
 866                             lk->lk_recurse, file, line);
 867                         WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
 868                             LK_TRYWIT(flags), file, line);
 869                         TD_LOCKS_INC(curthread);
 870                         break;
 871                 }
 872
 873                 for (;;) {
 874                         if (lk->lk_lock == LK_UNLOCKED &&
 875                             atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid))
 876                                 break;
 877 #ifdef HWPMC_HOOKS
 878                         PMC_SOFT_CALL( , , lock, failed);
 879 #endif
 880                         lock_profile_obtain_lock_failed(&lk->lock_object,
 881                             &contested, &waittime);
 882
 883                         /*
 884                          * If the lock is expected to not sleep just give up
 885                          * and return.
 886                          */
 887                         if (LK_TRYOP(flags)) {
 888                                 LOCK_LOG2(lk, "%s: %p fails the try operation",
 889                                     __func__, lk);
 890                                 error = EBUSY;
 891                                 break;
 892                         }
 893
 894                         /*
 895                          * Acquire the sleepqueue chain lock because we
 896                          * probabilly will need to manipulate waiters flags.
 897                          */
 898                         sleepq_lock(&lk->lock_object);
 899                         x = lk->lk_lock;
 900
 901                         /*
 902                          * if the lock has been released while we spun on
 903                          * the sleepqueue chain lock just try again.
 904                          */
 905                         if (x == LK_UNLOCKED) {
 906                                 sleepq_release(&lk->lock_object);
 907                                 continue;
 908                         }
 909
 910                         /*
 911                          * The lock can be in the state where there is a
 912                          * pending queue of waiters, but still no owner.
 913                          * This happens when the lock is contested and an
 914                          * owner is going to claim the lock.
 915                          * If curthread is the one successfully acquiring it
 916                          * claim lock ownership and return, preserving waiters
 917                          * flags.
 918                          */
 919                         v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
 920                         if ((x & ~v) == LK_UNLOCKED) {
 921                                 v &= ~LK_EXCLUSIVE_SPINNERS;
 922                                 if (atomic_cmpset_acq_ptr(&lk->lk_lock, x,
 923                                     tid | v)) {
 924                                         sleepq_release(&lk->lock_object);
 925                                         LOCK_LOG2(lk,
 926                                             "%s: %p claimed by a new writer",
 927                                             __func__, lk);
 928                                         break;
 929                                 }
 930                                 sleepq_release(&lk->lock_object);
 931                                 continue;
 932                         }
 933
 934                         /*
 935                          * Try to set the LK_EXCLUSIVE_WAITERS flag.  If we
 936                          * fail, loop back and retry.
 937                          */
 938                         if ((x & LK_EXCLUSIVE_WAITERS) == 0) {
 939                                 if (!atomic_cmpset_ptr(&lk->lk_lock, x,
 940                                     x | LK_EXCLUSIVE_WAITERS)) {
 941                                         sleepq_release(&lk->lock_object);
 942                                         continue;
 943                                 }
 944                                 LOCK_LOG2(lk, "%s: %p set excl waiters flag",
 945                                     __func__, lk);
 946                         }
 947
 948                         /*
 949                          * As far as we have been unable to acquire the
 950                          * exclusive lock and the exclusive waiters flag
 951                          * is set, we will sleep.
 952                          */
 953                         error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,
 954                             SQ_EXCLUSIVE_QUEUE);
 955                         flags &= ~LK_INTERLOCK;
 956                         if (error) {
 957                                 LOCK_LOG3(lk,
 958                                     "%s: interrupted sleep for %p with %d",
 959                                     __func__, lk, error);
 960                                 break;
 961                         }
 962                         LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
 963                             __func__, lk);
 964                 }
 965                 if (error == 0) {
 966 #ifdef LOCK_PROFILING
 967                         lockmgr_note_exclusive_acquire(lk, contested, waittime,
 968                             file, line, flags);
 969 #else
 970                         lockmgr_note_exclusive_acquire(lk, 0, 0, file, line,
 971                             flags);
 972 #endif
 973                 }
 974                 break;
 975         case LK_DOWNGRADE:
 976                 _lockmgr_assert(lk, KA_XLOCKED, file, line);
 977                 LOCK_LOG_LOCK("XDOWNGRADE", &lk->lock_object, 0, 0, file, line);
 978                 WITNESS_DOWNGRADE(&lk->lock_object, 0, file, line);
 979
 980                 /*
 981                  * Panic if the lock is recursed.
 982                  */
 983                 if (lockmgr_xlocked(lk) && lockmgr_recursed(lk)) {
 984                         if (flags & LK_INTERLOCK)
 985                                 class->lc_unlock(ilk);
 986                         panic("%s: downgrade a recursed lockmgr %s @ %s:%d\n",
 987                             __func__, iwmesg, file, line);
 988                 }
 989                 TD_SLOCKS_INC(curthread);
 990
 991                 /*
 992                  * In order to preserve waiters flags, just spin.
 993                  */
 994                 for (;;) {
 995                         x = lk->lk_lock;
 996                         MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
 997                         x &= LK_ALL_WAITERS;
 998                         if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x,
 999                             LK_SHARERS_LOCK(1) | x))
1000                                 break;
1001                         cpu_spinwait();
1002                 }
1003                 break;
1004         case LK_RELEASE:
1005                 _lockmgr_assert(lk, KA_LOCKED, file, line);
1006                 x = lk->lk_lock;
1007
1008                 if ((x & LK_SHARE) == 0) {
1009
1010                         /*
1011                          * As first option, treact the lock as if it has not
1012                          * any waiter.
1013                          * Fix-up the tid var if the lock has been disowned.
1014                          */
1015                         if (LK_HOLDER(x) == LK_KERNPROC)
1016                                 tid = LK_KERNPROC;
1017                         else {
1018                                 WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE,
1019                                     file, line);
1020                                 TD_LOCKS_DEC(curthread);
1021                         }
1022                         LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0,
1023                             lk->lk_recurse, file, line);
1024
1025                         /*
1026                          * The lock is held in exclusive mode.
1027                          * If the lock is recursed also, then unrecurse it.
1028                          */
1029                         if (lockmgr_xlocked(lk) && lockmgr_recursed(lk)) {
1030                                 LOCK_LOG2(lk, "%s: %p unrecursing", __func__,
1031                                     lk);
1032                                 lk->lk_recurse--;
1033                                 break;
1034                         }
1035                         if (tid != LK_KERNPROC)
1036                                 lock_profile_release_lock(&lk->lock_object);
1037
1038                         if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid,
1039                             LK_UNLOCKED))
1040                                 break;
1041
1042                         sleepq_lock(&lk->lock_object);
1043                         x = lk->lk_lock;
1044                         v = LK_UNLOCKED;
1045
1046                         /*
1047                          * If the lock has exclusive waiters, give them
1048                          * preference in order to avoid deadlock with
1049                          * shared runners up.
1050                          * If interruptible sleeps left the exclusive queue
1051                          * empty avoid a starvation for the threads sleeping
1052                          * on the shared queue by giving them precedence
1053                          * and cleaning up the exclusive waiters bit anyway.
1054                          * Please note that lk_exslpfail count may be lying
1055                          * about the real number of waiters with the
1056                          * LK_SLEEPFAIL flag on because they may be used in
1057                          * conjunction with interruptible sleeps so
1058                          * lk_exslpfail might be considered an 'upper limit'
1059                          * bound, including the edge cases.
1060                          */
1061                         MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
1062                         realexslp = sleepq_sleepcnt(&lk->lock_object,
1063                             SQ_EXCLUSIVE_QUEUE);
1064                         if ((x & LK_EXCLUSIVE_WAITERS) != 0 && realexslp != 0) {
1065                                 if (lk->lk_exslpfail < realexslp) {
1066                                         lk->lk_exslpfail = 0;
1067                                         queue = SQ_EXCLUSIVE_QUEUE;
1068                                         v |= (x & LK_SHARED_WAITERS);
1069                                 } else {
1070                                         lk->lk_exslpfail = 0;
1071                                         LOCK_LOG2(lk,
1072                                         "%s: %p has only LK_SLEEPFAIL sleepers",
1073                                             __func__, lk);
1074                                         LOCK_LOG2(lk,
1075                         "%s: %p waking up threads on the exclusive queue",
1076                                             __func__, lk);
1077                                         wakeup_swapper =
1078                                             sleepq_broadcast(&lk->lock_object,
1079                                             SLEEPQ_LK, 0, SQ_EXCLUSIVE_QUEUE);
1080                                         queue = SQ_SHARED_QUEUE;
1081                                 }
1082                         } else {
1083
1084                                 /*
1085                                  * Exclusive waiters sleeping with LK_SLEEPFAIL
1086                                  * on and using interruptible sleeps/timeout
1087                                  * may have left spourious lk_exslpfail counts
1088                                  * on, so clean it up anyway.
1089                                  */
1090                                 lk->lk_exslpfail = 0;
1091                                 queue = SQ_SHARED_QUEUE;
1092                         }
1093
1094                         LOCK_LOG3(lk,
1095                             "%s: %p waking up threads on the %s queue",
1096                             __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
1097                             "exclusive");
1098                         atomic_store_rel_ptr(&lk->lk_lock, v);
1099                         wakeup_swapper |= sleepq_broadcast(&lk->lock_object,
1100                             SLEEPQ_LK, 0, queue);
1101                         sleepq_release(&lk->lock_object);
1102                         break;
1103                 } else
1104                         wakeup_swapper = wakeupshlk(lk, file, line);
1105                 break;
1106         case LK_DRAIN:
1107                 if (LK_CAN_WITNESS(flags))
1108                         WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
1109                             LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ?
1110                             ilk : NULL);
1111
1112                 /*
1113                  * Trying to drain a lock we already own will result in a
1114                  * deadlock.
1115                  */
1116                 if (lockmgr_xlocked(lk)) {
1117                         if (flags & LK_INTERLOCK)
1118                                 class->lc_unlock(ilk);
1119                         panic("%s: draining %s with the lock held @ %s:%d\n",
1120                             __func__, iwmesg, file, line);
1121                 }
1122
1123                 for (;;) {
1124                         if (lk->lk_lock == LK_UNLOCKED &&
1125                             atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid))
1126                                 break;
1127
1128 #ifdef HWPMC_HOOKS
1129                         PMC_SOFT_CALL( , , lock, failed);
1130 #endif
1131                         lock_profile_obtain_lock_failed(&lk->lock_object,
1132                             &contested, &waittime);
1133
1134                         /*
1135                          * If the lock is expected to not sleep just give up
1136                          * and return.
1137                          */
1138                         if (LK_TRYOP(flags)) {
1139                                 LOCK_LOG2(lk, "%s: %p fails the try operation",
1140                                     __func__, lk);
1141                                 error = EBUSY;
1142                                 break;
1143                         }
1144
1145                         /*
1146                          * Acquire the sleepqueue chain lock because we
1147                          * probabilly will need to manipulate waiters flags.
1148                          */
1149                         sleepq_lock(&lk->lock_object);
1150                         x = lk->lk_lock;
1151
1152                         /*
1153                          * if the lock has been released while we spun on
1154                          * the sleepqueue chain lock just try again.
1155                          */
1156                         if (x == LK_UNLOCKED) {
1157                                 sleepq_release(&lk->lock_object);
1158                                 continue;
1159                         }
1160
1161                         v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
1162                         if ((x & ~v) == LK_UNLOCKED) {
1163                                 v = (x & ~LK_EXCLUSIVE_SPINNERS);
1164
1165                                 /*
1166                                  * If interruptible sleeps left the exclusive
1167                                  * queue empty avoid a starvation for the
1168                                  * threads sleeping on the shared queue by
1169                                  * giving them precedence and cleaning up the
1170                                  * exclusive waiters bit anyway.
1171                                  * Please note that lk_exslpfail count may be
1172                                  * lying about the real number of waiters with
1173                                  * the LK_SLEEPFAIL flag on because they may
1174                                  * be used in conjunction with interruptible
1175                                  * sleeps so lk_exslpfail might be considered
1176                                  * an 'upper limit' bound, including the edge
1177                                  * cases.
1178                                  */
1179                                 if (v & LK_EXCLUSIVE_WAITERS) {
1180                                         queue = SQ_EXCLUSIVE_QUEUE;
1181                                         v &= ~LK_EXCLUSIVE_WAITERS;
1182                                 } else {
1183
1184                                         /*
1185                                          * Exclusive waiters sleeping with
1186                                          * LK_SLEEPFAIL on and using
1187                                          * interruptible sleeps/timeout may
1188                                          * have left spourious lk_exslpfail
1189                                          * counts on, so clean it up anyway.
1190                                          */
1191                                         MPASS(v & LK_SHARED_WAITERS);
1192                                         lk->lk_exslpfail = 0;
1193                                         queue = SQ_SHARED_QUEUE;
1194                                         v &= ~LK_SHARED_WAITERS;
1195                                 }
1196                                 if (queue == SQ_EXCLUSIVE_QUEUE) {
1197                                         realexslp =
1198                                             sleepq_sleepcnt(&lk->lock_object,
1199                                             SQ_EXCLUSIVE_QUEUE);
1200                                         if (lk->lk_exslpfail >= realexslp) {
1201                                                 lk->lk_exslpfail = 0;
1202                                                 queue = SQ_SHARED_QUEUE;
1203                                                 v &= ~LK_SHARED_WAITERS;
1204                                                 if (realexslp != 0) {
1205                                                         LOCK_LOG2(lk,
1206                                         "%s: %p has only LK_SLEEPFAIL sleepers",
1207                                                             __func__, lk);
1208                                                         LOCK_LOG2(lk,
1209                         "%s: %p waking up threads on the exclusive queue",
1210                                                             __func__, lk);
1211                                                         wakeup_swapper =
1212                                                             sleepq_broadcast(
1213                                                             &lk->lock_object,
1214                                                             SLEEPQ_LK, 0,
1215                                                             SQ_EXCLUSIVE_QUEUE);
1216                                                 }
1217                                         } else
1218                                                 lk->lk_exslpfail = 0;
1219                                 }
1220                                 if (!atomic_cmpset_ptr(&lk->lk_lock, x, v)) {
1221                                         sleepq_release(&lk->lock_object);
1222                                         continue;
1223                                 }
1224                                 LOCK_LOG3(lk,
1225                                 "%s: %p waking up all threads on the %s queue",
1226                                     __func__, lk, queue == SQ_SHARED_QUEUE ?
1227                                     "shared" : "exclusive");
1228                                 wakeup_swapper |= sleepq_broadcast(
1229                                     &lk->lock_object, SLEEPQ_LK, 0, queue);
1230
1231                                 /*
1232                                  * If shared waiters have been woken up we need
1233                                  * to wait for one of them to acquire the lock
1234                                  * before to set the exclusive waiters in
1235                                  * order to avoid a deadlock.
1236                                  */
1237                                 if (queue == SQ_SHARED_QUEUE) {
1238                                         for (v = lk->lk_lock;
1239                                             (v & LK_SHARE) && !LK_SHARERS(v);
1240                                             v = lk->lk_lock)
1241                                                 cpu_spinwait();
1242                                 }
1243                         }
1244
1245                         /*
1246                          * Try to set the LK_EXCLUSIVE_WAITERS flag.  If we
1247                          * fail, loop back and retry.
1248                          */
1249                         if ((x & LK_EXCLUSIVE_WAITERS) == 0) {
1250                                 if (!atomic_cmpset_ptr(&lk->lk_lock, x,
1251                                     x | LK_EXCLUSIVE_WAITERS)) {
1252                                         sleepq_release(&lk->lock_object);
1253                                         continue;
1254                                 }
1255                                 LOCK_LOG2(lk, "%s: %p set drain waiters flag",
1256                                     __func__, lk);
1257                         }
1258
1259                         /*
1260                          * As far as we have been unable to acquire the
1261                          * exclusive lock and the exclusive waiters flag
1262                          * is set, we will sleep.
1263                          */
1264                         if (flags & LK_INTERLOCK) {
1265                                 class->lc_unlock(ilk);
1266                                 flags &= ~LK_INTERLOCK;
1267                         }
1268                         GIANT_SAVE();
1269                         sleepq_add(&lk->lock_object, NULL, iwmesg, SLEEPQ_LK,
1270                             SQ_EXCLUSIVE_QUEUE);
1271                         sleepq_wait(&lk->lock_object, ipri & PRIMASK);
1272                         GIANT_RESTORE();
1273                         LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
1274                             __func__, lk);
1275                 }
1276
1277                 if (error == 0) {
1278                         lock_profile_obtain_lock_success(&lk->lock_object,
1279                             contested, waittime, file, line);
1280                         LOCK_LOG_LOCK("DRAIN", &lk->lock_object, 0,
1281                             lk->lk_recurse, file, line);
1282                         WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
1283                             LK_TRYWIT(flags), file, line);
1284                         TD_LOCKS_INC(curthread);
1285                         STACK_SAVE(lk);
1286                 }
1287                 break;
1288         default:
1289                 if (flags & LK_INTERLOCK)
1290                         class->lc_unlock(ilk);
1291                 panic("%s: unknown lockmgr request 0x%x\n", __func__, op);
1292         }
1293
1294         if (flags & LK_INTERLOCK)
1295                 class->lc_unlock(ilk);
1296         if (wakeup_swapper)
1297                 kick_proc0();
1298
1299         return (error);
1300 }
1301
1302 void
1303 _lockmgr_disown(struct lock *lk, const char *file, int line)
1304 {
1305         uintptr_t tid, x;
1306
1307         if (SCHEDULER_STOPPED())
1308                 return;
1309
1310         tid = (uintptr_t)curthread;
1311         _lockmgr_assert(lk, KA_XLOCKED, file, line);
1312
1313         /*
1314          * Panic if the lock is recursed.
1315          */
1316         if (lockmgr_xlocked(lk) && lockmgr_recursed(lk))
1317                 panic("%s: disown a recursed lockmgr @ %s:%d\n",
1318                     __func__,  file, line);
1319
1320         /*
1321          * If the owner is already LK_KERNPROC just skip the whole operation.
1322          */
1323         if (LK_HOLDER(lk->lk_lock) != tid)
1324                 return;
1325         lock_profile_release_lock(&lk->lock_object);
1326         LOCK_LOG_LOCK("XDISOWN", &lk->lock_object, 0, 0, file, line);
1327         WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
1328         TD_LOCKS_DEC(curthread);
1329         STACK_SAVE(lk);
1330
1331         /*
1332          * In order to preserve waiters flags, just spin.
1333          */
1334         for (;;) {
1335                 x = lk->lk_lock;
1336                 MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
1337                 x &= LK_ALL_WAITERS;
1338                 if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x,
1339                     LK_KERNPROC | x))
1340                         return;
1341                 cpu_spinwait();
1342         }
1343 }
1344
1345 void
1346 lockmgr_printinfo(const struct lock *lk)
1347 {
1348         struct thread *td;
1349         uintptr_t x;
1350
1351         if (lk->lk_lock == LK_UNLOCKED)
1352                 printf("lock type %s: UNLOCKED\n", lk->lock_object.lo_name);
1353         else if (lk->lk_lock & LK_SHARE)
1354                 printf("lock type %s: SHARED (count %ju)\n",
1355                     lk->lock_object.lo_name,
1356                     (uintmax_t)LK_SHARERS(lk->lk_lock));
1357         else {
1358                 td = lockmgr_xholder(lk);
1359                 if (td == (struct thread *)LK_KERNPROC)
1360                         printf("lock type %s: EXCL by KERNPROC\n",
1361                             lk->lock_object.lo_name);
1362                 else
1363                         printf("lock type %s: EXCL by thread %p "
1364                             "(pid %d, %s, tid %d)\n", lk->lock_object.lo_name,
1365                             td, td->td_proc->p_pid, td->td_proc->p_comm,
1366                             td->td_tid);
1367         }
1368
1369         x = lk->lk_lock;
1370         if (x & LK_EXCLUSIVE_WAITERS)
1371                 printf(" with exclusive waiters pending\n");
1372         if (x & LK_SHARED_WAITERS)
1373                 printf(" with shared waiters pending\n");
1374         if (x & LK_EXCLUSIVE_SPINNERS)
1375                 printf(" with exclusive spinners pending\n");
1376
1377         STACK_PRINT(lk);
1378 }
1379
1380 int
1381 lockstatus(const struct lock *lk)
1382 {
1383         uintptr_t v, x;
1384         int ret;
1385
1386         ret = LK_SHARED;
1387         x = lk->lk_lock;
1388         v = LK_HOLDER(x);
1389
1390         if ((x & LK_SHARE) == 0) {
1391                 if (v == (uintptr_t)curthread || v == LK_KERNPROC)
1392                         ret = LK_EXCLUSIVE;
1393                 else
1394                         ret = LK_EXCLOTHER;
1395         } else if (x == LK_UNLOCKED)
1396                 ret = 0;
1397
1398         return (ret);
1399 }
1400
1401 #ifdef INVARIANT_SUPPORT
1402
1403 FEATURE(invariant_support,
1404     "Support for modules compiled with INVARIANTS option");
1405
1406 #ifndef INVARIANTS
1407 #undef  _lockmgr_assert
1408 #endif
1409
1410 void
1411 _lockmgr_assert(const struct lock *lk, int what, const char *file, int line)
1412 {
1413         int slocked = 0;
1414
1415         if (panicstr != NULL)
1416                 return;
1417         switch (what) {
1418         case KA_SLOCKED:
1419         case KA_SLOCKED | KA_NOTRECURSED:
1420         case KA_SLOCKED | KA_RECURSED:
1421                 slocked = 1;
1422         case KA_LOCKED:
1423         case KA_LOCKED | KA_NOTRECURSED:
1424         case KA_LOCKED | KA_RECURSED:
1425 #ifdef WITNESS
1426
1427                 /*
1428                  * We cannot trust WITNESS if the lock is held in exclusive
1429                  * mode and a call to lockmgr_disown() happened.
1430                  * Workaround this skipping the check if the lock is held in
1431                  * exclusive mode even for the KA_LOCKED case.
1432                  */
1433                 if (slocked || (lk->lk_lock & LK_SHARE)) {
1434                         witness_assert(&lk->lock_object, what, file, line);
1435                         break;
1436                 }
1437 #endif
1438                 if (lk->lk_lock == LK_UNLOCKED ||
1439                     ((lk->lk_lock & LK_SHARE) == 0 && (slocked ||
1440                     (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk)))))
1441                         panic("Lock %s not %slocked @ %s:%d\n",
1442                             lk->lock_object.lo_name, slocked ? "share" : "",
1443                             file, line);
1444
1445                 if ((lk->lk_lock & LK_SHARE) == 0) {
1446                         if (lockmgr_recursed(lk)) {
1447                                 if (what & KA_NOTRECURSED)
1448                                         panic("Lock %s recursed @ %s:%d\n",
1449                                             lk->lock_object.lo_name, file,
1450                                             line);
1451                         } else if (what & KA_RECURSED)
1452                                 panic("Lock %s not recursed @ %s:%d\n",
1453                                     lk->lock_object.lo_name, file, line);
1454                 }
1455                 break;
1456         case KA_XLOCKED:
1457         case KA_XLOCKED | KA_NOTRECURSED:
1458         case KA_XLOCKED | KA_RECURSED:
1459                 if (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk))
1460                         panic("Lock %s not exclusively locked @ %s:%d\n",
1461                             lk->lock_object.lo_name, file, line);
1462                 if (lockmgr_recursed(lk)) {
1463                         if (what & KA_NOTRECURSED)
1464                                 panic("Lock %s recursed @ %s:%d\n",
1465                                     lk->lock_object.lo_name, file, line);
1466                 } else if (what & KA_RECURSED)
1467                         panic("Lock %s not recursed @ %s:%d\n",
1468                             lk->lock_object.lo_name, file, line);
1469                 break;
1470         case KA_UNLOCKED:
1471                 if (lockmgr_xlocked(lk) || lockmgr_disowned(lk))
1472                         panic("Lock %s exclusively locked @ %s:%d\n",
1473                             lk->lock_object.lo_name, file, line);
1474                 break;
1475         default:
1476                 panic("Unknown lockmgr assertion: %d @ %s:%d\n", what, file,
1477                     line);
1478         }
1479 }
1480 #endif
1481
1482 #ifdef DDB
1483 int
1484 lockmgr_chain(struct thread *td, struct thread **ownerp)
1485 {
1486         struct lock *lk;
1487
1488         lk = td->td_wchan;
1489
1490         if (LOCK_CLASS(&lk->lock_object) != &lock_class_lockmgr)
1491                 return (0);
1492         db_printf("blocked on lockmgr %s", lk->lock_object.lo_name);
1493         if (lk->lk_lock & LK_SHARE)
1494                 db_printf("SHARED (count %ju)\n",
1495                     (uintmax_t)LK_SHARERS(lk->lk_lock));
1496         else
1497                 db_printf("EXCL\n");
1498         *ownerp = lockmgr_xholder(lk);
1499
1500         return (1);
1501 }
1502
1503 static void
1504 db_show_lockmgr(const struct lock_object *lock)
1505 {
1506         struct thread *td;
1507         const struct lock *lk;
1508
1509         lk = (const struct lock *)lock;
1510
1511         db_printf(" state: ");
1512         if (lk->lk_lock == LK_UNLOCKED)
1513                 db_printf("UNLOCKED\n");
1514         else if (lk->lk_lock & LK_SHARE)
1515                 db_printf("SLOCK: %ju\n", (uintmax_t)LK_SHARERS(lk->lk_lock));
1516         else {
1517                 td = lockmgr_xholder(lk);
1518                 if (td == (struct thread *)LK_KERNPROC)
1519                         db_printf("XLOCK: LK_KERNPROC\n");
1520                 else
1521                         db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
1522                             td->td_tid, td->td_proc->p_pid,
1523                             td->td_proc->p_comm);
1524                 if (lockmgr_recursed(lk))
1525                         db_printf(" recursed: %d\n", lk->lk_recurse);
1526         }
1527         db_printf(" waiters: ");
1528         switch (lk->lk_lock & LK_ALL_WAITERS) {
1529         case LK_SHARED_WAITERS:
1530                 db_printf("shared\n");
1531                 break;
1532         case LK_EXCLUSIVE_WAITERS:
1533                 db_printf("exclusive\n");
1534                 break;
1535         case LK_ALL_WAITERS:
1536                 db_printf("shared and exclusive\n");
1537                 break;
1538         default:
1539                 db_printf("none\n");
1540         }
1541         db_printf(" spinners: ");
1542         if (lk->lk_lock & LK_EXCLUSIVE_SPINNERS)
1543                 db_printf("exclusive\n");
1544         else
1545                 db_printf("none\n");
1546 }
1547 #endif