sys/kern/kern_sx.c

   1 /*-
   2  * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org>
   3  * Copyright (c) 2001 Jason Evans <jasone@freebsd.org>
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice(s), this list of conditions and the following disclaimer as
  11  *    the first lines of this file unmodified other than the possible
  12  *    addition of one or more copyright notices.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice(s), this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
  18  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  19  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  20  * DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
  21  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  22  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  23  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  24  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
  27  * DAMAGE.
  28  */
  29
  30 /*
  31  * Shared/exclusive locks.  This implementation attempts to ensure
  32  * deterministic lock granting behavior, so that slocks and xlocks are
  33  * interleaved.
  34  *
  35  * Priority propagation will not generally raise the priority of lock holders,
  36  * so should not be relied upon in combination with sx locks.
  37  */
  38
  39 #include "opt_ddb.h"
  40 #include "opt_hwpmc_hooks.h"
  41 #include "opt_no_adaptive_sx.h"
  42
  43 #include <sys/cdefs.h>
  44 __FBSDID("$FreeBSD$");
  45
  46 #include <sys/param.h>
  47 #include <sys/systm.h>
  48 #include <sys/kdb.h>
  49 #include <sys/ktr.h>
  50 #include <sys/lock.h>
  51 #include <sys/mutex.h>
  52 #include <sys/proc.h>
  53 #include <sys/sched.h>
  54 #include <sys/sleepqueue.h>
  55 #include <sys/sx.h>
  56 #include <sys/sysctl.h>
  57
  58 #if defined(SMP) && !defined(NO_ADAPTIVE_SX)
  59 #include <machine/cpu.h>
  60 #endif
  61
  62 #ifdef DDB
  63 #include <ddb/ddb.h>
  64 #endif
  65
  66 #if defined(SMP) && !defined(NO_ADAPTIVE_SX)
  67 #define ADAPTIVE_SX
  68 #endif
  69
  70 CTASSERT((SX_NOADAPTIVE & LO_CLASSFLAGS) == SX_NOADAPTIVE);
  71
  72 #ifdef HWPMC_HOOKS
  73 #include <sys/pmckern.h>
  74 PMC_SOFT_DECLARE( , , lock, failed);
  75 #endif
  76
  77 /* Handy macros for sleep queues. */
  78 #define SQ_EXCLUSIVE_QUEUE      0
  79 #define SQ_SHARED_QUEUE         1
  80
  81 /*
  82  * Variations on DROP_GIANT()/PICKUP_GIANT() for use in this file.  We
  83  * drop Giant anytime we have to sleep or if we adaptively spin.
  84  */
  85 #define GIANT_DECLARE                                                   \
  86         int _giantcnt = 0;                                              \
  87         WITNESS_SAVE_DECL(Giant)                                        \
  88
  89 #define GIANT_SAVE() do {                                               \
  90         if (mtx_owned(&Giant)) {                                        \
  91                 WITNESS_SAVE(&Giant.lock_object, Giant);                \
  92                 while (mtx_owned(&Giant)) {                             \
  93                         _giantcnt++;                                    \
  94                         mtx_unlock(&Giant);                             \
  95                 }                                                       \
  96         }                                                               \
  97 } while (0)
  98
  99 #define GIANT_RESTORE() do {                                            \
 100         if (_giantcnt > 0) {                                            \
 101                 mtx_assert(&Giant, MA_NOTOWNED);                        \
 102                 while (_giantcnt--)                                     \
 103                         mtx_lock(&Giant);                               \
 104                 WITNESS_RESTORE(&Giant.lock_object, Giant);             \
 105         }                                                               \
 106 } while (0)
 107
 108 /*
 109  * Returns true if an exclusive lock is recursed.  It assumes
 110  * curthread currently has an exclusive lock.
 111  */
 112 #define sx_recursed(sx)         ((sx)->sx_recurse != 0)
 113
 114 static void     assert_sx(const struct lock_object *lock, int what);
 115 #ifdef DDB
 116 static void     db_show_sx(const struct lock_object *lock);
 117 #endif
 118 static void     lock_sx(struct lock_object *lock, uintptr_t how);
 119 #ifdef KDTRACE_HOOKS
 120 static int      owner_sx(const struct lock_object *lock, struct thread **owner);
 121 #endif
 122 static uintptr_t unlock_sx(struct lock_object *lock);
 123
 124 struct lock_class lock_class_sx = {
 125         .lc_name = "sx",
 126         .lc_flags = LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE | LC_UPGRADABLE,
 127         .lc_assert = assert_sx,
 128 #ifdef DDB
 129         .lc_ddb_show = db_show_sx,
 130 #endif
 131         .lc_lock = lock_sx,
 132         .lc_unlock = unlock_sx,
 133 #ifdef KDTRACE_HOOKS
 134         .lc_owner = owner_sx,
 135 #endif
 136 };
 137
 138 #ifndef INVARIANTS
 139 #define _sx_assert(sx, what, file, line)
 140 #endif
 141
 142 #ifdef ADAPTIVE_SX
 143 static u_int asx_retries = 10;
 144 static u_int asx_loops = 10000;
 145 static SYSCTL_NODE(_debug, OID_AUTO, sx, CTLFLAG_RD, NULL, "sxlock debugging");
 146 SYSCTL_UINT(_debug_sx, OID_AUTO, retries, CTLFLAG_RW, &asx_retries, 0, "");
 147 SYSCTL_UINT(_debug_sx, OID_AUTO, loops, CTLFLAG_RW, &asx_loops, 0, "");
 148 #endif
 149
 150 void
 151 assert_sx(const struct lock_object *lock, int what)
 152 {
 153
 154         sx_assert((const struct sx *)lock, what);
 155 }
 156
 157 void
 158 lock_sx(struct lock_object *lock, uintptr_t how)
 159 {
 160         struct sx *sx;
 161
 162         sx = (struct sx *)lock;
 163         if (how)
 164                 sx_slock(sx);
 165         else
 166                 sx_xlock(sx);
 167 }
 168
 169 uintptr_t
 170 unlock_sx(struct lock_object *lock)
 171 {
 172         struct sx *sx;
 173
 174         sx = (struct sx *)lock;
 175         sx_assert(sx, SA_LOCKED | SA_NOTRECURSED);
 176         if (sx_xlocked(sx)) {
 177                 sx_xunlock(sx);
 178                 return (0);
 179         } else {
 180                 sx_sunlock(sx);
 181                 return (1);
 182         }
 183 }
 184
 185 #ifdef KDTRACE_HOOKS
 186 int
 187 owner_sx(const struct lock_object *lock, struct thread **owner)
 188 {
 189         const struct sx *sx = (const struct sx *)lock;
 190         uintptr_t x = sx->sx_lock;
 191
 192         *owner = (struct thread *)SX_OWNER(x);
 193         return ((x & SX_LOCK_SHARED) != 0 ? (SX_SHARERS(x) != 0) :
 194             (*owner != NULL));
 195 }
 196 #endif
 197
 198 void
 199 sx_sysinit(void *arg)
 200 {
 201         struct sx_args *sargs = arg;
 202
 203         sx_init_flags(sargs->sa_sx, sargs->sa_desc, sargs->sa_flags);
 204 }
 205
 206 void
 207 sx_init_flags(struct sx *sx, const char *description, int opts)
 208 {
 209         int flags;
 210
 211         MPASS((opts & ~(SX_QUIET | SX_RECURSE | SX_NOWITNESS | SX_DUPOK |
 212             SX_NOPROFILE | SX_NOADAPTIVE | SX_NEW)) == 0);
 213         ASSERT_ATOMIC_LOAD_PTR(sx->sx_lock,
 214             ("%s: sx_lock not aligned for %s: %p", __func__, description,
 215             &sx->sx_lock));
 216
 217         flags = LO_SLEEPABLE | LO_UPGRADABLE;
 218         if (opts & SX_DUPOK)
 219                 flags |= LO_DUPOK;
 220         if (opts & SX_NOPROFILE)
 221                 flags |= LO_NOPROFILE;
 222         if (!(opts & SX_NOWITNESS))
 223                 flags |= LO_WITNESS;
 224         if (opts & SX_RECURSE)
 225                 flags |= LO_RECURSABLE;
 226         if (opts & SX_QUIET)
 227                 flags |= LO_QUIET;
 228         if (opts & SX_NEW)
 229                 flags |= LO_NEW;
 230
 231         flags |= opts & SX_NOADAPTIVE;
 232         lock_init(&sx->lock_object, &lock_class_sx, description, NULL, flags);
 233         sx->sx_lock = SX_LOCK_UNLOCKED;
 234         sx->sx_recurse = 0;
 235 }
 236
 237 void
 238 sx_destroy(struct sx *sx)
 239 {
 240
 241         KASSERT(sx->sx_lock == SX_LOCK_UNLOCKED, ("sx lock still held"));
 242         KASSERT(sx->sx_recurse == 0, ("sx lock still recursed"));
 243         sx->sx_lock = SX_LOCK_DESTROYED;
 244         lock_destroy(&sx->lock_object);
 245 }
 246
 247 int
 248 _sx_slock(struct sx *sx, int opts, const char *file, int line)
 249 {
 250         int error = 0;
 251
 252         if (SCHEDULER_STOPPED())
 253                 return (0);
 254         KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
 255             ("sx_slock() by idle thread %p on sx %s @ %s:%d",
 256             curthread, sx->lock_object.lo_name, file, line));
 257         KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
 258             ("sx_slock() of destroyed sx @ %s:%d", file, line));
 259         WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER, file, line, NULL);
 260         error = __sx_slock(sx, opts, file, line);
 261         if (!error) {
 262                 LOCK_LOG_LOCK("SLOCK", &sx->lock_object, 0, 0, file, line);
 263                 WITNESS_LOCK(&sx->lock_object, 0, file, line);
 264                 TD_LOCKS_INC(curthread);
 265         }
 266
 267         return (error);
 268 }
 269
 270 int
 271 sx_try_slock_(struct sx *sx, const char *file, int line)
 272 {
 273         uintptr_t x;
 274
 275         if (SCHEDULER_STOPPED())
 276                 return (1);
 277
 278         KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
 279             ("sx_try_slock() by idle thread %p on sx %s @ %s:%d",
 280             curthread, sx->lock_object.lo_name, file, line));
 281
 282         for (;;) {
 283                 x = sx->sx_lock;
 284                 KASSERT(x != SX_LOCK_DESTROYED,
 285                     ("sx_try_slock() of destroyed sx @ %s:%d", file, line));
 286                 if (!(x & SX_LOCK_SHARED))
 287                         break;
 288                 if (atomic_cmpset_acq_ptr(&sx->sx_lock, x, x + SX_ONE_SHARER)) {
 289                         LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 1, file, line);
 290                         WITNESS_LOCK(&sx->lock_object, LOP_TRYLOCK, file, line);
 291                         LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire,
 292                             sx, 0, 0, file, line, LOCKSTAT_READER);
 293                         TD_LOCKS_INC(curthread);
 294                         return (1);
 295                 }
 296         }
 297
 298         LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 0, file, line);
 299         return (0);
 300 }
 301
 302 int
 303 _sx_xlock(struct sx *sx, int opts, const char *file, int line)
 304 {
 305         int error = 0;
 306
 307         if (SCHEDULER_STOPPED())
 308                 return (0);
 309         KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
 310             ("sx_xlock() by idle thread %p on sx %s @ %s:%d",
 311             curthread, sx->lock_object.lo_name, file, line));
 312         KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
 313             ("sx_xlock() of destroyed sx @ %s:%d", file, line));
 314         WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
 315             line, NULL);
 316         error = __sx_xlock(sx, curthread, opts, file, line);
 317         if (!error) {
 318                 LOCK_LOG_LOCK("XLOCK", &sx->lock_object, 0, sx->sx_recurse,
 319                     file, line);
 320                 WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line);
 321                 TD_LOCKS_INC(curthread);
 322         }
 323
 324         return (error);
 325 }
 326
 327 int
 328 sx_try_xlock_(struct sx *sx, const char *file, int line)
 329 {
 330         int rval;
 331
 332         if (SCHEDULER_STOPPED())
 333                 return (1);
 334
 335         KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
 336             ("sx_try_xlock() by idle thread %p on sx %s @ %s:%d",
 337             curthread, sx->lock_object.lo_name, file, line));
 338         KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
 339             ("sx_try_xlock() of destroyed sx @ %s:%d", file, line));
 340
 341         if (sx_xlocked(sx) &&
 342             (sx->lock_object.lo_flags & LO_RECURSABLE) != 0) {
 343                 sx->sx_recurse++;
 344                 atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
 345                 rval = 1;
 346         } else
 347                 rval = atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED,
 348                     (uintptr_t)curthread);
 349         LOCK_LOG_TRY("XLOCK", &sx->lock_object, 0, rval, file, line);
 350         if (rval) {
 351                 WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
 352                     file, line);
 353                 if (!sx_recursed(sx))
 354                         LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire,
 355                             sx, 0, 0, file, line, LOCKSTAT_WRITER);
 356                 TD_LOCKS_INC(curthread);
 357         }
 358
 359         return (rval);
 360 }
 361
 362 void
 363 _sx_sunlock(struct sx *sx, const char *file, int line)
 364 {
 365
 366         if (SCHEDULER_STOPPED())
 367                 return;
 368         KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
 369             ("sx_sunlock() of destroyed sx @ %s:%d", file, line));
 370         _sx_assert(sx, SA_SLOCKED, file, line);
 371         WITNESS_UNLOCK(&sx->lock_object, 0, file, line);
 372         LOCK_LOG_LOCK("SUNLOCK", &sx->lock_object, 0, 0, file, line);
 373         __sx_sunlock(sx, file, line);
 374         TD_LOCKS_DEC(curthread);
 375 }
 376
 377 void
 378 _sx_xunlock(struct sx *sx, const char *file, int line)
 379 {
 380
 381         if (SCHEDULER_STOPPED())
 382                 return;
 383         KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
 384             ("sx_xunlock() of destroyed sx @ %s:%d", file, line));
 385         _sx_assert(sx, SA_XLOCKED, file, line);
 386         WITNESS_UNLOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line);
 387         LOCK_LOG_LOCK("XUNLOCK", &sx->lock_object, 0, sx->sx_recurse, file,
 388             line);
 389         __sx_xunlock(sx, curthread, file, line);
 390         TD_LOCKS_DEC(curthread);
 391 }
 392
 393 /*
 394  * Try to do a non-blocking upgrade from a shared lock to an exclusive lock.
 395  * This will only succeed if this thread holds a single shared lock.
 396  * Return 1 if if the upgrade succeed, 0 otherwise.
 397  */
 398 int
 399 sx_try_upgrade_(struct sx *sx, const char *file, int line)
 400 {
 401         uintptr_t x;
 402         int success;
 403
 404         if (SCHEDULER_STOPPED())
 405                 return (1);
 406
 407         KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
 408             ("sx_try_upgrade() of destroyed sx @ %s:%d", file, line));
 409         _sx_assert(sx, SA_SLOCKED, file, line);
 410
 411         /*
 412          * Try to switch from one shared lock to an exclusive lock.  We need
 413          * to maintain the SX_LOCK_EXCLUSIVE_WAITERS flag if set so that
 414          * we will wake up the exclusive waiters when we drop the lock.
 415          */
 416         x = sx->sx_lock & SX_LOCK_EXCLUSIVE_WAITERS;
 417         success = atomic_cmpset_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) | x,
 418             (uintptr_t)curthread | x);
 419         LOCK_LOG_TRY("XUPGRADE", &sx->lock_object, 0, success, file, line);
 420         if (success) {
 421                 WITNESS_UPGRADE(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
 422                     file, line);
 423                 LOCKSTAT_RECORD0(sx__upgrade, sx);
 424         }
 425         return (success);
 426 }
 427
 428 /*
 429  * Downgrade an unrecursed exclusive lock into a single shared lock.
 430  */
 431 void
 432 sx_downgrade_(struct sx *sx, const char *file, int line)
 433 {
 434         uintptr_t x;
 435         int wakeup_swapper;
 436
 437         if (SCHEDULER_STOPPED())
 438                 return;
 439
 440         KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
 441             ("sx_downgrade() of destroyed sx @ %s:%d", file, line));
 442         _sx_assert(sx, SA_XLOCKED | SA_NOTRECURSED, file, line);
 443 #ifndef INVARIANTS
 444         if (sx_recursed(sx))
 445                 panic("downgrade of a recursed lock");
 446 #endif
 447
 448         WITNESS_DOWNGRADE(&sx->lock_object, 0, file, line);
 449
 450         /*
 451          * Try to switch from an exclusive lock with no shared waiters
 452          * to one sharer with no shared waiters.  If there are
 453          * exclusive waiters, we don't need to lock the sleep queue so
 454          * long as we preserve the flag.  We do one quick try and if
 455          * that fails we grab the sleepq lock to keep the flags from
 456          * changing and do it the slow way.
 457          *
 458          * We have to lock the sleep queue if there are shared waiters
 459          * so we can wake them up.
 460          */
 461         x = sx->sx_lock;
 462         if (!(x & SX_LOCK_SHARED_WAITERS) &&
 463             atomic_cmpset_rel_ptr(&sx->sx_lock, x, SX_SHARERS_LOCK(1) |
 464             (x & SX_LOCK_EXCLUSIVE_WAITERS))) {
 465                 LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line);
 466                 return;
 467         }
 468
 469         /*
 470          * Lock the sleep queue so we can read the waiters bits
 471          * without any races and wakeup any shared waiters.
 472          */
 473         sleepq_lock(&sx->lock_object);
 474
 475         /*
 476          * Preserve SX_LOCK_EXCLUSIVE_WAITERS while downgraded to a single
 477          * shared lock.  If there are any shared waiters, wake them up.
 478          */
 479         wakeup_swapper = 0;
 480         x = sx->sx_lock;
 481         atomic_store_rel_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) |
 482             (x & SX_LOCK_EXCLUSIVE_WAITERS));
 483         if (x & SX_LOCK_SHARED_WAITERS)
 484                 wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX,
 485                     0, SQ_SHARED_QUEUE);
 486         sleepq_release(&sx->lock_object);
 487
 488         LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line);
 489         LOCKSTAT_RECORD0(sx__downgrade, sx);
 490
 491         if (wakeup_swapper)
 492                 kick_proc0();
 493 }
 494
 495 /*
 496  * This function represents the so-called 'hard case' for sx_xlock
 497  * operation.  All 'easy case' failures are redirected to this.  Note
 498  * that ideally this would be a static function, but it needs to be
 499  * accessible from at least sx.h.
 500  */
 501 int
 502 _sx_xlock_hard(struct sx *sx, uintptr_t tid, int opts, const char *file,
 503     int line)
 504 {
 505         GIANT_DECLARE;
 506 #ifdef ADAPTIVE_SX
 507         volatile struct thread *owner;
 508         u_int i, spintries = 0;
 509 #endif
 510         uintptr_t x;
 511 #ifdef LOCK_PROFILING
 512         uint64_t waittime = 0;
 513         int contested = 0;
 514 #endif
 515         int error = 0;
 516 #ifdef  KDTRACE_HOOKS
 517         uintptr_t state;
 518         uint64_t spin_cnt = 0;
 519         uint64_t sleep_cnt = 0;
 520         int64_t sleep_time = 0;
 521         int64_t all_time = 0;
 522 #endif
 523
 524         if (SCHEDULER_STOPPED())
 525                 return (0);
 526
 527         /* If we already hold an exclusive lock, then recurse. */
 528         if (sx_xlocked(sx)) {
 529                 KASSERT((sx->lock_object.lo_flags & LO_RECURSABLE) != 0,
 530             ("_sx_xlock_hard: recursed on non-recursive sx %s @ %s:%d\n",
 531                     sx->lock_object.lo_name, file, line));
 532                 sx->sx_recurse++;
 533                 atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
 534                 if (LOCK_LOG_TEST(&sx->lock_object, 0))
 535                         CTR2(KTR_LOCK, "%s: %p recursing", __func__, sx);
 536                 return (0);
 537         }
 538
 539         if (LOCK_LOG_TEST(&sx->lock_object, 0))
 540                 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__,
 541                     sx->lock_object.lo_name, (void *)sx->sx_lock, file, line);
 542
 543 #ifdef KDTRACE_HOOKS
 544         all_time -= lockstat_nsecs(&sx->lock_object);
 545         state = sx->sx_lock;
 546 #endif
 547         while (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) {
 548 #ifdef KDTRACE_HOOKS
 549                 spin_cnt++;
 550 #endif
 551 #ifdef HWPMC_HOOKS
 552                 PMC_SOFT_CALL( , , lock, failed);
 553 #endif
 554                 lock_profile_obtain_lock_failed(&sx->lock_object, &contested,
 555                     &waittime);
 556 #ifdef ADAPTIVE_SX
 557                 /*
 558                  * If the lock is write locked and the owner is
 559                  * running on another CPU, spin until the owner stops
 560                  * running or the state of the lock changes.
 561                  */
 562                 x = sx->sx_lock;
 563                 if ((sx->lock_object.lo_flags & SX_NOADAPTIVE) == 0) {
 564                         if ((x & SX_LOCK_SHARED) == 0) {
 565                                 x = SX_OWNER(x);
 566                                 owner = (struct thread *)x;
 567                                 if (TD_IS_RUNNING(owner)) {
 568                                         if (LOCK_LOG_TEST(&sx->lock_object, 0))
 569                                                 CTR3(KTR_LOCK,
 570                                             "%s: spinning on %p held by %p",
 571                                                     __func__, sx, owner);
 572                                         KTR_STATE1(KTR_SCHED, "thread",
 573                                             sched_tdname(curthread), "spinning",
 574                                             "lockname:\"%s\"",
 575                                             sx->lock_object.lo_name);
 576                                         GIANT_SAVE();
 577                                         while (SX_OWNER(sx->sx_lock) == x &&
 578                                             TD_IS_RUNNING(owner)) {
 579                                                 cpu_spinwait();
 580 #ifdef KDTRACE_HOOKS
 581                                                 spin_cnt++;
 582 #endif
 583                                         }
 584                                         KTR_STATE0(KTR_SCHED, "thread",
 585                                             sched_tdname(curthread), "running");
 586                                         continue;
 587                                 }
 588                         } else if (SX_SHARERS(x) && spintries < asx_retries) {
 589                                 KTR_STATE1(KTR_SCHED, "thread",
 590                                     sched_tdname(curthread), "spinning",
 591                                     "lockname:\"%s\"", sx->lock_object.lo_name);
 592                                 GIANT_SAVE();
 593                                 spintries++;
 594                                 for (i = 0; i < asx_loops; i++) {
 595                                         if (LOCK_LOG_TEST(&sx->lock_object, 0))
 596                                                 CTR4(KTR_LOCK,
 597                                     "%s: shared spinning on %p with %u and %u",
 598                                                     __func__, sx, spintries, i);
 599                                         x = sx->sx_lock;
 600                                         if ((x & SX_LOCK_SHARED) == 0 ||
 601                                             SX_SHARERS(x) == 0)
 602                                                 break;
 603                                         cpu_spinwait();
 604 #ifdef KDTRACE_HOOKS
 605                                         spin_cnt++;
 606 #endif
 607                                 }
 608                                 KTR_STATE0(KTR_SCHED, "thread",
 609                                     sched_tdname(curthread), "running");
 610                                 if (i != asx_loops)
 611                                         continue;
 612                         }
 613                 }
 614 #endif
 615
 616                 sleepq_lock(&sx->lock_object);
 617                 x = sx->sx_lock;
 618
 619                 /*
 620                  * If the lock was released while spinning on the
 621                  * sleep queue chain lock, try again.
 622                  */
 623                 if (x == SX_LOCK_UNLOCKED) {
 624                         sleepq_release(&sx->lock_object);
 625                         continue;
 626                 }
 627
 628 #ifdef ADAPTIVE_SX
 629                 /*
 630                  * The current lock owner might have started executing
 631                  * on another CPU (or the lock could have changed
 632                  * owners) while we were waiting on the sleep queue
 633                  * chain lock.  If so, drop the sleep queue lock and try
 634                  * again.
 635                  */
 636                 if (!(x & SX_LOCK_SHARED) &&
 637                     (sx->lock_object.lo_flags & SX_NOADAPTIVE) == 0) {
 638                         owner = (struct thread *)SX_OWNER(x);
 639                         if (TD_IS_RUNNING(owner)) {
 640                                 sleepq_release(&sx->lock_object);
 641                                 continue;
 642                         }
 643                 }
 644 #endif
 645
 646                 /*
 647                  * If an exclusive lock was released with both shared
 648                  * and exclusive waiters and a shared waiter hasn't
 649                  * woken up and acquired the lock yet, sx_lock will be
 650                  * set to SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS.
 651                  * If we see that value, try to acquire it once.  Note
 652                  * that we have to preserve SX_LOCK_EXCLUSIVE_WAITERS
 653                  * as there are other exclusive waiters still.  If we
 654                  * fail, restart the loop.
 655                  */
 656                 if (x == (SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS)) {
 657                         if (atomic_cmpset_acq_ptr(&sx->sx_lock,
 658                             SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS,
 659                             tid | SX_LOCK_EXCLUSIVE_WAITERS)) {
 660                                 sleepq_release(&sx->lock_object);
 661                                 CTR2(KTR_LOCK, "%s: %p claimed by new writer",
 662                                     __func__, sx);
 663                                 break;
 664                         }
 665                         sleepq_release(&sx->lock_object);
 666                         continue;
 667                 }
 668
 669                 /*
 670                  * Try to set the SX_LOCK_EXCLUSIVE_WAITERS.  If we fail,
 671                  * than loop back and retry.
 672                  */
 673                 if (!(x & SX_LOCK_EXCLUSIVE_WAITERS)) {
 674                         if (!atomic_cmpset_ptr(&sx->sx_lock, x,
 675                             x | SX_LOCK_EXCLUSIVE_WAITERS)) {
 676                                 sleepq_release(&sx->lock_object);
 677                                 continue;
 678                         }
 679                         if (LOCK_LOG_TEST(&sx->lock_object, 0))
 680                                 CTR2(KTR_LOCK, "%s: %p set excl waiters flag",
 681                                     __func__, sx);
 682                 }
 683
 684                 /*
 685                  * Since we have been unable to acquire the exclusive
 686                  * lock and the exclusive waiters flag is set, we have
 687                  * to sleep.
 688                  */
 689                 if (LOCK_LOG_TEST(&sx->lock_object, 0))
 690                         CTR2(KTR_LOCK, "%s: %p blocking on sleep queue",
 691                             __func__, sx);
 692
 693 #ifdef KDTRACE_HOOKS
 694                 sleep_time -= lockstat_nsecs(&sx->lock_object);
 695 #endif
 696                 GIANT_SAVE();
 697                 sleepq_add(&sx->lock_object, NULL, sx->lock_object.lo_name,
 698                     SLEEPQ_SX | ((opts & SX_INTERRUPTIBLE) ?
 699                     SLEEPQ_INTERRUPTIBLE : 0), SQ_EXCLUSIVE_QUEUE);
 700                 if (!(opts & SX_INTERRUPTIBLE))
 701                         sleepq_wait(&sx->lock_object, 0);
 702                 else
 703                         error = sleepq_wait_sig(&sx->lock_object, 0);
 704 #ifdef KDTRACE_HOOKS
 705                 sleep_time += lockstat_nsecs(&sx->lock_object);
 706                 sleep_cnt++;
 707 #endif
 708                 if (error) {
 709                         if (LOCK_LOG_TEST(&sx->lock_object, 0))
 710                                 CTR2(KTR_LOCK,
 711                         "%s: interruptible sleep by %p suspended by signal",
 712                                     __func__, sx);
 713                         break;
 714                 }
 715                 if (LOCK_LOG_TEST(&sx->lock_object, 0))
 716                         CTR2(KTR_LOCK, "%s: %p resuming from sleep queue",
 717                             __func__, sx);
 718         }
 719 #ifdef KDTRACE_HOOKS
 720         all_time += lockstat_nsecs(&sx->lock_object);
 721         if (sleep_time)
 722                 LOCKSTAT_RECORD4(sx__block, sx, sleep_time,
 723                     LOCKSTAT_WRITER, (state & SX_LOCK_SHARED) == 0,
 724                     (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state));
 725         if (spin_cnt > sleep_cnt)
 726                 LOCKSTAT_RECORD4(sx__spin, sx, all_time - sleep_time,
 727                     LOCKSTAT_WRITER, (state & SX_LOCK_SHARED) == 0,
 728                     (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state));
 729 #endif
 730         if (!error)
 731                 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire, sx,
 732                     contested, waittime, file, line, LOCKSTAT_WRITER);
 733         GIANT_RESTORE();
 734         return (error);
 735 }
 736
 737 /*
 738  * This function represents the so-called 'hard case' for sx_xunlock
 739  * operation.  All 'easy case' failures are redirected to this.  Note
 740  * that ideally this would be a static function, but it needs to be
 741  * accessible from at least sx.h.
 742  */
 743 void
 744 _sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line)
 745 {
 746         uintptr_t x;
 747         int queue, wakeup_swapper;
 748
 749         if (SCHEDULER_STOPPED())
 750                 return;
 751
 752         MPASS(!(sx->sx_lock & SX_LOCK_SHARED));
 753
 754         /* If the lock is recursed, then unrecurse one level. */
 755         if (sx_xlocked(sx) && sx_recursed(sx)) {
 756                 if ((--sx->sx_recurse) == 0)
 757                         atomic_clear_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
 758                 if (LOCK_LOG_TEST(&sx->lock_object, 0))
 759                         CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, sx);
 760                 return;
 761         }
 762         MPASS(sx->sx_lock & (SX_LOCK_SHARED_WAITERS |
 763             SX_LOCK_EXCLUSIVE_WAITERS));
 764         if (LOCK_LOG_TEST(&sx->lock_object, 0))
 765                 CTR2(KTR_LOCK, "%s: %p contested", __func__, sx);
 766
 767         sleepq_lock(&sx->lock_object);
 768         x = SX_LOCK_UNLOCKED;
 769
 770         /*
 771          * The wake up algorithm here is quite simple and probably not
 772          * ideal.  It gives precedence to shared waiters if they are
 773          * present.  For this condition, we have to preserve the
 774          * state of the exclusive waiters flag.
 775          * If interruptible sleeps left the shared queue empty avoid a
 776          * starvation for the threads sleeping on the exclusive queue by giving
 777          * them precedence and cleaning up the shared waiters bit anyway.
 778          */
 779         if ((sx->sx_lock & SX_LOCK_SHARED_WAITERS) != 0 &&
 780             sleepq_sleepcnt(&sx->lock_object, SQ_SHARED_QUEUE) != 0) {
 781                 queue = SQ_SHARED_QUEUE;
 782                 x |= (sx->sx_lock & SX_LOCK_EXCLUSIVE_WAITERS);
 783         } else
 784                 queue = SQ_EXCLUSIVE_QUEUE;
 785
 786         /* Wake up all the waiters for the specific queue. */
 787         if (LOCK_LOG_TEST(&sx->lock_object, 0))
 788                 CTR3(KTR_LOCK, "%s: %p waking up all threads on %s queue",
 789                     __func__, sx, queue == SQ_SHARED_QUEUE ? "shared" :
 790                     "exclusive");
 791         atomic_store_rel_ptr(&sx->sx_lock, x);
 792         wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0,
 793             queue);
 794         sleepq_release(&sx->lock_object);
 795         if (wakeup_swapper)
 796                 kick_proc0();
 797 }
 798
 799 /*
 800  * This function represents the so-called 'hard case' for sx_slock
 801  * operation.  All 'easy case' failures are redirected to this.  Note
 802  * that ideally this would be a static function, but it needs to be
 803  * accessible from at least sx.h.
 804  */
 805 int
 806 _sx_slock_hard(struct sx *sx, int opts, const char *file, int line)
 807 {
 808         GIANT_DECLARE;
 809 #ifdef ADAPTIVE_SX
 810         volatile struct thread *owner;
 811 #endif
 812 #ifdef LOCK_PROFILING
 813         uint64_t waittime = 0;
 814         int contested = 0;
 815 #endif
 816         uintptr_t x;
 817         int error = 0;
 818 #ifdef KDTRACE_HOOKS
 819         uintptr_t state;
 820         uint64_t spin_cnt = 0;
 821         uint64_t sleep_cnt = 0;
 822         int64_t sleep_time = 0;
 823         int64_t all_time = 0;
 824 #endif
 825
 826         if (SCHEDULER_STOPPED())
 827                 return (0);
 828
 829 #ifdef KDTRACE_HOOKS
 830         state = sx->sx_lock;
 831         all_time -= lockstat_nsecs(&sx->lock_object);
 832 #endif
 833
 834         /*
 835          * As with rwlocks, we don't make any attempt to try to block
 836          * shared locks once there is an exclusive waiter.
 837          */
 838         for (;;) {
 839 #ifdef KDTRACE_HOOKS
 840                 spin_cnt++;
 841 #endif
 842                 x = sx->sx_lock;
 843
 844                 /*
 845                  * If no other thread has an exclusive lock then try to bump up
 846                  * the count of sharers.  Since we have to preserve the state
 847                  * of SX_LOCK_EXCLUSIVE_WAITERS, if we fail to acquire the
 848                  * shared lock loop back and retry.
 849                  */
 850                 if (x & SX_LOCK_SHARED) {
 851                         MPASS(!(x & SX_LOCK_SHARED_WAITERS));
 852                         if (atomic_cmpset_acq_ptr(&sx->sx_lock, x,
 853                             x + SX_ONE_SHARER)) {
 854                                 if (LOCK_LOG_TEST(&sx->lock_object, 0))
 855                                         CTR4(KTR_LOCK,
 856                                             "%s: %p succeed %p -> %p", __func__,
 857                                             sx, (void *)x,
 858                                             (void *)(x + SX_ONE_SHARER));
 859                                 break;
 860                         }
 861                         continue;
 862                 }
 863 #ifdef HWPMC_HOOKS
 864                 PMC_SOFT_CALL( , , lock, failed);
 865 #endif
 866                 lock_profile_obtain_lock_failed(&sx->lock_object, &contested,
 867                     &waittime);
 868
 869 #ifdef ADAPTIVE_SX
 870                 /*
 871                  * If the owner is running on another CPU, spin until
 872                  * the owner stops running or the state of the lock
 873                  * changes.
 874                  */
 875                 if ((sx->lock_object.lo_flags & SX_NOADAPTIVE) == 0) {
 876                         x = SX_OWNER(x);
 877                         owner = (struct thread *)x;
 878                         if (TD_IS_RUNNING(owner)) {
 879                                 if (LOCK_LOG_TEST(&sx->lock_object, 0))
 880                                         CTR3(KTR_LOCK,
 881                                             "%s: spinning on %p held by %p",
 882                                             __func__, sx, owner);
 883                                 KTR_STATE1(KTR_SCHED, "thread",
 884                                     sched_tdname(curthread), "spinning",
 885                                     "lockname:\"%s\"", sx->lock_object.lo_name);
 886                                 GIANT_SAVE();
 887                                 while (SX_OWNER(sx->sx_lock) == x &&
 888                                     TD_IS_RUNNING(owner)) {
 889 #ifdef KDTRACE_HOOKS
 890                                         spin_cnt++;
 891 #endif
 892                                         cpu_spinwait();
 893                                 }
 894                                 KTR_STATE0(KTR_SCHED, "thread",
 895                                     sched_tdname(curthread), "running");
 896                                 continue;
 897                         }
 898                 }
 899 #endif
 900
 901                 /*
 902                  * Some other thread already has an exclusive lock, so
 903                  * start the process of blocking.
 904                  */
 905                 sleepq_lock(&sx->lock_object);
 906                 x = sx->sx_lock;
 907
 908                 /*
 909                  * The lock could have been released while we spun.
 910                  * In this case loop back and retry.
 911                  */
 912                 if (x & SX_LOCK_SHARED) {
 913                         sleepq_release(&sx->lock_object);
 914                         continue;
 915                 }
 916
 917 #ifdef ADAPTIVE_SX
 918                 /*
 919                  * If the owner is running on another CPU, spin until
 920                  * the owner stops running or the state of the lock
 921                  * changes.
 922                  */
 923                 if (!(x & SX_LOCK_SHARED) &&
 924                     (sx->lock_object.lo_flags & SX_NOADAPTIVE) == 0) {
 925                         owner = (struct thread *)SX_OWNER(x);
 926                         if (TD_IS_RUNNING(owner)) {
 927                                 sleepq_release(&sx->lock_object);
 928                                 continue;
 929                         }
 930                 }
 931 #endif
 932
 933                 /*
 934                  * Try to set the SX_LOCK_SHARED_WAITERS flag.  If we
 935                  * fail to set it drop the sleep queue lock and loop
 936                  * back.
 937                  */
 938                 if (!(x & SX_LOCK_SHARED_WAITERS)) {
 939                         if (!atomic_cmpset_ptr(&sx->sx_lock, x,
 940                             x | SX_LOCK_SHARED_WAITERS)) {
 941                                 sleepq_release(&sx->lock_object);
 942                                 continue;
 943                         }
 944                         if (LOCK_LOG_TEST(&sx->lock_object, 0))
 945                                 CTR2(KTR_LOCK, "%s: %p set shared waiters flag",
 946                                     __func__, sx);
 947                 }
 948
 949                 /*
 950                  * Since we have been unable to acquire the shared lock,
 951                  * we have to sleep.
 952                  */
 953                 if (LOCK_LOG_TEST(&sx->lock_object, 0))
 954                         CTR2(KTR_LOCK, "%s: %p blocking on sleep queue",
 955                             __func__, sx);
 956
 957 #ifdef KDTRACE_HOOKS
 958                 sleep_time -= lockstat_nsecs(&sx->lock_object);
 959 #endif
 960                 GIANT_SAVE();
 961                 sleepq_add(&sx->lock_object, NULL, sx->lock_object.lo_name,
 962                     SLEEPQ_SX | ((opts & SX_INTERRUPTIBLE) ?
 963                     SLEEPQ_INTERRUPTIBLE : 0), SQ_SHARED_QUEUE);
 964                 if (!(opts & SX_INTERRUPTIBLE))
 965                         sleepq_wait(&sx->lock_object, 0);
 966                 else
 967                         error = sleepq_wait_sig(&sx->lock_object, 0);
 968 #ifdef KDTRACE_HOOKS
 969                 sleep_time += lockstat_nsecs(&sx->lock_object);
 970                 sleep_cnt++;
 971 #endif
 972                 if (error) {
 973                         if (LOCK_LOG_TEST(&sx->lock_object, 0))
 974                                 CTR2(KTR_LOCK,
 975                         "%s: interruptible sleep by %p suspended by signal",
 976                                     __func__, sx);
 977                         break;
 978                 }
 979                 if (LOCK_LOG_TEST(&sx->lock_object, 0))
 980                         CTR2(KTR_LOCK, "%s: %p resuming from sleep queue",
 981                             __func__, sx);
 982         }
 983 #ifdef KDTRACE_HOOKS
 984         all_time += lockstat_nsecs(&sx->lock_object);
 985         if (sleep_time)
 986                 LOCKSTAT_RECORD4(sx__block, sx, sleep_time,
 987                     LOCKSTAT_READER, (state & SX_LOCK_SHARED) == 0,
 988                     (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state));
 989         if (spin_cnt > sleep_cnt)
 990                 LOCKSTAT_RECORD4(sx__spin, sx, all_time - sleep_time,
 991                     LOCKSTAT_READER, (state & SX_LOCK_SHARED) == 0,
 992                     (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state));
 993 #endif
 994         if (error == 0)
 995                 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire, sx,
 996                     contested, waittime, file, line, LOCKSTAT_READER);
 997         GIANT_RESTORE();
 998         return (error);
 999 }
1000
1001 /*
1002  * This function represents the so-called 'hard case' for sx_sunlock
1003  * operation.  All 'easy case' failures are redirected to this.  Note
1004  * that ideally this would be a static function, but it needs to be
1005  * accessible from at least sx.h.
1006  */
1007 void
1008 _sx_sunlock_hard(struct sx *sx, const char *file, int line)
1009 {
1010         uintptr_t x;
1011         int wakeup_swapper;
1012
1013         if (SCHEDULER_STOPPED())
1014                 return;
1015
1016         for (;;) {
1017                 x = sx->sx_lock;
1018
1019                 /*
1020                  * We should never have sharers while at least one thread
1021                  * holds a shared lock.
1022                  */
1023                 KASSERT(!(x & SX_LOCK_SHARED_WAITERS),
1024                     ("%s: waiting sharers", __func__));
1025
1026                 /*
1027                  * See if there is more than one shared lock held.  If
1028                  * so, just drop one and return.
1029                  */
1030                 if (SX_SHARERS(x) > 1) {
1031                         if (atomic_cmpset_rel_ptr(&sx->sx_lock, x,
1032                             x - SX_ONE_SHARER)) {
1033                                 if (LOCK_LOG_TEST(&sx->lock_object, 0))
1034                                         CTR4(KTR_LOCK,
1035                                             "%s: %p succeeded %p -> %p",
1036                                             __func__, sx, (void *)x,
1037                                             (void *)(x - SX_ONE_SHARER));
1038                                 break;
1039                         }
1040                         continue;
1041                 }
1042
1043                 /*
1044                  * If there aren't any waiters for an exclusive lock,
1045                  * then try to drop it quickly.
1046                  */
1047                 if (!(x & SX_LOCK_EXCLUSIVE_WAITERS)) {
1048                         MPASS(x == SX_SHARERS_LOCK(1));
1049                         if (atomic_cmpset_rel_ptr(&sx->sx_lock,
1050                             SX_SHARERS_LOCK(1), SX_LOCK_UNLOCKED)) {
1051                                 if (LOCK_LOG_TEST(&sx->lock_object, 0))
1052                                         CTR2(KTR_LOCK, "%s: %p last succeeded",
1053                                             __func__, sx);
1054                                 break;
1055                         }
1056                         continue;
1057                 }
1058
1059                 /*
1060                  * At this point, there should just be one sharer with
1061                  * exclusive waiters.
1062                  */
1063                 MPASS(x == (SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS));
1064
1065                 sleepq_lock(&sx->lock_object);
1066
1067                 /*
1068                  * Wake up semantic here is quite simple:
1069                  * Just wake up all the exclusive waiters.
1070                  * Note that the state of the lock could have changed,
1071                  * so if it fails loop back and retry.
1072                  */
1073                 if (!atomic_cmpset_rel_ptr(&sx->sx_lock,
1074                     SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS,
1075                     SX_LOCK_UNLOCKED)) {
1076                         sleepq_release(&sx->lock_object);
1077                         continue;
1078                 }
1079                 if (LOCK_LOG_TEST(&sx->lock_object, 0))
1080                         CTR2(KTR_LOCK, "%s: %p waking up all thread on"
1081                             "exclusive queue", __func__, sx);
1082                 wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX,
1083                     0, SQ_EXCLUSIVE_QUEUE);
1084                 sleepq_release(&sx->lock_object);
1085                 if (wakeup_swapper)
1086                         kick_proc0();
1087                 break;
1088         }
1089 }
1090
1091 #ifdef INVARIANT_SUPPORT
1092 #ifndef INVARIANTS
1093 #undef  _sx_assert
1094 #endif
1095
1096 /*
1097  * In the non-WITNESS case, sx_assert() can only detect that at least
1098  * *some* thread owns an slock, but it cannot guarantee that *this*
1099  * thread owns an slock.
1100  */
1101 void
1102 _sx_assert(const struct sx *sx, int what, const char *file, int line)
1103 {
1104 #ifndef WITNESS
1105         int slocked = 0;
1106 #endif
1107
1108         if (panicstr != NULL)
1109                 return;
1110         switch (what) {
1111         case SA_SLOCKED:
1112         case SA_SLOCKED | SA_NOTRECURSED:
1113         case SA_SLOCKED | SA_RECURSED:
1114 #ifndef WITNESS
1115                 slocked = 1;
1116                 /* FALLTHROUGH */
1117 #endif
1118         case SA_LOCKED:
1119         case SA_LOCKED | SA_NOTRECURSED:
1120         case SA_LOCKED | SA_RECURSED:
1121 #ifdef WITNESS
1122                 witness_assert(&sx->lock_object, what, file, line);
1123 #else
1124                 /*
1125                  * If some other thread has an exclusive lock or we
1126                  * have one and are asserting a shared lock, fail.
1127                  * Also, if no one has a lock at all, fail.
1128                  */
1129                 if (sx->sx_lock == SX_LOCK_UNLOCKED ||
1130                     (!(sx->sx_lock & SX_LOCK_SHARED) && (slocked ||
1131                     sx_xholder(sx) != curthread)))
1132                         panic("Lock %s not %slocked @ %s:%d\n",
1133                             sx->lock_object.lo_name, slocked ? "share " : "",
1134                             file, line);
1135
1136                 if (!(sx->sx_lock & SX_LOCK_SHARED)) {
1137                         if (sx_recursed(sx)) {
1138                                 if (what & SA_NOTRECURSED)
1139                                         panic("Lock %s recursed @ %s:%d\n",
1140                                             sx->lock_object.lo_name, file,
1141                                             line);
1142                         } else if (what & SA_RECURSED)
1143                                 panic("Lock %s not recursed @ %s:%d\n",
1144                                     sx->lock_object.lo_name, file, line);
1145                 }
1146 #endif
1147                 break;
1148         case SA_XLOCKED:
1149         case SA_XLOCKED | SA_NOTRECURSED:
1150         case SA_XLOCKED | SA_RECURSED:
1151                 if (sx_xholder(sx) != curthread)
1152                         panic("Lock %s not exclusively locked @ %s:%d\n",
1153                             sx->lock_object.lo_name, file, line);
1154                 if (sx_recursed(sx)) {
1155                         if (what & SA_NOTRECURSED)
1156                                 panic("Lock %s recursed @ %s:%d\n",
1157                                     sx->lock_object.lo_name, file, line);
1158                 } else if (what & SA_RECURSED)
1159                         panic("Lock %s not recursed @ %s:%d\n",
1160                             sx->lock_object.lo_name, file, line);
1161                 break;
1162         case SA_UNLOCKED:
1163 #ifdef WITNESS
1164                 witness_assert(&sx->lock_object, what, file, line);
1165 #else
1166                 /*
1167                  * If we hold an exclusve lock fail.  We can't
1168                  * reliably check to see if we hold a shared lock or
1169                  * not.
1170                  */
1171                 if (sx_xholder(sx) == curthread)
1172                         panic("Lock %s exclusively locked @ %s:%d\n",
1173                             sx->lock_object.lo_name, file, line);
1174 #endif
1175                 break;
1176         default:
1177                 panic("Unknown sx lock assertion: %d @ %s:%d", what, file,
1178                     line);
1179         }
1180 }
1181 #endif  /* INVARIANT_SUPPORT */
1182
1183 #ifdef DDB
1184 static void
1185 db_show_sx(const struct lock_object *lock)
1186 {
1187         struct thread *td;
1188         const struct sx *sx;
1189
1190         sx = (const struct sx *)lock;
1191
1192         db_printf(" state: ");
1193         if (sx->sx_lock == SX_LOCK_UNLOCKED)
1194                 db_printf("UNLOCKED\n");
1195         else if (sx->sx_lock == SX_LOCK_DESTROYED) {
1196                 db_printf("DESTROYED\n");
1197                 return;
1198         } else if (sx->sx_lock & SX_LOCK_SHARED)
1199                 db_printf("SLOCK: %ju\n", (uintmax_t)SX_SHARERS(sx->sx_lock));
1200         else {
1201                 td = sx_xholder(sx);
1202                 db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
1203                     td->td_tid, td->td_proc->p_pid, td->td_name);
1204                 if (sx_recursed(sx))
1205                         db_printf(" recursed: %d\n", sx->sx_recurse);
1206         }
1207
1208         db_printf(" waiters: ");
1209         switch(sx->sx_lock &
1210             (SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS)) {
1211         case SX_LOCK_SHARED_WAITERS:
1212                 db_printf("shared\n");
1213                 break;
1214         case SX_LOCK_EXCLUSIVE_WAITERS:
1215                 db_printf("exclusive\n");
1216                 break;
1217         case SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS:
1218                 db_printf("exclusive and shared\n");
1219                 break;
1220         default:
1221                 db_printf("none\n");
1222         }
1223 }
1224
1225 /*
1226  * Check to see if a thread that is blocked on a sleep queue is actually
1227  * blocked on an sx lock.  If so, output some details and return true.
1228  * If the lock has an exclusive owner, return that in *ownerp.
1229  */
1230 int
1231 sx_chain(struct thread *td, struct thread **ownerp)
1232 {
1233         struct sx *sx;
1234
1235         /*
1236          * Check to see if this thread is blocked on an sx lock.
1237          * First, we check the lock class.  If that is ok, then we
1238          * compare the lock name against the wait message.
1239          */
1240         sx = td->td_wchan;
1241         if (LOCK_CLASS(&sx->lock_object) != &lock_class_sx ||
1242             sx->lock_object.lo_name != td->td_wmesg)
1243                 return (0);
1244
1245         /* We think we have an sx lock, so output some details. */
1246         db_printf("blocked on sx \"%s\" ", td->td_wmesg);
1247         *ownerp = sx_xholder(sx);
1248         if (sx->sx_lock & SX_LOCK_SHARED)
1249                 db_printf("SLOCK (count %ju)\n",
1250                     (uintmax_t)SX_SHARERS(sx->sx_lock));
1251         else
1252                 db_printf("XLOCK\n");
1253         return (1);
1254 }
1255 #endif