sys/kern/kern_rwlock.c

   1 /*-
   2  * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  * 3. Neither the name of the author nor the names of any co-contributors
  14  *    may be used to endorse or promote products derived from this software
  15  *    without specific prior written permission.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 /*
  31  * Machine independent bits of reader/writer lock implementation.
  32  */
  33
  34 #include <sys/cdefs.h>
  35 __FBSDID("$FreeBSD$");
  36
  37 #include "opt_ddb.h"
  38 #include "opt_hwpmc_hooks.h"
  39 #include "opt_kdtrace.h"
  40 #include "opt_no_adaptive_rwlocks.h"
  41
  42 #include <sys/param.h>
  43 #include <sys/kdb.h>
  44 #include <sys/ktr.h>
  45 #include <sys/kernel.h>
  46 #include <sys/lock.h>
  47 #include <sys/mutex.h>
  48 #include <sys/proc.h>
  49 #include <sys/rwlock.h>
  50 #include <sys/sysctl.h>
  51 #include <sys/systm.h>
  52 #include <sys/turnstile.h>
  53
  54 #include <machine/cpu.h>
  55
  56 #if defined(SMP) && !defined(NO_ADAPTIVE_RWLOCKS)
  57 #define ADAPTIVE_RWLOCKS
  58 #endif
  59
  60 #ifdef HWPMC_HOOKS
  61 #include <sys/pmckern.h>
  62 PMC_SOFT_DECLARE( , , lock, failed);
  63 #endif
  64
  65 #ifdef ADAPTIVE_RWLOCKS
  66 #define ROWNER_RETRIES  10
  67 #define ROWNER_LOOPS    10000
  68 #endif
  69
  70 #ifdef DDB
  71 #include <ddb/ddb.h>
  72
  73 static void     db_show_rwlock(struct lock_object *lock);
  74 #endif
  75 static void     assert_rw(struct lock_object *lock, int what);
  76 static void     lock_rw(struct lock_object *lock, int how);
  77 #ifdef KDTRACE_HOOKS
  78 static int      owner_rw(struct lock_object *lock, struct thread **owner);
  79 #endif
  80 static int      unlock_rw(struct lock_object *lock);
  81
  82 struct lock_class lock_class_rw = {
  83         .lc_name = "rw",
  84         .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE | LC_UPGRADABLE,
  85         .lc_assert = assert_rw,
  86 #ifdef DDB
  87         .lc_ddb_show = db_show_rwlock,
  88 #endif
  89         .lc_lock = lock_rw,
  90         .lc_unlock = unlock_rw,
  91 #ifdef KDTRACE_HOOKS
  92         .lc_owner = owner_rw,
  93 #endif
  94 };
  95
  96 /*
  97  * Return a pointer to the owning thread if the lock is write-locked or
  98  * NULL if the lock is unlocked or read-locked.
  99  */
 100 #define rw_wowner(rw)                                                   \
 101         ((rw)->rw_lock & RW_LOCK_READ ? NULL :                          \
 102             (struct thread *)RW_OWNER((rw)->rw_lock))
 103
 104 /*
 105  * Returns if a write owner is recursed.  Write ownership is not assured
 106  * here and should be previously checked.
 107  */
 108 #define rw_recursed(rw)         ((rw)->rw_recurse != 0)
 109
 110 /*
 111  * Return true if curthread helds the lock.
 112  */
 113 #define rw_wlocked(rw)          (rw_wowner((rw)) == curthread)
 114
 115 /*
 116  * Return a pointer to the owning thread for this lock who should receive
 117  * any priority lent by threads that block on this lock.  Currently this
 118  * is identical to rw_wowner().
 119  */
 120 #define rw_owner(rw)            rw_wowner(rw)
 121
 122 #ifndef INVARIANTS
 123 #define _rw_assert(rw, what, file, line)
 124 #endif
 125
 126 void
 127 assert_rw(struct lock_object *lock, int what)
 128 {
 129
 130         rw_assert((struct rwlock *)lock, what);
 131 }
 132
 133 void
 134 lock_rw(struct lock_object *lock, int how)
 135 {
 136         struct rwlock *rw;
 137
 138         rw = (struct rwlock *)lock;
 139         if (how)
 140                 rw_wlock(rw);
 141         else
 142                 rw_rlock(rw);
 143 }
 144
 145 int
 146 unlock_rw(struct lock_object *lock)
 147 {
 148         struct rwlock *rw;
 149
 150         rw = (struct rwlock *)lock;
 151         rw_assert(rw, RA_LOCKED | LA_NOTRECURSED);
 152         if (rw->rw_lock & RW_LOCK_READ) {
 153                 rw_runlock(rw);
 154                 return (0);
 155         } else {
 156                 rw_wunlock(rw);
 157                 return (1);
 158         }
 159 }
 160
 161 #ifdef KDTRACE_HOOKS
 162 int
 163 owner_rw(struct lock_object *lock, struct thread **owner)
 164 {
 165         struct rwlock *rw = (struct rwlock *)lock;
 166         uintptr_t x = rw->rw_lock;
 167
 168         *owner = rw_wowner(rw);
 169         return ((x & RW_LOCK_READ) != 0 ?  (RW_READERS(x) != 0) :
 170             (*owner != NULL));
 171 }
 172 #endif
 173
 174 void
 175 rw_init_flags(struct rwlock *rw, const char *name, int opts)
 176 {
 177         int flags;
 178
 179         MPASS((opts & ~(RW_DUPOK | RW_NOPROFILE | RW_NOWITNESS | RW_QUIET |
 180             RW_RECURSE)) == 0);
 181         ASSERT_ATOMIC_LOAD_PTR(rw->rw_lock,
 182             ("%s: rw_lock not aligned for %s: %p", __func__, name,
 183             &rw->rw_lock));
 184
 185         flags = LO_UPGRADABLE;
 186         if (opts & RW_DUPOK)
 187                 flags |= LO_DUPOK;
 188         if (opts & RW_NOPROFILE)
 189                 flags |= LO_NOPROFILE;
 190         if (!(opts & RW_NOWITNESS))
 191                 flags |= LO_WITNESS;
 192         if (opts & RW_RECURSE)
 193                 flags |= LO_RECURSABLE;
 194         if (opts & RW_QUIET)
 195                 flags |= LO_QUIET;
 196
 197         rw->rw_lock = RW_UNLOCKED;
 198         rw->rw_recurse = 0;
 199         lock_init(&rw->lock_object, &lock_class_rw, name, NULL, flags);
 200 }
 201
 202 void
 203 rw_destroy(struct rwlock *rw)
 204 {
 205
 206         KASSERT(rw->rw_lock == RW_UNLOCKED, ("rw lock %p not unlocked", rw));
 207         KASSERT(rw->rw_recurse == 0, ("rw lock %p still recursed", rw));
 208         rw->rw_lock = RW_DESTROYED;
 209         lock_destroy(&rw->lock_object);
 210 }
 211
 212 void
 213 rw_sysinit(void *arg)
 214 {
 215         struct rw_args *args = arg;
 216
 217         rw_init(args->ra_rw, args->ra_desc);
 218 }
 219
 220 void
 221 rw_sysinit_flags(void *arg)
 222 {
 223         struct rw_args_flags *args = arg;
 224
 225         rw_init_flags(args->ra_rw, args->ra_desc, args->ra_flags);
 226 }
 227
 228 int
 229 rw_wowned(struct rwlock *rw)
 230 {
 231
 232         return (rw_wowner(rw) == curthread);
 233 }
 234
 235 void
 236 _rw_wlock(struct rwlock *rw, const char *file, int line)
 237 {
 238
 239         if (SCHEDULER_STOPPED())
 240                 return;
 241         KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
 242             ("rw_wlock() by idle thread %p on rwlock %s @ %s:%d",
 243             curthread, rw->lock_object.lo_name, file, line));
 244         KASSERT(rw->rw_lock != RW_DESTROYED,
 245             ("rw_wlock() of destroyed rwlock @ %s:%d", file, line));
 246         WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
 247             line, NULL);
 248         __rw_wlock(rw, curthread, file, line);
 249         LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line);
 250         WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
 251         curthread->td_locks++;
 252 }
 253
 254 int
 255 _rw_try_wlock(struct rwlock *rw, const char *file, int line)
 256 {
 257         int rval;
 258
 259         if (SCHEDULER_STOPPED())
 260                 return (1);
 261
 262         KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
 263             ("rw_try_wlock() by idle thread %p on rwlock %s @ %s:%d",
 264             curthread, rw->lock_object.lo_name, file, line));
 265         KASSERT(rw->rw_lock != RW_DESTROYED,
 266             ("rw_try_wlock() of destroyed rwlock @ %s:%d", file, line));
 267
 268         if (rw_wlocked(rw) &&
 269             (rw->lock_object.lo_flags & LO_RECURSABLE) != 0) {
 270                 rw->rw_recurse++;
 271                 rval = 1;
 272         } else
 273                 rval = atomic_cmpset_acq_ptr(&rw->rw_lock, RW_UNLOCKED,
 274                     (uintptr_t)curthread);
 275
 276         LOCK_LOG_TRY("WLOCK", &rw->lock_object, 0, rval, file, line);
 277         if (rval) {
 278                 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
 279                     file, line);
 280                 curthread->td_locks++;
 281         }
 282         return (rval);
 283 }
 284
 285 void
 286 _rw_wunlock(struct rwlock *rw, const char *file, int line)
 287 {
 288
 289         if (SCHEDULER_STOPPED())
 290                 return;
 291         KASSERT(rw->rw_lock != RW_DESTROYED,
 292             ("rw_wunlock() of destroyed rwlock @ %s:%d", file, line));
 293         _rw_assert(rw, RA_WLOCKED, file, line);
 294         curthread->td_locks--;
 295         WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
 296         LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file,
 297             line);
 298         if (!rw_recursed(rw))
 299                 LOCKSTAT_PROFILE_RELEASE_LOCK(LS_RW_WUNLOCK_RELEASE, rw);
 300         __rw_wunlock(rw, curthread, file, line);
 301 }
 302 /*
 303  * Determines whether a new reader can acquire a lock.  Succeeds if the
 304  * reader already owns a read lock and the lock is locked for read to
 305  * prevent deadlock from reader recursion.  Also succeeds if the lock
 306  * is unlocked and has no writer waiters or spinners.  Failing otherwise
 307  * prioritizes writers before readers.
 308  */
 309 #define RW_CAN_READ(_rw)                                                \
 310     ((curthread->td_rw_rlocks && (_rw) & RW_LOCK_READ) || ((_rw) &      \
 311     (RW_LOCK_READ | RW_LOCK_WRITE_WAITERS | RW_LOCK_WRITE_SPINNER)) ==  \
 312     RW_LOCK_READ)
 313
 314 void
 315 _rw_rlock(struct rwlock *rw, const char *file, int line)
 316 {
 317         struct turnstile *ts;
 318 #ifdef ADAPTIVE_RWLOCKS
 319         volatile struct thread *owner;
 320         int spintries = 0;
 321         int i;
 322 #endif
 323 #ifdef LOCK_PROFILING
 324         uint64_t waittime = 0;
 325         int contested = 0;
 326 #endif
 327         uintptr_t v;
 328 #ifdef KDTRACE_HOOKS
 329         uint64_t spin_cnt = 0;
 330         uint64_t sleep_cnt = 0;
 331         int64_t sleep_time = 0;
 332 #endif
 333
 334         if (SCHEDULER_STOPPED())
 335                 return;
 336
 337         KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
 338             ("rw_rlock() by idle thread %p on rwlock %s @ %s:%d",
 339             curthread, rw->lock_object.lo_name, file, line));
 340         KASSERT(rw->rw_lock != RW_DESTROYED,
 341             ("rw_rlock() of destroyed rwlock @ %s:%d", file, line));
 342         KASSERT(rw_wowner(rw) != curthread,
 343             ("rw_rlock: wlock already held for %s @ %s:%d",
 344             rw->lock_object.lo_name, file, line));
 345         WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL);
 346
 347         for (;;) {
 348 #ifdef KDTRACE_HOOKS
 349                 spin_cnt++;
 350 #endif
 351                 /*
 352                  * Handle the easy case.  If no other thread has a write
 353                  * lock, then try to bump up the count of read locks.  Note
 354                  * that we have to preserve the current state of the
 355                  * RW_LOCK_WRITE_WAITERS flag.  If we fail to acquire a
 356                  * read lock, then rw_lock must have changed, so restart
 357                  * the loop.  Note that this handles the case of a
 358                  * completely unlocked rwlock since such a lock is encoded
 359                  * as a read lock with no waiters.
 360                  */
 361                 v = rw->rw_lock;
 362                 if (RW_CAN_READ(v)) {
 363                         /*
 364                          * The RW_LOCK_READ_WAITERS flag should only be set
 365                          * if the lock has been unlocked and write waiters
 366                          * were present.
 367                          */
 368                         if (atomic_cmpset_acq_ptr(&rw->rw_lock, v,
 369                             v + RW_ONE_READER)) {
 370                                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
 371                                         CTR4(KTR_LOCK,
 372                                             "%s: %p succeed %p -> %p", __func__,
 373                                             rw, (void *)v,
 374                                             (void *)(v + RW_ONE_READER));
 375                                 break;
 376                         }
 377                         continue;
 378                 }
 379 #ifdef HWPMC_HOOKS
 380                 PMC_SOFT_CALL( , , lock, failed);
 381 #endif
 382                 lock_profile_obtain_lock_failed(&rw->lock_object,
 383                     &contested, &waittime);
 384
 385 #ifdef ADAPTIVE_RWLOCKS
 386                 /*
 387                  * If the owner is running on another CPU, spin until
 388                  * the owner stops running or the state of the lock
 389                  * changes.
 390                  */
 391                 if ((v & RW_LOCK_READ) == 0) {
 392                         owner = (struct thread *)RW_OWNER(v);
 393                         if (TD_IS_RUNNING(owner)) {
 394                                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
 395                                         CTR3(KTR_LOCK,
 396                                             "%s: spinning on %p held by %p",
 397                                             __func__, rw, owner);
 398                                 while ((struct thread*)RW_OWNER(rw->rw_lock) ==
 399                                     owner && TD_IS_RUNNING(owner)) {
 400                                         cpu_spinwait();
 401 #ifdef KDTRACE_HOOKS
 402                                         spin_cnt++;
 403 #endif
 404                                 }
 405                                 continue;
 406                         }
 407                 } else if (spintries < ROWNER_RETRIES) {
 408                         spintries++;
 409                         for (i = 0; i < ROWNER_LOOPS; i++) {
 410                                 v = rw->rw_lock;
 411                                 if ((v & RW_LOCK_READ) == 0 || RW_CAN_READ(v))
 412                                         break;
 413                                 cpu_spinwait();
 414                         }
 415                         if (i != ROWNER_LOOPS)
 416                                 continue;
 417                 }
 418 #endif
 419
 420                 /*
 421                  * Okay, now it's the hard case.  Some other thread already
 422                  * has a write lock or there are write waiters present,
 423                  * acquire the turnstile lock so we can begin the process
 424                  * of blocking.
 425                  */
 426                 ts = turnstile_trywait(&rw->lock_object);
 427
 428                 /*
 429                  * The lock might have been released while we spun, so
 430                  * recheck its state and restart the loop if needed.
 431                  */
 432                 v = rw->rw_lock;
 433                 if (RW_CAN_READ(v)) {
 434                         turnstile_cancel(ts);
 435                         continue;
 436                 }
 437
 438 #ifdef ADAPTIVE_RWLOCKS
 439                 /*
 440                  * The current lock owner might have started executing
 441                  * on another CPU (or the lock could have changed
 442                  * owners) while we were waiting on the turnstile
 443                  * chain lock.  If so, drop the turnstile lock and try
 444                  * again.
 445                  */
 446                 if ((v & RW_LOCK_READ) == 0) {
 447                         owner = (struct thread *)RW_OWNER(v);
 448                         if (TD_IS_RUNNING(owner)) {
 449                                 turnstile_cancel(ts);
 450                                 continue;
 451                         }
 452                 }
 453 #endif
 454
 455                 /*
 456                  * The lock is held in write mode or it already has waiters.
 457                  */
 458                 MPASS(!RW_CAN_READ(v));
 459
 460                 /*
 461                  * If the RW_LOCK_READ_WAITERS flag is already set, then
 462                  * we can go ahead and block.  If it is not set then try
 463                  * to set it.  If we fail to set it drop the turnstile
 464                  * lock and restart the loop.
 465                  */
 466                 if (!(v & RW_LOCK_READ_WAITERS)) {
 467                         if (!atomic_cmpset_ptr(&rw->rw_lock, v,
 468                             v | RW_LOCK_READ_WAITERS)) {
 469                                 turnstile_cancel(ts);
 470                                 continue;
 471                         }
 472                         if (LOCK_LOG_TEST(&rw->lock_object, 0))
 473                                 CTR2(KTR_LOCK, "%s: %p set read waiters flag",
 474                                     __func__, rw);
 475                 }
 476
 477                 /*
 478                  * We were unable to acquire the lock and the read waiters
 479                  * flag is set, so we must block on the turnstile.
 480                  */
 481                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
 482                         CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__,
 483                             rw);
 484 #ifdef KDTRACE_HOOKS
 485                 sleep_time -= lockstat_nsecs();
 486 #endif
 487                 turnstile_wait(ts, rw_owner(rw), TS_SHARED_QUEUE);
 488 #ifdef KDTRACE_HOOKS
 489                 sleep_time += lockstat_nsecs();
 490                 sleep_cnt++;
 491 #endif
 492                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
 493                         CTR2(KTR_LOCK, "%s: %p resuming from turnstile",
 494                             __func__, rw);
 495         }
 496
 497         /*
 498          * TODO: acquire "owner of record" here.  Here be turnstile dragons
 499          * however.  turnstiles don't like owners changing between calls to
 500          * turnstile_wait() currently.
 501          */
 502         LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_RLOCK_ACQUIRE, rw, contested,
 503             waittime, file, line);
 504         LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line);
 505         WITNESS_LOCK(&rw->lock_object, 0, file, line);
 506         curthread->td_locks++;
 507         curthread->td_rw_rlocks++;
 508 #ifdef KDTRACE_HOOKS
 509         if (sleep_time)
 510                 LOCKSTAT_RECORD1(LS_RW_RLOCK_BLOCK, rw, sleep_time);
 511
 512         /*
 513          * Record only the loops spinning and not sleeping.
 514          */
 515         if (spin_cnt > sleep_cnt)
 516                 LOCKSTAT_RECORD1(LS_RW_RLOCK_SPIN, rw, (spin_cnt - sleep_cnt));
 517 #endif
 518 }
 519
 520 int
 521 _rw_try_rlock(struct rwlock *rw, const char *file, int line)
 522 {
 523         uintptr_t x;
 524
 525         if (SCHEDULER_STOPPED())
 526                 return (1);
 527
 528         KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
 529             ("rw_try_rlock() by idle thread %p on rwlock %s @ %s:%d",
 530             curthread, rw->lock_object.lo_name, file, line));
 531
 532         for (;;) {
 533                 x = rw->rw_lock;
 534                 KASSERT(rw->rw_lock != RW_DESTROYED,
 535                     ("rw_try_rlock() of destroyed rwlock @ %s:%d", file, line));
 536                 if (!(x & RW_LOCK_READ))
 537                         break;
 538                 if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, x + RW_ONE_READER)) {
 539                         LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 1, file,
 540                             line);
 541                         WITNESS_LOCK(&rw->lock_object, LOP_TRYLOCK, file, line);
 542                         curthread->td_locks++;
 543                         curthread->td_rw_rlocks++;
 544                         return (1);
 545                 }
 546         }
 547
 548         LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 0, file, line);
 549         return (0);
 550 }
 551
 552 void
 553 _rw_runlock(struct rwlock *rw, const char *file, int line)
 554 {
 555         struct turnstile *ts;
 556         uintptr_t x, v, queue;
 557
 558         if (SCHEDULER_STOPPED())
 559                 return;
 560
 561         KASSERT(rw->rw_lock != RW_DESTROYED,
 562             ("rw_runlock() of destroyed rwlock @ %s:%d", file, line));
 563         _rw_assert(rw, RA_RLOCKED, file, line);
 564         curthread->td_locks--;
 565         curthread->td_rw_rlocks--;
 566         WITNESS_UNLOCK(&rw->lock_object, 0, file, line);
 567         LOCK_LOG_LOCK("RUNLOCK", &rw->lock_object, 0, 0, file, line);
 568
 569         /* TODO: drop "owner of record" here. */
 570
 571         for (;;) {
 572                 /*
 573                  * See if there is more than one read lock held.  If so,
 574                  * just drop one and return.
 575                  */
 576                 x = rw->rw_lock;
 577                 if (RW_READERS(x) > 1) {
 578                         if (atomic_cmpset_rel_ptr(&rw->rw_lock, x,
 579                             x - RW_ONE_READER)) {
 580                                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
 581                                         CTR4(KTR_LOCK,
 582                                             "%s: %p succeeded %p -> %p",
 583                                             __func__, rw, (void *)x,
 584                                             (void *)(x - RW_ONE_READER));
 585                                 break;
 586                         }
 587                         continue;
 588                 }
 589                 /*
 590                  * If there aren't any waiters for a write lock, then try
 591                  * to drop it quickly.
 592                  */
 593                 if (!(x & RW_LOCK_WAITERS)) {
 594                         MPASS((x & ~RW_LOCK_WRITE_SPINNER) ==
 595                             RW_READERS_LOCK(1));
 596                         if (atomic_cmpset_rel_ptr(&rw->rw_lock, x,
 597                             RW_UNLOCKED)) {
 598                                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
 599                                         CTR2(KTR_LOCK, "%s: %p last succeeded",
 600                                             __func__, rw);
 601                                 break;
 602                         }
 603                         continue;
 604                 }
 605                 /*
 606                  * Ok, we know we have waiters and we think we are the
 607                  * last reader, so grab the turnstile lock.
 608                  */
 609                 turnstile_chain_lock(&rw->lock_object);
 610                 v = rw->rw_lock & (RW_LOCK_WAITERS | RW_LOCK_WRITE_SPINNER);
 611                 MPASS(v & RW_LOCK_WAITERS);
 612
 613                 /*
 614                  * Try to drop our lock leaving the lock in a unlocked
 615                  * state.
 616                  *
 617                  * If you wanted to do explicit lock handoff you'd have to
 618                  * do it here.  You'd also want to use turnstile_signal()
 619                  * and you'd have to handle the race where a higher
 620                  * priority thread blocks on the write lock before the
 621                  * thread you wakeup actually runs and have the new thread
 622                  * "steal" the lock.  For now it's a lot simpler to just
 623                  * wakeup all of the waiters.
 624                  *
 625                  * As above, if we fail, then another thread might have
 626                  * acquired a read lock, so drop the turnstile lock and
 627                  * restart.
 628                  */
 629                 x = RW_UNLOCKED;
 630                 if (v & RW_LOCK_WRITE_WAITERS) {
 631                         queue = TS_EXCLUSIVE_QUEUE;
 632                         x |= (v & RW_LOCK_READ_WAITERS);
 633                 } else
 634                         queue = TS_SHARED_QUEUE;
 635                 if (!atomic_cmpset_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v,
 636                     x)) {
 637                         turnstile_chain_unlock(&rw->lock_object);
 638                         continue;
 639                 }
 640                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
 641                         CTR2(KTR_LOCK, "%s: %p last succeeded with waiters",
 642                             __func__, rw);
 643
 644                 /*
 645                  * Ok.  The lock is released and all that's left is to
 646                  * wake up the waiters.  Note that the lock might not be
 647                  * free anymore, but in that case the writers will just
 648                  * block again if they run before the new lock holder(s)
 649                  * release the lock.
 650                  */
 651                 ts = turnstile_lookup(&rw->lock_object);
 652                 MPASS(ts != NULL);
 653                 turnstile_broadcast(ts, queue);
 654                 turnstile_unpend(ts, TS_SHARED_LOCK);
 655                 turnstile_chain_unlock(&rw->lock_object);
 656                 break;
 657         }
 658         LOCKSTAT_PROFILE_RELEASE_LOCK(LS_RW_RUNLOCK_RELEASE, rw);
 659 }
 660
 661 /*
 662  * This function is called when we are unable to obtain a write lock on the
 663  * first try.  This means that at least one other thread holds either a
 664  * read or write lock.
 665  */
 666 void
 667 _rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
 668 {
 669         struct turnstile *ts;
 670 #ifdef ADAPTIVE_RWLOCKS
 671         volatile struct thread *owner;
 672         int spintries = 0;
 673         int i;
 674 #endif
 675         uintptr_t v, x;
 676 #ifdef LOCK_PROFILING
 677         uint64_t waittime = 0;
 678         int contested = 0;
 679 #endif
 680 #ifdef KDTRACE_HOOKS
 681         uint64_t spin_cnt = 0;
 682         uint64_t sleep_cnt = 0;
 683         int64_t sleep_time = 0;
 684 #endif
 685
 686         if (SCHEDULER_STOPPED())
 687                 return;
 688
 689         if (rw_wlocked(rw)) {
 690                 KASSERT(rw->lock_object.lo_flags & LO_RECURSABLE,
 691                     ("%s: recursing but non-recursive rw %s @ %s:%d\n",
 692                     __func__, rw->lock_object.lo_name, file, line));
 693                 rw->rw_recurse++;
 694                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
 695                         CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw);
 696                 return;
 697         }
 698
 699         if (LOCK_LOG_TEST(&rw->lock_object, 0))
 700                 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__,
 701                     rw->lock_object.lo_name, (void *)rw->rw_lock, file, line);
 702
 703         while (!_rw_write_lock(rw, tid)) {
 704 #ifdef KDTRACE_HOOKS
 705                 spin_cnt++;
 706 #endif
 707 #ifdef HWPMC_HOOKS
 708                 PMC_SOFT_CALL( , , lock, failed);
 709 #endif
 710                 lock_profile_obtain_lock_failed(&rw->lock_object,
 711                     &contested, &waittime);
 712 #ifdef ADAPTIVE_RWLOCKS
 713                 /*
 714                  * If the lock is write locked and the owner is
 715                  * running on another CPU, spin until the owner stops
 716                  * running or the state of the lock changes.
 717                  */
 718                 v = rw->rw_lock;
 719                 owner = (struct thread *)RW_OWNER(v);
 720                 if (!(v & RW_LOCK_READ) && TD_IS_RUNNING(owner)) {
 721                         if (LOCK_LOG_TEST(&rw->lock_object, 0))
 722                                 CTR3(KTR_LOCK, "%s: spinning on %p held by %p",
 723                                     __func__, rw, owner);
 724                         while ((struct thread*)RW_OWNER(rw->rw_lock) == owner &&
 725                             TD_IS_RUNNING(owner)) {
 726                                 cpu_spinwait();
 727 #ifdef KDTRACE_HOOKS
 728                                 spin_cnt++;
 729 #endif
 730                         }
 731                         continue;
 732                 }
 733                 if ((v & RW_LOCK_READ) && RW_READERS(v) &&
 734                     spintries < ROWNER_RETRIES) {
 735                         if (!(v & RW_LOCK_WRITE_SPINNER)) {
 736                                 if (!atomic_cmpset_ptr(&rw->rw_lock, v,
 737                                     v | RW_LOCK_WRITE_SPINNER)) {
 738                                         continue;
 739                                 }
 740                         }
 741                         spintries++;
 742                         for (i = 0; i < ROWNER_LOOPS; i++) {
 743                                 if ((rw->rw_lock & RW_LOCK_WRITE_SPINNER) == 0)
 744                                         break;
 745                                 cpu_spinwait();
 746                         }
 747 #ifdef KDTRACE_HOOKS
 748                         spin_cnt += ROWNER_LOOPS - i;
 749 #endif
 750                         if (i != ROWNER_LOOPS)
 751                                 continue;
 752                 }
 753 #endif
 754                 ts = turnstile_trywait(&rw->lock_object);
 755                 v = rw->rw_lock;
 756
 757 #ifdef ADAPTIVE_RWLOCKS
 758                 /*
 759                  * The current lock owner might have started executing
 760                  * on another CPU (or the lock could have changed
 761                  * owners) while we were waiting on the turnstile
 762                  * chain lock.  If so, drop the turnstile lock and try
 763                  * again.
 764                  */
 765                 if (!(v & RW_LOCK_READ)) {
 766                         owner = (struct thread *)RW_OWNER(v);
 767                         if (TD_IS_RUNNING(owner)) {
 768                                 turnstile_cancel(ts);
 769                                 continue;
 770                         }
 771                 }
 772 #endif
 773                 /*
 774                  * Check for the waiters flags about this rwlock.
 775                  * If the lock was released, without maintain any pending
 776                  * waiters queue, simply try to acquire it.
 777                  * If a pending waiters queue is present, claim the lock
 778                  * ownership and maintain the pending queue.
 779                  */
 780                 x = v & (RW_LOCK_WAITERS | RW_LOCK_WRITE_SPINNER);
 781                 if ((v & ~x) == RW_UNLOCKED) {
 782                         x &= ~RW_LOCK_WRITE_SPINNER;
 783                         if (atomic_cmpset_acq_ptr(&rw->rw_lock, v, tid | x)) {
 784                                 if (x)
 785                                         turnstile_claim(ts);
 786                                 else
 787                                         turnstile_cancel(ts);
 788                                 break;
 789                         }
 790                         turnstile_cancel(ts);
 791                         continue;
 792                 }
 793                 /*
 794                  * If the RW_LOCK_WRITE_WAITERS flag isn't set, then try to
 795                  * set it.  If we fail to set it, then loop back and try
 796                  * again.
 797                  */
 798                 if (!(v & RW_LOCK_WRITE_WAITERS)) {
 799                         if (!atomic_cmpset_ptr(&rw->rw_lock, v,
 800                             v | RW_LOCK_WRITE_WAITERS)) {
 801                                 turnstile_cancel(ts);
 802                                 continue;
 803                         }
 804                         if (LOCK_LOG_TEST(&rw->lock_object, 0))
 805                                 CTR2(KTR_LOCK, "%s: %p set write waiters flag",
 806                                     __func__, rw);
 807                 }
 808                 /*
 809                  * We were unable to acquire the lock and the write waiters
 810                  * flag is set, so we must block on the turnstile.
 811                  */
 812                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
 813                         CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__,
 814                             rw);
 815 #ifdef KDTRACE_HOOKS
 816                 sleep_time -= lockstat_nsecs();
 817 #endif
 818                 turnstile_wait(ts, rw_owner(rw), TS_EXCLUSIVE_QUEUE);
 819 #ifdef KDTRACE_HOOKS
 820                 sleep_time += lockstat_nsecs();
 821                 sleep_cnt++;
 822 #endif
 823                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
 824                         CTR2(KTR_LOCK, "%s: %p resuming from turnstile",
 825                             __func__, rw);
 826 #ifdef ADAPTIVE_RWLOCKS
 827                 spintries = 0;
 828 #endif
 829         }
 830         LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_WLOCK_ACQUIRE, rw, contested,
 831             waittime, file, line);
 832 #ifdef KDTRACE_HOOKS
 833         if (sleep_time)
 834                 LOCKSTAT_RECORD1(LS_RW_WLOCK_BLOCK, rw, sleep_time);
 835
 836         /*
 837          * Record only the loops spinning and not sleeping.
 838          */
 839         if (spin_cnt > sleep_cnt)
 840                 LOCKSTAT_RECORD1(LS_RW_WLOCK_SPIN, rw, (spin_cnt - sleep_cnt));
 841 #endif
 842 }
 843
 844 /*
 845  * This function is called if the first try at releasing a write lock failed.
 846  * This means that one of the 2 waiter bits must be set indicating that at
 847  * least one thread is waiting on this lock.
 848  */
 849 void
 850 _rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
 851 {
 852         struct turnstile *ts;
 853         uintptr_t v;
 854         int queue;
 855
 856         if (SCHEDULER_STOPPED())
 857                 return;
 858
 859         if (rw_wlocked(rw) && rw_recursed(rw)) {
 860                 rw->rw_recurse--;
 861                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
 862                         CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw);
 863                 return;
 864         }
 865
 866         KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS),
 867             ("%s: neither of the waiter flags are set", __func__));
 868
 869         if (LOCK_LOG_TEST(&rw->lock_object, 0))
 870                 CTR2(KTR_LOCK, "%s: %p contested", __func__, rw);
 871
 872         turnstile_chain_lock(&rw->lock_object);
 873         ts = turnstile_lookup(&rw->lock_object);
 874         MPASS(ts != NULL);
 875
 876         /*
 877          * Use the same algo as sx locks for now.  Prefer waking up shared
 878          * waiters if we have any over writers.  This is probably not ideal.
 879          *
 880          * 'v' is the value we are going to write back to rw_lock.  If we
 881          * have waiters on both queues, we need to preserve the state of
 882          * the waiter flag for the queue we don't wake up.  For now this is
 883          * hardcoded for the algorithm mentioned above.
 884          *
 885          * In the case of both readers and writers waiting we wakeup the
 886          * readers but leave the RW_LOCK_WRITE_WAITERS flag set.  If a
 887          * new writer comes in before a reader it will claim the lock up
 888          * above.  There is probably a potential priority inversion in
 889          * there that could be worked around either by waking both queues
 890          * of waiters or doing some complicated lock handoff gymnastics.
 891          */
 892         v = RW_UNLOCKED;
 893         if (rw->rw_lock & RW_LOCK_WRITE_WAITERS) {
 894                 queue = TS_EXCLUSIVE_QUEUE;
 895                 v |= (rw->rw_lock & RW_LOCK_READ_WAITERS);
 896         } else
 897                 queue = TS_SHARED_QUEUE;
 898
 899         /* Wake up all waiters for the specific queue. */
 900         if (LOCK_LOG_TEST(&rw->lock_object, 0))
 901                 CTR3(KTR_LOCK, "%s: %p waking up %s waiters", __func__, rw,
 902                     queue == TS_SHARED_QUEUE ? "read" : "write");
 903         turnstile_broadcast(ts, queue);
 904         atomic_store_rel_ptr(&rw->rw_lock, v);
 905         turnstile_unpend(ts, TS_EXCLUSIVE_LOCK);
 906         turnstile_chain_unlock(&rw->lock_object);
 907 }
 908
 909 /*
 910  * Attempt to do a non-blocking upgrade from a read lock to a write
 911  * lock.  This will only succeed if this thread holds a single read
 912  * lock.  Returns true if the upgrade succeeded and false otherwise.
 913  */
 914 int
 915 _rw_try_upgrade(struct rwlock *rw, const char *file, int line)
 916 {
 917         uintptr_t v, x, tid;
 918         struct turnstile *ts;
 919         int success;
 920
 921         if (SCHEDULER_STOPPED())
 922                 return (1);
 923
 924         KASSERT(rw->rw_lock != RW_DESTROYED,
 925             ("rw_try_upgrade() of destroyed rwlock @ %s:%d", file, line));
 926         _rw_assert(rw, RA_RLOCKED, file, line);
 927
 928         /*
 929          * Attempt to switch from one reader to a writer.  If there
 930          * are any write waiters, then we will have to lock the
 931          * turnstile first to prevent races with another writer
 932          * calling turnstile_wait() before we have claimed this
 933          * turnstile.  So, do the simple case of no waiters first.
 934          */
 935         tid = (uintptr_t)curthread;
 936         success = 0;
 937         for (;;) {
 938                 v = rw->rw_lock;
 939                 if (RW_READERS(v) > 1)
 940                         break;
 941                 if (!(v & RW_LOCK_WAITERS)) {
 942                         success = atomic_cmpset_ptr(&rw->rw_lock, v, tid);
 943                         if (!success)
 944                                 continue;
 945                         break;
 946                 }
 947
 948                 /*
 949                  * Ok, we think we have waiters, so lock the turnstile.
 950                  */
 951                 ts = turnstile_trywait(&rw->lock_object);
 952                 v = rw->rw_lock;
 953                 if (RW_READERS(v) > 1) {
 954                         turnstile_cancel(ts);
 955                         break;
 956                 }
 957                 /*
 958                  * Try to switch from one reader to a writer again.  This time
 959                  * we honor the current state of the waiters flags.
 960                  * If we obtain the lock with the flags set, then claim
 961                  * ownership of the turnstile.
 962                  */
 963                 x = rw->rw_lock & RW_LOCK_WAITERS;
 964                 success = atomic_cmpset_ptr(&rw->rw_lock, v, tid | x);
 965                 if (success) {
 966                         if (x)
 967                                 turnstile_claim(ts);
 968                         else
 969                                 turnstile_cancel(ts);
 970                         break;
 971                 }
 972                 turnstile_cancel(ts);
 973         }
 974         LOCK_LOG_TRY("WUPGRADE", &rw->lock_object, 0, success, file, line);
 975         if (success) {
 976                 curthread->td_rw_rlocks--;
 977                 WITNESS_UPGRADE(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
 978                     file, line);
 979                 LOCKSTAT_RECORD0(LS_RW_TRYUPGRADE_UPGRADE, rw);
 980         }
 981         return (success);
 982 }
 983
 984 /*
 985  * Downgrade a write lock into a single read lock.
 986  */
 987 void
 988 _rw_downgrade(struct rwlock *rw, const char *file, int line)
 989 {
 990         struct turnstile *ts;
 991         uintptr_t tid, v;
 992         int rwait, wwait;
 993
 994         if (SCHEDULER_STOPPED())
 995                 return;
 996
 997         KASSERT(rw->rw_lock != RW_DESTROYED,
 998             ("rw_downgrade() of destroyed rwlock @ %s:%d", file, line));
 999         _rw_assert(rw, RA_WLOCKED | RA_NOTRECURSED, file, line);
1000 #ifndef INVARIANTS
1001         if (rw_recursed(rw))
1002                 panic("downgrade of a recursed lock");
1003 #endif
1004
1005         WITNESS_DOWNGRADE(&rw->lock_object, 0, file, line);
1006
1007         /*
1008          * Convert from a writer to a single reader.  First we handle
1009          * the easy case with no waiters.  If there are any waiters, we
1010          * lock the turnstile and "disown" the lock.
1011          */
1012         tid = (uintptr_t)curthread;
1013         if (atomic_cmpset_rel_ptr(&rw->rw_lock, tid, RW_READERS_LOCK(1)))
1014                 goto out;
1015
1016         /*
1017          * Ok, we think we have waiters, so lock the turnstile so we can
1018          * read the waiter flags without any races.
1019          */
1020         turnstile_chain_lock(&rw->lock_object);
1021         v = rw->rw_lock & RW_LOCK_WAITERS;
1022         rwait = v & RW_LOCK_READ_WAITERS;
1023         wwait = v & RW_LOCK_WRITE_WAITERS;
1024         MPASS(rwait | wwait);
1025
1026         /*
1027          * Downgrade from a write lock while preserving waiters flag
1028          * and give up ownership of the turnstile.
1029          */
1030         ts = turnstile_lookup(&rw->lock_object);
1031         MPASS(ts != NULL);
1032         if (!wwait)
1033                 v &= ~RW_LOCK_READ_WAITERS;
1034         atomic_store_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v);
1035         /*
1036          * Wake other readers if there are no writers pending.  Otherwise they
1037          * won't be able to acquire the lock anyway.
1038          */
1039         if (rwait && !wwait) {
1040                 turnstile_broadcast(ts, TS_SHARED_QUEUE);
1041                 turnstile_unpend(ts, TS_EXCLUSIVE_LOCK);
1042         } else
1043                 turnstile_disown(ts);
1044         turnstile_chain_unlock(&rw->lock_object);
1045 out:
1046         curthread->td_rw_rlocks++;
1047         LOCK_LOG_LOCK("WDOWNGRADE", &rw->lock_object, 0, 0, file, line);
1048         LOCKSTAT_RECORD0(LS_RW_DOWNGRADE_DOWNGRADE, rw);
1049 }
1050
1051 #ifdef INVARIANT_SUPPORT
1052 #ifndef INVARIANTS
1053 #undef _rw_assert
1054 #endif
1055
1056 /*
1057  * In the non-WITNESS case, rw_assert() can only detect that at least
1058  * *some* thread owns an rlock, but it cannot guarantee that *this*
1059  * thread owns an rlock.
1060  */
1061 void
1062 _rw_assert(struct rwlock *rw, int what, const char *file, int line)
1063 {
1064
1065         if (panicstr != NULL)
1066                 return;
1067         switch (what) {
1068         case RA_LOCKED:
1069         case RA_LOCKED | RA_RECURSED:
1070         case RA_LOCKED | RA_NOTRECURSED:
1071         case RA_RLOCKED:
1072         case RA_RLOCKED | RA_RECURSED:
1073         case RA_RLOCKED | RA_NOTRECURSED:
1074 #ifdef WITNESS
1075                 witness_assert(&rw->lock_object, what, file, line);
1076 #else
1077                 /*
1078                  * If some other thread has a write lock or we have one
1079                  * and are asserting a read lock, fail.  Also, if no one
1080                  * has a lock at all, fail.
1081                  */
1082                 if (rw->rw_lock == RW_UNLOCKED ||
1083                     (!(rw->rw_lock & RW_LOCK_READ) && (what & RA_RLOCKED ||
1084                     rw_wowner(rw) != curthread)))
1085                         panic("Lock %s not %slocked @ %s:%d\n",
1086                             rw->lock_object.lo_name, (what & RA_RLOCKED) ?
1087                             "read " : "", file, line);
1088
1089                 if (!(rw->rw_lock & RW_LOCK_READ) && !(what & RA_RLOCKED)) {
1090                         if (rw_recursed(rw)) {
1091                                 if (what & RA_NOTRECURSED)
1092                                         panic("Lock %s recursed @ %s:%d\n",
1093                                             rw->lock_object.lo_name, file,
1094                                             line);
1095                         } else if (what & RA_RECURSED)
1096                                 panic("Lock %s not recursed @ %s:%d\n",
1097                                     rw->lock_object.lo_name, file, line);
1098                 }
1099 #endif
1100                 break;
1101         case RA_WLOCKED:
1102         case RA_WLOCKED | RA_RECURSED:
1103         case RA_WLOCKED | RA_NOTRECURSED:
1104                 if (rw_wowner(rw) != curthread)
1105                         panic("Lock %s not exclusively locked @ %s:%d\n",
1106                             rw->lock_object.lo_name, file, line);
1107                 if (rw_recursed(rw)) {
1108                         if (what & RA_NOTRECURSED)
1109                                 panic("Lock %s recursed @ %s:%d\n",
1110                                     rw->lock_object.lo_name, file, line);
1111                 } else if (what & RA_RECURSED)
1112                         panic("Lock %s not recursed @ %s:%d\n",
1113                             rw->lock_object.lo_name, file, line);
1114                 break;
1115         case RA_UNLOCKED:
1116 #ifdef WITNESS
1117                 witness_assert(&rw->lock_object, what, file, line);
1118 #else
1119                 /*
1120                  * If we hold a write lock fail.  We can't reliably check
1121                  * to see if we hold a read lock or not.
1122                  */
1123                 if (rw_wowner(rw) == curthread)
1124                         panic("Lock %s exclusively locked @ %s:%d\n",
1125                             rw->lock_object.lo_name, file, line);
1126 #endif
1127                 break;
1128         default:
1129                 panic("Unknown rw lock assertion: %d @ %s:%d", what, file,
1130                     line);
1131         }
1132 }
1133 #endif /* INVARIANT_SUPPORT */
1134
1135 #ifdef DDB
1136 void
1137 db_show_rwlock(struct lock_object *lock)
1138 {
1139         struct rwlock *rw;
1140         struct thread *td;
1141
1142         rw = (struct rwlock *)lock;
1143
1144         db_printf(" state: ");
1145         if (rw->rw_lock == RW_UNLOCKED)
1146                 db_printf("UNLOCKED\n");
1147         else if (rw->rw_lock == RW_DESTROYED) {
1148                 db_printf("DESTROYED\n");
1149                 return;
1150         } else if (rw->rw_lock & RW_LOCK_READ)
1151                 db_printf("RLOCK: %ju locks\n",
1152                     (uintmax_t)(RW_READERS(rw->rw_lock)));
1153         else {
1154                 td = rw_wowner(rw);
1155                 db_printf("WLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
1156                     td->td_tid, td->td_proc->p_pid, td->td_name);
1157                 if (rw_recursed(rw))
1158                         db_printf(" recursed: %u\n", rw->rw_recurse);
1159         }
1160         db_printf(" waiters: ");
1161         switch (rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS)) {
1162         case RW_LOCK_READ_WAITERS:
1163                 db_printf("readers\n");
1164                 break;
1165         case RW_LOCK_WRITE_WAITERS:
1166                 db_printf("writers\n");
1167                 break;
1168         case RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS:
1169                 db_printf("readers and writers\n");
1170                 break;
1171         default:
1172                 db_printf("none\n");
1173                 break;
1174         }
1175 }
1176
1177 #endif