sys/kern/kern_condvar.c

   1 /*-
   2  * Copyright (c) 2000 Jake Burkholder <jake@freebsd.org>.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #include <sys/cdefs.h>
  28 __FBSDID("$FreeBSD$");
  29
  30 #include "opt_ktrace.h"
  31
  32 #include <sys/param.h>
  33 #include <sys/systm.h>
  34 #include <sys/limits.h>
  35 #include <sys/lock.h>
  36 #include <sys/mutex.h>
  37 #include <sys/proc.h>
  38 #include <sys/kernel.h>
  39 #include <sys/ktr.h>
  40 #include <sys/condvar.h>
  41 #include <sys/sched.h>
  42 #include <sys/signalvar.h>
  43 #include <sys/sleepqueue.h>
  44 #include <sys/resourcevar.h>
  45 #ifdef KTRACE
  46 #include <sys/uio.h>
  47 #include <sys/ktrace.h>
  48 #endif
  49
  50 /*
  51  * A bound below which cv_waiters is valid.  Once cv_waiters reaches this bound,
  52  * cv_signal must manually check the wait queue for threads.
  53  */
  54 #define CV_WAITERS_BOUND        INT_MAX
  55
  56 #define CV_WAITERS_INC(cvp) do {                                        \
  57         if ((cvp)->cv_waiters < CV_WAITERS_BOUND)                       \
  58                 (cvp)->cv_waiters++;                                    \
  59 } while (0)
  60
  61 /*
  62  * Common sanity checks for cv_wait* functions.
  63  */
  64 #define CV_ASSERT(cvp, lock, td) do {                                   \
  65         KASSERT((td) != NULL, ("%s: td NULL", __func__));               \
  66         KASSERT(TD_IS_RUNNING(td), ("%s: not TDS_RUNNING", __func__));  \
  67         KASSERT((cvp) != NULL, ("%s: cvp NULL", __func__));             \
  68         KASSERT((lock) != NULL, ("%s: lock NULL", __func__));           \
  69 } while (0)
  70
  71 /*
  72  * Initialize a condition variable.  Must be called before use.
  73  */
  74 void
  75 cv_init(struct cv *cvp, const char *desc)
  76 {
  77
  78         cvp->cv_description = desc;
  79         cvp->cv_waiters = 0;
  80 }
  81
  82 /*
  83  * Destroy a condition variable.  The condition variable must be re-initialized
  84  * in order to be re-used.
  85  */
  86 void
  87 cv_destroy(struct cv *cvp)
  88 {
  89 #ifdef INVARIANTS
  90         struct sleepqueue *sq;
  91
  92         sleepq_lock(cvp);
  93         sq = sleepq_lookup(cvp);
  94         sleepq_release(cvp);
  95         KASSERT(sq == NULL, ("%s: associated sleep queue non-empty", __func__));
  96 #endif
  97 }
  98
  99 /*
 100  * Wait on a condition variable.  The current thread is placed on the condition
 101  * variable's wait queue and suspended.  A cv_signal or cv_broadcast on the same
 102  * condition variable will resume the thread.  The mutex is released before
 103  * sleeping and will be held on return.  It is recommended that the mutex be
 104  * held when cv_signal or cv_broadcast are called.
 105  */
 106 void
 107 _cv_wait(struct cv *cvp, struct lock_object *lock)
 108 {
 109         WITNESS_SAVE_DECL(lock_witness);
 110         struct lock_class *class;
 111         struct thread *td;
 112         uintptr_t lock_state;
 113
 114         td = curthread;
 115         lock_state = 0;
 116 #ifdef KTRACE
 117         if (KTRPOINT(td, KTR_CSW))
 118                 ktrcsw(1, 0, cv_wmesg(cvp));
 119 #endif
 120         CV_ASSERT(cvp, lock, td);
 121         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
 122             "Waiting on \"%s\"", cvp->cv_description);
 123         class = LOCK_CLASS(lock);
 124
 125         if (SCHEDULER_STOPPED_TD(td))
 126                 return;
 127
 128         sleepq_lock(cvp);
 129
 130         CV_WAITERS_INC(cvp);
 131         if (lock == &Giant.lock_object)
 132                 mtx_assert(&Giant, MA_OWNED);
 133         DROP_GIANT();
 134
 135         sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0);
 136         if (lock != &Giant.lock_object) {
 137                 if (class->lc_flags & LC_SLEEPABLE)
 138                         sleepq_release(cvp);
 139                 WITNESS_SAVE(lock, lock_witness);
 140                 lock_state = class->lc_unlock(lock);
 141                 if (class->lc_flags & LC_SLEEPABLE)
 142                         sleepq_lock(cvp);
 143         }
 144         sleepq_wait(cvp, 0);
 145
 146 #ifdef KTRACE
 147         if (KTRPOINT(td, KTR_CSW))
 148                 ktrcsw(0, 0, cv_wmesg(cvp));
 149 #endif
 150         PICKUP_GIANT();
 151         if (lock != &Giant.lock_object) {
 152                 class->lc_lock(lock, lock_state);
 153                 WITNESS_RESTORE(lock, lock_witness);
 154         }
 155 }
 156
 157 /*
 158  * Wait on a condition variable.  This function differs from cv_wait by
 159  * not acquiring the mutex after condition variable was signaled.
 160  */
 161 void
 162 _cv_wait_unlock(struct cv *cvp, struct lock_object *lock)
 163 {
 164         struct lock_class *class;
 165         struct thread *td;
 166
 167         td = curthread;
 168 #ifdef KTRACE
 169         if (KTRPOINT(td, KTR_CSW))
 170                 ktrcsw(1, 0, cv_wmesg(cvp));
 171 #endif
 172         CV_ASSERT(cvp, lock, td);
 173         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
 174             "Waiting on \"%s\"", cvp->cv_description);
 175         KASSERT(lock != &Giant.lock_object,
 176             ("cv_wait_unlock cannot be used with Giant"));
 177         class = LOCK_CLASS(lock);
 178
 179         if (SCHEDULER_STOPPED_TD(td)) {
 180                 class->lc_unlock(lock);
 181                 return;
 182         }
 183
 184         sleepq_lock(cvp);
 185
 186         CV_WAITERS_INC(cvp);
 187         DROP_GIANT();
 188
 189         sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0);
 190         if (class->lc_flags & LC_SLEEPABLE)
 191                 sleepq_release(cvp);
 192         class->lc_unlock(lock);
 193         if (class->lc_flags & LC_SLEEPABLE)
 194                 sleepq_lock(cvp);
 195         sleepq_wait(cvp, 0);
 196
 197 #ifdef KTRACE
 198         if (KTRPOINT(td, KTR_CSW))
 199                 ktrcsw(0, 0, cv_wmesg(cvp));
 200 #endif
 201         PICKUP_GIANT();
 202 }
 203
 204 /*
 205  * Wait on a condition variable, allowing interruption by signals.  Return 0 if
 206  * the thread was resumed with cv_signal or cv_broadcast, EINTR or ERESTART if
 207  * a signal was caught.  If ERESTART is returned the system call should be
 208  * restarted if possible.
 209  */
 210 int
 211 _cv_wait_sig(struct cv *cvp, struct lock_object *lock)
 212 {
 213         WITNESS_SAVE_DECL(lock_witness);
 214         struct lock_class *class;
 215         struct thread *td;
 216         uintptr_t lock_state;
 217         int rval;
 218
 219         td = curthread;
 220         lock_state = 0;
 221 #ifdef KTRACE
 222         if (KTRPOINT(td, KTR_CSW))
 223                 ktrcsw(1, 0, cv_wmesg(cvp));
 224 #endif
 225         CV_ASSERT(cvp, lock, td);
 226         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
 227             "Waiting on \"%s\"", cvp->cv_description);
 228         class = LOCK_CLASS(lock);
 229
 230         if (SCHEDULER_STOPPED_TD(td))
 231                 return (0);
 232
 233         sleepq_lock(cvp);
 234
 235         CV_WAITERS_INC(cvp);
 236         if (lock == &Giant.lock_object)
 237                 mtx_assert(&Giant, MA_OWNED);
 238         DROP_GIANT();
 239
 240         sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR |
 241             SLEEPQ_INTERRUPTIBLE, 0);
 242         if (lock != &Giant.lock_object) {
 243                 if (class->lc_flags & LC_SLEEPABLE)
 244                         sleepq_release(cvp);
 245                 WITNESS_SAVE(lock, lock_witness);
 246                 lock_state = class->lc_unlock(lock);
 247                 if (class->lc_flags & LC_SLEEPABLE)
 248                         sleepq_lock(cvp);
 249         }
 250         rval = sleepq_wait_sig(cvp, 0);
 251
 252 #ifdef KTRACE
 253         if (KTRPOINT(td, KTR_CSW))
 254                 ktrcsw(0, 0, cv_wmesg(cvp));
 255 #endif
 256         PICKUP_GIANT();
 257         if (lock != &Giant.lock_object) {
 258                 class->lc_lock(lock, lock_state);
 259                 WITNESS_RESTORE(lock, lock_witness);
 260         }
 261
 262         return (rval);
 263 }
 264
 265 /*
 266  * Wait on a condition variable for (at most) the value specified in sbt
 267  * argument. Returns 0 if the process was resumed by cv_signal or cv_broadcast,
 268  * EWOULDBLOCK if the timeout expires.
 269  */
 270 int
 271 _cv_timedwait_sbt(struct cv *cvp, struct lock_object *lock, sbintime_t sbt,
 272     sbintime_t pr, int flags)
 273 {
 274         WITNESS_SAVE_DECL(lock_witness);
 275         struct lock_class *class;
 276         struct thread *td;
 277         int lock_state, rval;
 278
 279         td = curthread;
 280         lock_state = 0;
 281 #ifdef KTRACE
 282         if (KTRPOINT(td, KTR_CSW))
 283                 ktrcsw(1, 0, cv_wmesg(cvp));
 284 #endif
 285         CV_ASSERT(cvp, lock, td);
 286         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
 287             "Waiting on \"%s\"", cvp->cv_description);
 288         class = LOCK_CLASS(lock);
 289
 290         if (SCHEDULER_STOPPED_TD(td))
 291                 return (0);
 292
 293         sleepq_lock(cvp);
 294
 295         CV_WAITERS_INC(cvp);
 296         if (lock == &Giant.lock_object)
 297                 mtx_assert(&Giant, MA_OWNED);
 298         DROP_GIANT();
 299
 300         sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0);
 301         sleepq_set_timeout_sbt(cvp, sbt, pr, flags);
 302         if (lock != &Giant.lock_object) {
 303                 if (class->lc_flags & LC_SLEEPABLE)
 304                         sleepq_release(cvp);
 305                 WITNESS_SAVE(lock, lock_witness);
 306                 lock_state = class->lc_unlock(lock);
 307                 if (class->lc_flags & LC_SLEEPABLE)
 308                         sleepq_lock(cvp);
 309         }
 310         rval = sleepq_timedwait(cvp, 0);
 311
 312 #ifdef KTRACE
 313         if (KTRPOINT(td, KTR_CSW))
 314                 ktrcsw(0, 0, cv_wmesg(cvp));
 315 #endif
 316         PICKUP_GIANT();
 317         if (lock != &Giant.lock_object) {
 318                 class->lc_lock(lock, lock_state);
 319                 WITNESS_RESTORE(lock, lock_witness);
 320         }
 321
 322         return (rval);
 323 }
 324
 325 /*
 326  * Wait on a condition variable for (at most) the value specified in sbt
 327  * argument, allowing interruption by signals.
 328  * Returns 0 if the thread was resumed by cv_signal or cv_broadcast,
 329  * EWOULDBLOCK if the timeout expires, and EINTR or ERESTART if a signal
 330  * was caught.
 331  */
 332 int
 333 _cv_timedwait_sig_sbt(struct cv *cvp, struct lock_object *lock,
 334     sbintime_t sbt, sbintime_t pr, int flags)
 335 {
 336         WITNESS_SAVE_DECL(lock_witness);
 337         struct lock_class *class;
 338         struct thread *td;
 339         int lock_state, rval;
 340
 341         td = curthread;
 342         lock_state = 0;
 343 #ifdef KTRACE
 344         if (KTRPOINT(td, KTR_CSW))
 345                 ktrcsw(1, 0, cv_wmesg(cvp));
 346 #endif
 347         CV_ASSERT(cvp, lock, td);
 348         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
 349             "Waiting on \"%s\"", cvp->cv_description);
 350         class = LOCK_CLASS(lock);
 351
 352         if (SCHEDULER_STOPPED_TD(td))
 353                 return (0);
 354
 355         sleepq_lock(cvp);
 356
 357         CV_WAITERS_INC(cvp);
 358         if (lock == &Giant.lock_object)
 359                 mtx_assert(&Giant, MA_OWNED);
 360         DROP_GIANT();
 361
 362         sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR |
 363             SLEEPQ_INTERRUPTIBLE, 0);
 364         sleepq_set_timeout_sbt(cvp, sbt, pr, flags);
 365         if (lock != &Giant.lock_object) {
 366                 if (class->lc_flags & LC_SLEEPABLE)
 367                         sleepq_release(cvp);
 368                 WITNESS_SAVE(lock, lock_witness);
 369                 lock_state = class->lc_unlock(lock);
 370                 if (class->lc_flags & LC_SLEEPABLE)
 371                         sleepq_lock(cvp);
 372         }
 373         rval = sleepq_timedwait_sig(cvp, 0);
 374
 375 #ifdef KTRACE
 376         if (KTRPOINT(td, KTR_CSW))
 377                 ktrcsw(0, 0, cv_wmesg(cvp));
 378 #endif
 379         PICKUP_GIANT();
 380         if (lock != &Giant.lock_object) {
 381                 class->lc_lock(lock, lock_state);
 382                 WITNESS_RESTORE(lock, lock_witness);
 383         }
 384
 385         return (rval);
 386 }
 387
 388 /*
 389  * Signal a condition variable, wakes up one waiting thread.  Will also wakeup
 390  * the swapper if the process is not in memory, so that it can bring the
 391  * sleeping process in.  Note that this may also result in additional threads
 392  * being made runnable.  Should be called with the same mutex as was passed to
 393  * cv_wait held.
 394  */
 395 void
 396 cv_signal(struct cv *cvp)
 397 {
 398         int wakeup_swapper;
 399
 400         if (cvp->cv_waiters == 0)
 401                 return;
 402         wakeup_swapper = 0;
 403         sleepq_lock(cvp);
 404         if (cvp->cv_waiters > 0) {
 405                 if (cvp->cv_waiters == CV_WAITERS_BOUND &&
 406                     sleepq_lookup(cvp) == NULL) {
 407                         cvp->cv_waiters = 0;
 408                 } else {
 409                         if (cvp->cv_waiters < CV_WAITERS_BOUND)
 410                                 cvp->cv_waiters--;
 411                         wakeup_swapper = sleepq_signal(cvp, SLEEPQ_CONDVAR, 0,
 412                             0);
 413                 }
 414         }
 415         sleepq_release(cvp);
 416         if (wakeup_swapper)
 417                 kick_proc0();
 418 }
 419
 420 /*
 421  * Broadcast a signal to a condition variable.  Wakes up all waiting threads.
 422  * Should be called with the same mutex as was passed to cv_wait held.
 423  */
 424 void
 425 cv_broadcastpri(struct cv *cvp, int pri)
 426 {
 427         int wakeup_swapper;
 428
 429         if (cvp->cv_waiters == 0)
 430                 return;
 431         /*
 432          * XXX sleepq_broadcast pri argument changed from -1 meaning
 433          * no pri to 0 meaning no pri.
 434          */
 435         wakeup_swapper = 0;
 436         if (pri == -1)
 437                 pri = 0;
 438         sleepq_lock(cvp);
 439         if (cvp->cv_waiters > 0) {
 440                 cvp->cv_waiters = 0;
 441                 wakeup_swapper = sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri, 0);
 442         }
 443         sleepq_release(cvp);
 444         if (wakeup_swapper)
 445                 kick_proc0();
 446 }