sys/kern/kern_condvar.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2000 Jake Burkholder <jake@freebsd.org>.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include "opt_ktrace.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/systm.h>
  36 #include <sys/limits.h>
  37 #include <sys/lock.h>
  38 #include <sys/mutex.h>
  39 #include <sys/proc.h>
  40 #include <sys/kernel.h>
  41 #include <sys/ktr.h>
  42 #include <sys/condvar.h>
  43 #include <sys/sched.h>
  44 #include <sys/signalvar.h>
  45 #include <sys/sleepqueue.h>
  46 #include <sys/resourcevar.h>
  47 #ifdef KTRACE
  48 #include <sys/uio.h>
  49 #include <sys/ktrace.h>
  50 #endif
  51
  52 /*
  53  * A bound below which cv_waiters is valid.  Once cv_waiters reaches this bound,
  54  * cv_signal must manually check the wait queue for threads.
  55  */
  56 #define CV_WAITERS_BOUND        INT_MAX
  57
  58 #define CV_WAITERS_INC(cvp) do {                                        \
  59         if ((cvp)->cv_waiters < CV_WAITERS_BOUND)                       \
  60                 (cvp)->cv_waiters++;                                    \
  61 } while (0)
  62
  63 /*
  64  * Common sanity checks for cv_wait* functions.
  65  */
  66 #define CV_ASSERT(cvp, lock, td) do {                                   \
  67         KASSERT((td) != NULL, ("%s: td NULL", __func__));               \
  68         KASSERT(TD_IS_RUNNING(td), ("%s: not TDS_RUNNING", __func__));  \
  69         KASSERT((cvp) != NULL, ("%s: cvp NULL", __func__));             \
  70         KASSERT((lock) != NULL, ("%s: lock NULL", __func__));           \
  71 } while (0)
  72
  73 /*
  74  * Initialize a condition variable.  Must be called before use.
  75  */
  76 void
  77 cv_init(struct cv *cvp, const char *desc)
  78 {
  79
  80         cvp->cv_description = desc;
  81         cvp->cv_waiters = 0;
  82 }
  83
  84 /*
  85  * Destroy a condition variable.  The condition variable must be re-initialized
  86  * in order to be re-used.
  87  */
  88 void
  89 cv_destroy(struct cv *cvp)
  90 {
  91 #ifdef INVARIANTS
  92         struct sleepqueue *sq;
  93
  94         sleepq_lock(cvp);
  95         sq = sleepq_lookup(cvp);
  96         sleepq_release(cvp);
  97         KASSERT(sq == NULL, ("%s: associated sleep queue non-empty", __func__));
  98 #endif
  99 }
 100
 101 /*
 102  * Wait on a condition variable.  The current thread is placed on the condition
 103  * variable's wait queue and suspended.  A cv_signal or cv_broadcast on the same
 104  * condition variable will resume the thread.  The mutex is released before
 105  * sleeping and will be held on return.  It is recommended that the mutex be
 106  * held when cv_signal or cv_broadcast are called.
 107  */
 108 void
 109 _cv_wait(struct cv *cvp, struct lock_object *lock)
 110 {
 111         WITNESS_SAVE_DECL(lock_witness);
 112         struct lock_class *class;
 113         struct thread *td;
 114         uintptr_t lock_state;
 115
 116         td = curthread;
 117         lock_state = 0;
 118 #ifdef KTRACE
 119         if (KTRPOINT(td, KTR_CSW))
 120                 ktrcsw(1, 0, cv_wmesg(cvp));
 121 #endif
 122         CV_ASSERT(cvp, lock, td);
 123         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
 124             "Waiting on \"%s\"", cvp->cv_description);
 125         class = LOCK_CLASS(lock);
 126
 127         if (SCHEDULER_STOPPED_TD(td))
 128                 return;
 129
 130         sleepq_lock(cvp);
 131
 132         CV_WAITERS_INC(cvp);
 133         if (lock == &Giant.lock_object)
 134                 mtx_assert(&Giant, MA_OWNED);
 135         DROP_GIANT();
 136
 137         sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0);
 138         if (lock != &Giant.lock_object) {
 139                 if (class->lc_flags & LC_SLEEPABLE)
 140                         sleepq_release(cvp);
 141                 WITNESS_SAVE(lock, lock_witness);
 142                 lock_state = class->lc_unlock(lock);
 143                 if (class->lc_flags & LC_SLEEPABLE)
 144                         sleepq_lock(cvp);
 145         }
 146         sleepq_wait(cvp, 0);
 147
 148 #ifdef KTRACE
 149         if (KTRPOINT(td, KTR_CSW))
 150                 ktrcsw(0, 0, cv_wmesg(cvp));
 151 #endif
 152         PICKUP_GIANT();
 153         if (lock != &Giant.lock_object) {
 154                 class->lc_lock(lock, lock_state);
 155                 WITNESS_RESTORE(lock, lock_witness);
 156         }
 157 }
 158
 159 /*
 160  * Wait on a condition variable.  This function differs from cv_wait by
 161  * not acquiring the mutex after condition variable was signaled.
 162  */
 163 void
 164 _cv_wait_unlock(struct cv *cvp, struct lock_object *lock)
 165 {
 166         struct lock_class *class;
 167         struct thread *td;
 168
 169         td = curthread;
 170 #ifdef KTRACE
 171         if (KTRPOINT(td, KTR_CSW))
 172                 ktrcsw(1, 0, cv_wmesg(cvp));
 173 #endif
 174         CV_ASSERT(cvp, lock, td);
 175         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
 176             "Waiting on \"%s\"", cvp->cv_description);
 177         KASSERT(lock != &Giant.lock_object,
 178             ("cv_wait_unlock cannot be used with Giant"));
 179         class = LOCK_CLASS(lock);
 180
 181         if (SCHEDULER_STOPPED_TD(td)) {
 182                 class->lc_unlock(lock);
 183                 return;
 184         }
 185
 186         sleepq_lock(cvp);
 187
 188         CV_WAITERS_INC(cvp);
 189         DROP_GIANT();
 190
 191         sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0);
 192         if (class->lc_flags & LC_SLEEPABLE)
 193                 sleepq_release(cvp);
 194         class->lc_unlock(lock);
 195         if (class->lc_flags & LC_SLEEPABLE)
 196                 sleepq_lock(cvp);
 197         sleepq_wait(cvp, 0);
 198
 199 #ifdef KTRACE
 200         if (KTRPOINT(td, KTR_CSW))
 201                 ktrcsw(0, 0, cv_wmesg(cvp));
 202 #endif
 203         PICKUP_GIANT();
 204 }
 205
 206 /*
 207  * Wait on a condition variable, allowing interruption by signals.  Return 0 if
 208  * the thread was resumed with cv_signal or cv_broadcast, EINTR or ERESTART if
 209  * a signal was caught.  If ERESTART is returned the system call should be
 210  * restarted if possible.
 211  */
 212 int
 213 _cv_wait_sig(struct cv *cvp, struct lock_object *lock)
 214 {
 215         WITNESS_SAVE_DECL(lock_witness);
 216         struct lock_class *class;
 217         struct thread *td;
 218         uintptr_t lock_state;
 219         int rval;
 220
 221         td = curthread;
 222         lock_state = 0;
 223 #ifdef KTRACE
 224         if (KTRPOINT(td, KTR_CSW))
 225                 ktrcsw(1, 0, cv_wmesg(cvp));
 226 #endif
 227         CV_ASSERT(cvp, lock, td);
 228         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
 229             "Waiting on \"%s\"", cvp->cv_description);
 230         class = LOCK_CLASS(lock);
 231
 232         if (SCHEDULER_STOPPED_TD(td))
 233                 return (0);
 234
 235         sleepq_lock(cvp);
 236
 237         CV_WAITERS_INC(cvp);
 238         if (lock == &Giant.lock_object)
 239                 mtx_assert(&Giant, MA_OWNED);
 240         DROP_GIANT();
 241
 242         sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR |
 243             SLEEPQ_INTERRUPTIBLE, 0);
 244         if (lock != &Giant.lock_object) {
 245                 if (class->lc_flags & LC_SLEEPABLE)
 246                         sleepq_release(cvp);
 247                 WITNESS_SAVE(lock, lock_witness);
 248                 lock_state = class->lc_unlock(lock);
 249                 if (class->lc_flags & LC_SLEEPABLE)
 250                         sleepq_lock(cvp);
 251         }
 252         rval = sleepq_wait_sig(cvp, 0);
 253
 254 #ifdef KTRACE
 255         if (KTRPOINT(td, KTR_CSW))
 256                 ktrcsw(0, 0, cv_wmesg(cvp));
 257 #endif
 258         PICKUP_GIANT();
 259         if (lock != &Giant.lock_object) {
 260                 class->lc_lock(lock, lock_state);
 261                 WITNESS_RESTORE(lock, lock_witness);
 262         }
 263
 264         return (rval);
 265 }
 266
 267 /*
 268  * Wait on a condition variable for (at most) the value specified in sbt
 269  * argument. Returns 0 if the process was resumed by cv_signal or cv_broadcast,
 270  * EWOULDBLOCK if the timeout expires.
 271  */
 272 int
 273 _cv_timedwait_sbt(struct cv *cvp, struct lock_object *lock, sbintime_t sbt,
 274     sbintime_t pr, int flags)
 275 {
 276         WITNESS_SAVE_DECL(lock_witness);
 277         struct lock_class *class;
 278         struct thread *td;
 279         int lock_state, rval;
 280
 281         td = curthread;
 282         lock_state = 0;
 283 #ifdef KTRACE
 284         if (KTRPOINT(td, KTR_CSW))
 285                 ktrcsw(1, 0, cv_wmesg(cvp));
 286 #endif
 287         CV_ASSERT(cvp, lock, td);
 288         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
 289             "Waiting on \"%s\"", cvp->cv_description);
 290         class = LOCK_CLASS(lock);
 291
 292         if (SCHEDULER_STOPPED_TD(td))
 293                 return (0);
 294
 295         sleepq_lock(cvp);
 296
 297         CV_WAITERS_INC(cvp);
 298         if (lock == &Giant.lock_object)
 299                 mtx_assert(&Giant, MA_OWNED);
 300         DROP_GIANT();
 301
 302         sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0);
 303         sleepq_set_timeout_sbt(cvp, sbt, pr, flags);
 304         if (lock != &Giant.lock_object) {
 305                 if (class->lc_flags & LC_SLEEPABLE)
 306                         sleepq_release(cvp);
 307                 WITNESS_SAVE(lock, lock_witness);
 308                 lock_state = class->lc_unlock(lock);
 309                 if (class->lc_flags & LC_SLEEPABLE)
 310                         sleepq_lock(cvp);
 311         }
 312         rval = sleepq_timedwait(cvp, 0);
 313
 314 #ifdef KTRACE
 315         if (KTRPOINT(td, KTR_CSW))
 316                 ktrcsw(0, 0, cv_wmesg(cvp));
 317 #endif
 318         PICKUP_GIANT();
 319         if (lock != &Giant.lock_object) {
 320                 class->lc_lock(lock, lock_state);
 321                 WITNESS_RESTORE(lock, lock_witness);
 322         }
 323
 324         return (rval);
 325 }
 326
 327 /*
 328  * Wait on a condition variable for (at most) the value specified in sbt
 329  * argument, allowing interruption by signals.
 330  * Returns 0 if the thread was resumed by cv_signal or cv_broadcast,
 331  * EWOULDBLOCK if the timeout expires, and EINTR or ERESTART if a signal
 332  * was caught.
 333  */
 334 int
 335 _cv_timedwait_sig_sbt(struct cv *cvp, struct lock_object *lock,
 336     sbintime_t sbt, sbintime_t pr, int flags)
 337 {
 338         WITNESS_SAVE_DECL(lock_witness);
 339         struct lock_class *class;
 340         struct thread *td;
 341         int lock_state, rval;
 342
 343         td = curthread;
 344         lock_state = 0;
 345 #ifdef KTRACE
 346         if (KTRPOINT(td, KTR_CSW))
 347                 ktrcsw(1, 0, cv_wmesg(cvp));
 348 #endif
 349         CV_ASSERT(cvp, lock, td);
 350         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
 351             "Waiting on \"%s\"", cvp->cv_description);
 352         class = LOCK_CLASS(lock);
 353
 354         if (SCHEDULER_STOPPED_TD(td))
 355                 return (0);
 356
 357         sleepq_lock(cvp);
 358
 359         CV_WAITERS_INC(cvp);
 360         if (lock == &Giant.lock_object)
 361                 mtx_assert(&Giant, MA_OWNED);
 362         DROP_GIANT();
 363
 364         sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR |
 365             SLEEPQ_INTERRUPTIBLE, 0);
 366         sleepq_set_timeout_sbt(cvp, sbt, pr, flags);
 367         if (lock != &Giant.lock_object) {
 368                 if (class->lc_flags & LC_SLEEPABLE)
 369                         sleepq_release(cvp);
 370                 WITNESS_SAVE(lock, lock_witness);
 371                 lock_state = class->lc_unlock(lock);
 372                 if (class->lc_flags & LC_SLEEPABLE)
 373                         sleepq_lock(cvp);
 374         }
 375         rval = sleepq_timedwait_sig(cvp, 0);
 376
 377 #ifdef KTRACE
 378         if (KTRPOINT(td, KTR_CSW))
 379                 ktrcsw(0, 0, cv_wmesg(cvp));
 380 #endif
 381         PICKUP_GIANT();
 382         if (lock != &Giant.lock_object) {
 383                 class->lc_lock(lock, lock_state);
 384                 WITNESS_RESTORE(lock, lock_witness);
 385         }
 386
 387         return (rval);
 388 }
 389
 390 /*
 391  * Signal a condition variable, wakes up one waiting thread.  Will also wakeup
 392  * the swapper if the process is not in memory, so that it can bring the
 393  * sleeping process in.  Note that this may also result in additional threads
 394  * being made runnable.  Should be called with the same mutex as was passed to
 395  * cv_wait held.
 396  */
 397 void
 398 cv_signal(struct cv *cvp)
 399 {
 400         int wakeup_swapper;
 401
 402         if (cvp->cv_waiters == 0)
 403                 return;
 404         wakeup_swapper = 0;
 405         sleepq_lock(cvp);
 406         if (cvp->cv_waiters > 0) {
 407                 if (cvp->cv_waiters == CV_WAITERS_BOUND &&
 408                     sleepq_lookup(cvp) == NULL) {
 409                         cvp->cv_waiters = 0;
 410                 } else {
 411                         if (cvp->cv_waiters < CV_WAITERS_BOUND)
 412                                 cvp->cv_waiters--;
 413                         wakeup_swapper = sleepq_signal(cvp, SLEEPQ_CONDVAR, 0,
 414                             0);
 415                 }
 416         }
 417         sleepq_release(cvp);
 418         if (wakeup_swapper)
 419                 kick_proc0();
 420 }
 421
 422 /*
 423  * Broadcast a signal to a condition variable.  Wakes up all waiting threads.
 424  * Should be called with the same mutex as was passed to cv_wait held.
 425  */
 426 void
 427 cv_broadcastpri(struct cv *cvp, int pri)
 428 {
 429         int wakeup_swapper;
 430
 431         if (cvp->cv_waiters == 0)
 432                 return;
 433         /*
 434          * XXX sleepq_broadcast pri argument changed from -1 meaning
 435          * no pri to 0 meaning no pri.
 436          */
 437         wakeup_swapper = 0;
 438         if (pri == -1)
 439                 pri = 0;
 440         sleepq_lock(cvp);
 441         if (cvp->cv_waiters > 0) {
 442                 cvp->cv_waiters = 0;
 443                 wakeup_swapper = sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri, 0);
 444         }
 445         sleepq_release(cvp);
 446         if (wakeup_swapper)
 447                 kick_proc0();
 448 }