sys/kern/kern_condvar.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause
   3  *
   4  * Copyright (c) 2000 Jake Burkholder <jake@freebsd.org>.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 #include <sys/cdefs.h>
  30 #include "opt_ktrace.h"
  31
  32 #include <sys/param.h>
  33 #include <sys/systm.h>
  34 #include <sys/limits.h>
  35 #include <sys/lock.h>
  36 #include <sys/mutex.h>
  37 #include <sys/proc.h>
  38 #include <sys/kernel.h>
  39 #include <sys/ktr.h>
  40 #include <sys/ktrace.h>
  41 #include <sys/condvar.h>
  42 #include <sys/sched.h>
  43 #include <sys/signalvar.h>
  44 #include <sys/sleepqueue.h>
  45 #include <sys/resourcevar.h>
  46 #ifdef KTRACE
  47 #include <sys/uio.h>
  48 #include <sys/user.h>
  49 #endif
  50
  51 /*
  52  * A bound below which cv_waiters is valid.  Once cv_waiters reaches this bound,
  53  * cv_signal must manually check the wait queue for threads.
  54  */
  55 #define CV_WAITERS_BOUND        INT_MAX
  56
  57 #define CV_WAITERS_INC(cvp) do {                                        \
  58         if ((cvp)->cv_waiters < CV_WAITERS_BOUND)                       \
  59                 (cvp)->cv_waiters++;                                    \
  60 } while (0)
  61
  62 /*
  63  * Common sanity checks for cv_wait* functions.
  64  */
  65 #define CV_ASSERT(cvp, lock, td) do {                                   \
  66         KASSERT((td) != NULL, ("%s: td NULL", __func__));               \
  67         KASSERT(TD_IS_RUNNING(td), ("%s: not TDS_RUNNING", __func__));  \
  68         KASSERT((cvp) != NULL, ("%s: cvp NULL", __func__));             \
  69         KASSERT((lock) != NULL, ("%s: lock NULL", __func__));           \
  70 } while (0)
  71
  72 /*
  73  * Initialize a condition variable.  Must be called before use.
  74  */
  75 void
  76 cv_init(struct cv *cvp, const char *desc)
  77 {
  78
  79         cvp->cv_description = desc;
  80         cvp->cv_waiters = 0;
  81 }
  82
  83 /*
  84  * Destroy a condition variable.  The condition variable must be re-initialized
  85  * in order to be re-used.
  86  */
  87 void
  88 cv_destroy(struct cv *cvp)
  89 {
  90 #ifdef INVARIANTS
  91         struct sleepqueue *sq;
  92
  93         sleepq_lock(cvp);
  94         sq = sleepq_lookup(cvp);
  95         sleepq_release(cvp);
  96         KASSERT(sq == NULL, ("%s: associated sleep queue non-empty", __func__));
  97 #endif
  98 }
  99
 100 /*
 101  * Wait on a condition variable.  The current thread is placed on the condition
 102  * variable's wait queue and suspended.  A cv_signal or cv_broadcast on the same
 103  * condition variable will resume the thread.  The mutex is released before
 104  * sleeping and will be held on return.  It is recommended that the mutex be
 105  * held when cv_signal or cv_broadcast are called.
 106  */
 107 void
 108 _cv_wait(struct cv *cvp, struct lock_object *lock)
 109 {
 110         WITNESS_SAVE_DECL(lock_witness);
 111 #ifdef KTRACE
 112         char wmesg[WMESGLEN + 1];
 113 #endif
 114         struct lock_class *class;
 115         struct thread *td __ktrace_used;
 116         uintptr_t lock_state;
 117
 118         td = curthread;
 119         CV_ASSERT(cvp, lock, td);
 120         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
 121             "Waiting on \"%s\"", cvp->cv_description);
 122
 123         if (SCHEDULER_STOPPED())
 124                 return;
 125
 126 #ifdef KTRACE
 127         if (KTRPOINT(td, KTR_CSW)) {
 128                 strlcpy(wmesg, cv_wmesg(cvp), sizeof(wmesg));
 129                 ktrcsw(1, 0, wmesg);
 130         } else {
 131                 wmesg[0] = '\0';
 132         }
 133 #endif
 134
 135         class = LOCK_CLASS(lock);
 136         lock_state = 0;
 137         sleepq_lock(cvp);
 138
 139         CV_WAITERS_INC(cvp);
 140         if (lock == &Giant.lock_object)
 141                 mtx_assert(&Giant, MA_OWNED);
 142         DROP_GIANT();
 143
 144         sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0);
 145         if (lock != &Giant.lock_object) {
 146                 if (class->lc_flags & LC_SLEEPABLE)
 147                         sleepq_release(cvp);
 148                 WITNESS_SAVE(lock, lock_witness);
 149                 lock_state = class->lc_unlock(lock);
 150                 if (class->lc_flags & LC_SLEEPABLE)
 151                         sleepq_lock(cvp);
 152         }
 153         sleepq_wait(cvp, 0);
 154
 155 #ifdef KTRACE
 156         if (KTRPOINT(td, KTR_CSW))
 157                 ktrcsw(0, 0, wmesg);
 158 #endif
 159         PICKUP_GIANT();
 160         if (lock != &Giant.lock_object) {
 161                 class->lc_lock(lock, lock_state);
 162                 WITNESS_RESTORE(lock, lock_witness);
 163         }
 164 }
 165
 166 /*
 167  * Wait on a condition variable.  This function differs from cv_wait by
 168  * not acquiring the mutex after condition variable was signaled.
 169  */
 170 void
 171 _cv_wait_unlock(struct cv *cvp, struct lock_object *lock)
 172 {
 173 #ifdef KTRACE
 174         char wmesg[WMESGLEN + 1];
 175 #endif
 176         struct lock_class *class;
 177         struct thread *td __ktrace_used;
 178
 179         td = curthread;
 180         CV_ASSERT(cvp, lock, td);
 181         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
 182             "Waiting on \"%s\"", cvp->cv_description);
 183         KASSERT(lock != &Giant.lock_object,
 184             ("cv_wait_unlock cannot be used with Giant"));
 185         class = LOCK_CLASS(lock);
 186
 187         if (SCHEDULER_STOPPED()) {
 188                 class->lc_unlock(lock);
 189                 return;
 190         }
 191
 192 #ifdef KTRACE
 193         if (KTRPOINT(td, KTR_CSW)) {
 194                 strlcpy(wmesg, cv_wmesg(cvp), sizeof(wmesg));
 195                 ktrcsw(1, 0, wmesg);
 196         } else {
 197                 wmesg[0] = '\0';
 198         }
 199 #endif
 200
 201         sleepq_lock(cvp);
 202
 203         CV_WAITERS_INC(cvp);
 204         DROP_GIANT();
 205
 206         sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0);
 207         if (class->lc_flags & LC_SLEEPABLE)
 208                 sleepq_release(cvp);
 209         class->lc_unlock(lock);
 210         if (class->lc_flags & LC_SLEEPABLE)
 211                 sleepq_lock(cvp);
 212         sleepq_wait(cvp, 0);
 213
 214 #ifdef KTRACE
 215         if (KTRPOINT(td, KTR_CSW))
 216                 ktrcsw(0, 0, wmesg);
 217 #endif
 218         PICKUP_GIANT();
 219 }
 220
 221 /*
 222  * Wait on a condition variable, allowing interruption by signals.  Return 0 if
 223  * the thread was resumed with cv_signal or cv_broadcast, EINTR or ERESTART if
 224  * a signal was caught.  If ERESTART is returned the system call should be
 225  * restarted if possible.
 226  */
 227 int
 228 _cv_wait_sig(struct cv *cvp, struct lock_object *lock)
 229 {
 230         WITNESS_SAVE_DECL(lock_witness);
 231 #ifdef KTRACE
 232         char wmesg[WMESGLEN + 1];
 233 #endif
 234         struct lock_class *class;
 235         struct thread *td __ktrace_used;
 236         uintptr_t lock_state;
 237         int rval;
 238
 239         td = curthread;
 240         CV_ASSERT(cvp, lock, td);
 241         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
 242             "Waiting on \"%s\"", cvp->cv_description);
 243
 244         if (SCHEDULER_STOPPED())
 245                 return (0);
 246
 247 #ifdef KTRACE
 248         if (KTRPOINT(td, KTR_CSW)) {
 249                 strlcpy(wmesg, cv_wmesg(cvp), sizeof(wmesg));
 250                 ktrcsw(1, 0, wmesg);
 251         } else {
 252                 wmesg[0] = '\0';
 253         }
 254 #endif
 255
 256         class = LOCK_CLASS(lock);
 257         lock_state = 0;
 258         sleepq_lock(cvp);
 259
 260         CV_WAITERS_INC(cvp);
 261         if (lock == &Giant.lock_object)
 262                 mtx_assert(&Giant, MA_OWNED);
 263         DROP_GIANT();
 264
 265         sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR |
 266             SLEEPQ_INTERRUPTIBLE, 0);
 267         if (lock != &Giant.lock_object) {
 268                 if (class->lc_flags & LC_SLEEPABLE)
 269                         sleepq_release(cvp);
 270                 WITNESS_SAVE(lock, lock_witness);
 271                 lock_state = class->lc_unlock(lock);
 272                 if (class->lc_flags & LC_SLEEPABLE)
 273                         sleepq_lock(cvp);
 274         }
 275         rval = sleepq_wait_sig(cvp, 0);
 276
 277 #ifdef KTRACE
 278         if (KTRPOINT(td, KTR_CSW))
 279                 ktrcsw(0, 0, wmesg);
 280 #endif
 281         PICKUP_GIANT();
 282         if (lock != &Giant.lock_object) {
 283                 class->lc_lock(lock, lock_state);
 284                 WITNESS_RESTORE(lock, lock_witness);
 285         }
 286
 287         return (rval);
 288 }
 289
 290 /*
 291  * Wait on a condition variable for (at most) the value specified in sbt
 292  * argument. Returns 0 if the process was resumed by cv_signal or cv_broadcast,
 293  * EWOULDBLOCK if the timeout expires.
 294  */
 295 int
 296 _cv_timedwait_sbt(struct cv *cvp, struct lock_object *lock, sbintime_t sbt,
 297     sbintime_t pr, int flags)
 298 {
 299         WITNESS_SAVE_DECL(lock_witness);
 300 #ifdef KTRACE
 301         char wmesg[WMESGLEN + 1];
 302 #endif
 303         struct lock_class *class;
 304         struct thread *td __ktrace_used;
 305         int lock_state, rval;
 306
 307         td = curthread;
 308         CV_ASSERT(cvp, lock, td);
 309         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
 310             "Waiting on \"%s\"", cvp->cv_description);
 311
 312         if (SCHEDULER_STOPPED())
 313                 return (0);
 314
 315 #ifdef KTRACE
 316         if (KTRPOINT(td, KTR_CSW)) {
 317                 strlcpy(wmesg, cv_wmesg(cvp), sizeof(wmesg));
 318                 ktrcsw(1, 0, wmesg);
 319         } else {
 320                 wmesg[0] = '\0';
 321         }
 322 #endif
 323
 324         class = LOCK_CLASS(lock);
 325         lock_state = 0;
 326         sleepq_lock(cvp);
 327
 328         CV_WAITERS_INC(cvp);
 329         if (lock == &Giant.lock_object)
 330                 mtx_assert(&Giant, MA_OWNED);
 331         DROP_GIANT();
 332
 333         sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0);
 334         sleepq_set_timeout_sbt(cvp, sbt, pr, flags);
 335         if (lock != &Giant.lock_object) {
 336                 if (class->lc_flags & LC_SLEEPABLE)
 337                         sleepq_release(cvp);
 338                 WITNESS_SAVE(lock, lock_witness);
 339                 lock_state = class->lc_unlock(lock);
 340                 if (class->lc_flags & LC_SLEEPABLE)
 341                         sleepq_lock(cvp);
 342         }
 343         rval = sleepq_timedwait(cvp, 0);
 344
 345 #ifdef KTRACE
 346         if (KTRPOINT(td, KTR_CSW))
 347                 ktrcsw(0, 0, wmesg);
 348 #endif
 349         PICKUP_GIANT();
 350         if (lock != &Giant.lock_object) {
 351                 class->lc_lock(lock, lock_state);
 352                 WITNESS_RESTORE(lock, lock_witness);
 353         }
 354
 355         return (rval);
 356 }
 357
 358 /*
 359  * Wait on a condition variable for (at most) the value specified in sbt
 360  * argument, allowing interruption by signals.
 361  * Returns 0 if the thread was resumed by cv_signal or cv_broadcast,
 362  * EWOULDBLOCK if the timeout expires, and EINTR or ERESTART if a signal
 363  * was caught.
 364  */
 365 int
 366 _cv_timedwait_sig_sbt(struct cv *cvp, struct lock_object *lock,
 367     sbintime_t sbt, sbintime_t pr, int flags)
 368 {
 369         WITNESS_SAVE_DECL(lock_witness);
 370 #ifdef KTRACE
 371         char wmesg[WMESGLEN + 1];
 372 #endif
 373         struct lock_class *class;
 374         struct thread *td __ktrace_used;
 375         int lock_state, rval;
 376
 377         td = curthread;
 378         CV_ASSERT(cvp, lock, td);
 379         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
 380             "Waiting on \"%s\"", cvp->cv_description);
 381
 382         if (SCHEDULER_STOPPED())
 383                 return (0);
 384
 385 #ifdef KTRACE
 386         if (KTRPOINT(td, KTR_CSW)) {
 387                 strlcpy(wmesg, cv_wmesg(cvp), sizeof(wmesg));
 388                 ktrcsw(1, 0, wmesg);
 389         } else {
 390                 wmesg[0] = '\0';
 391         }
 392 #endif
 393
 394         class = LOCK_CLASS(lock);
 395         lock_state = 0;
 396         sleepq_lock(cvp);
 397
 398         CV_WAITERS_INC(cvp);
 399         if (lock == &Giant.lock_object)
 400                 mtx_assert(&Giant, MA_OWNED);
 401         DROP_GIANT();
 402
 403         sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR |
 404             SLEEPQ_INTERRUPTIBLE, 0);
 405         sleepq_set_timeout_sbt(cvp, sbt, pr, flags);
 406         if (lock != &Giant.lock_object) {
 407                 if (class->lc_flags & LC_SLEEPABLE)
 408                         sleepq_release(cvp);
 409                 WITNESS_SAVE(lock, lock_witness);
 410                 lock_state = class->lc_unlock(lock);
 411                 if (class->lc_flags & LC_SLEEPABLE)
 412                         sleepq_lock(cvp);
 413         }
 414         rval = sleepq_timedwait_sig(cvp, 0);
 415
 416 #ifdef KTRACE
 417         if (KTRPOINT(td, KTR_CSW))
 418                 ktrcsw(0, 0, wmesg);
 419 #endif
 420         PICKUP_GIANT();
 421         if (lock != &Giant.lock_object) {
 422                 class->lc_lock(lock, lock_state);
 423                 WITNESS_RESTORE(lock, lock_witness);
 424         }
 425
 426         return (rval);
 427 }
 428
 429 /*
 430  * Signal a condition variable, wakes up one waiting thread.  Will also wakeup
 431  * the swapper if the process is not in memory, so that it can bring the
 432  * sleeping process in.  Note that this may also result in additional threads
 433  * being made runnable.  Should be called with the same mutex as was passed to
 434  * cv_wait held.
 435  */
 436 void
 437 cv_signal(struct cv *cvp)
 438 {
 439
 440         if (cvp->cv_waiters == 0)
 441                 return;
 442         sleepq_lock(cvp);
 443         if (cvp->cv_waiters == 0) {
 444                 sleepq_release(cvp);
 445                 return;
 446         }
 447         if (cvp->cv_waiters == CV_WAITERS_BOUND && sleepq_lookup(cvp) == NULL) {
 448                 cvp->cv_waiters = 0;
 449                 sleepq_release(cvp);
 450         } else {
 451                 if (cvp->cv_waiters < CV_WAITERS_BOUND)
 452                         cvp->cv_waiters--;
 453                 if (sleepq_signal(cvp, SLEEPQ_CONDVAR | SLEEPQ_DROP, 0, 0))
 454                         kick_proc0();
 455         }
 456 }
 457
 458 /*
 459  * Broadcast a signal to a condition variable.  Wakes up all waiting threads.
 460  * Should be called with the same mutex as was passed to cv_wait held.
 461  */
 462 void
 463 cv_broadcastpri(struct cv *cvp, int pri)
 464 {
 465         int wakeup_swapper;
 466
 467         if (cvp->cv_waiters == 0)
 468                 return;
 469         /*
 470          * XXX sleepq_broadcast pri argument changed from -1 meaning
 471          * no pri to 0 meaning no pri.
 472          */
 473         wakeup_swapper = 0;
 474         if (pri == -1)
 475                 pri = 0;
 476         sleepq_lock(cvp);
 477         if (cvp->cv_waiters > 0) {
 478                 cvp->cv_waiters = 0;
 479                 wakeup_swapper = sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri, 0);
 480         }
 481         sleepq_release(cvp);
 482         if (wakeup_swapper)
 483                 kick_proc0();
 484 }