2 * Copyright (c) 2000 Jake Burkholder <jake@freebsd.org>.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
30 #include "opt_ktrace.h"
32 #include <sys/param.h>
33 #include <sys/systm.h>
35 #include <sys/mutex.h>
37 #include <sys/kernel.h>
39 #include <sys/condvar.h>
40 #include <sys/sched.h>
41 #include <sys/signalvar.h>
42 #include <sys/resourcevar.h>
45 #include <sys/ktrace.h>
49 * Common sanity checks for cv_wait* functions.
51 #define CV_ASSERT(cvp, mp, td) do { \
52 KASSERT((td) != NULL, ("%s: curthread NULL", __func__)); \
53 KASSERT(TD_IS_RUNNING(td), ("%s: not TDS_RUNNING", __func__)); \
54 KASSERT((cvp) != NULL, ("%s: cvp NULL", __func__)); \
55 KASSERT((mp) != NULL, ("%s: mp NULL", __func__)); \
56 mtx_assert((mp), MA_OWNED | MA_NOTRECURSED); \
60 #define CV_WAIT_VALIDATE(cvp, mp) do { \
61 if (TAILQ_EMPTY(&(cvp)->cv_waitq)) { \
63 (cvp)->cv_mtx = (mp); \
66 * Other waiter; assert that we're using the \
69 KASSERT((cvp)->cv_mtx == (mp), \
70 ("%s: Multiple mutexes", __func__)); \
74 #define CV_SIGNAL_VALIDATE(cvp) do { \
75 if (!TAILQ_EMPTY(&(cvp)->cv_waitq)) { \
76 KASSERT(mtx_owned((cvp)->cv_mtx), \
77 ("%s: Mutex not owned", __func__)); \
82 #define CV_WAIT_VALIDATE(cvp, mp)
83 #define CV_SIGNAL_VALIDATE(cvp)
86 static void cv_timedwait_end(void *arg);
89 * Initialize a condition variable. Must be called before use.
92 cv_init(struct cv *cvp, const char *desc)
95 TAILQ_INIT(&cvp->cv_waitq);
97 cvp->cv_description = desc;
101 * Destroy a condition variable. The condition variable must be re-initialized
102 * in order to be re-used.
105 cv_destroy(struct cv *cvp)
108 KASSERT(cv_waitq_empty(cvp), ("%s: cv_waitq non-empty", __func__));
112 * Common code for cv_wait* functions. All require sched_lock.
119 cv_switch(struct thread *td)
122 td->td_proc->p_stats->p_ru.ru_nvcsw++;
124 CTR3(KTR_PROC, "cv_switch: resume thread %p (pid %d, %s)", td,
125 td->td_proc->p_pid, td->td_proc->p_comm);
129 * Switch context, catching signals.
132 cv_switch_catch(struct thread *td)
138 * We put ourselves on the sleep queue and start our timeout before
139 * calling cursig, as we could stop there, and a wakeup or a SIGCONT (or
140 * both) could occur while we were stopped. A SIGCONT would cause us to
141 * be marked as TDS_SLP without resuming us, thus we must be ready for
142 * sleep when cursig is called. If the wakeup happens while we're
143 * stopped, td->td_wchan will be 0 upon return from cursig,
144 * and TD_ON_SLEEPQ() will return false.
146 td->td_flags |= TDF_SINTR;
147 mtx_unlock_spin(&sched_lock);
150 mtx_lock(&p->p_sigacts->ps_mtx);
152 mtx_unlock(&p->p_sigacts->ps_mtx);
153 if (thread_suspend_check(1))
155 mtx_lock_spin(&sched_lock);
158 if (TD_ON_SLEEPQ(td))
161 } else if (TD_ON_SLEEPQ(td)) {
164 td->td_flags &= ~TDF_SINTR;
170 * Add a thread to the wait queue of a condition variable.
173 cv_waitq_add(struct cv *cvp, struct thread *td)
176 td->td_flags |= TDF_CVWAITQ;
177 TD_SET_ON_SLEEPQ(td);
179 td->td_wmesg = cvp->cv_description;
180 CTR3(KTR_PROC, "cv_waitq_add: thread %p (pid %d, %s)", td,
181 td->td_proc->p_pid, td->td_proc->p_comm);
182 TAILQ_INSERT_TAIL(&cvp->cv_waitq, td, td_slpq);
183 sched_sleep(td, td->td_priority);
187 * Wait on a condition variable. The current thread is placed on the condition
188 * variable's wait queue and suspended. A cv_signal or cv_broadcast on the same
189 * condition variable will resume the thread. The mutex is released before
190 * sleeping and will be held on return. It is recommended that the mutex be
191 * held when cv_signal or cv_broadcast are called.
194 cv_wait(struct cv *cvp, struct mtx *mp)
197 WITNESS_SAVE_DECL(mp);
201 if (KTRPOINT(td, KTR_CSW))
204 CV_ASSERT(cvp, mp, td);
205 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, &mp->mtx_object,
206 "Waiting on \"%s\"", cvp->cv_description);
207 WITNESS_SAVE(&mp->mtx_object, mp);
211 * During autoconfiguration, just give interrupts
212 * a chance, then just return. Don't run any other
213 * thread or panic below, in case this is the idle
214 * process and already asleep.
219 mtx_lock_spin(&sched_lock);
221 CV_WAIT_VALIDATE(cvp, mp);
226 cv_waitq_add(cvp, td);
229 mtx_unlock_spin(&sched_lock);
231 if (KTRPOINT(td, KTR_CSW))
236 WITNESS_RESTORE(&mp->mtx_object, mp);
240 * Wait on a condition variable, allowing interruption by signals. Return 0 if
241 * the thread was resumed with cv_signal or cv_broadcast, EINTR or ERESTART if
242 * a signal was caught. If ERESTART is returned the system call should be
243 * restarted if possible.
246 cv_wait_sig(struct cv *cvp, struct mtx *mp)
252 WITNESS_SAVE_DECL(mp);
258 if (KTRPOINT(td, KTR_CSW))
261 CV_ASSERT(cvp, mp, td);
262 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, &mp->mtx_object,
263 "Waiting on \"%s\"", cvp->cv_description);
264 WITNESS_SAVE(&mp->mtx_object, mp);
266 if (cold || panicstr) {
268 * After a panic, or during autoconfiguration, just give
269 * interrupts a chance, then just return; don't run any other
270 * procs or panic below, in case this is the idle process and
276 mtx_lock_spin(&sched_lock);
278 CV_WAIT_VALIDATE(cvp, mp);
283 cv_waitq_add(cvp, td);
284 sig = cv_switch_catch(td);
286 mtx_unlock_spin(&sched_lock);
289 mtx_lock(&p->p_sigacts->ps_mtx);
291 sig = cursig(td); /* XXXKSE */
292 if (sig == 0 && td->td_flags & TDF_INTERRUPT)
293 rval = td->td_intrval;
296 if (SIGISMEMBER(p->p_sigacts->ps_sigintr, sig))
301 mtx_unlock(&p->p_sigacts->ps_mtx);
302 if (p->p_flag & P_WEXIT)
307 if (KTRPOINT(td, KTR_CSW))
312 WITNESS_RESTORE(&mp->mtx_object, mp);
318 * Wait on a condition variable for at most timo/hz seconds. Returns 0 if the
319 * process was resumed by cv_signal or cv_broadcast, EWOULDBLOCK if the timeout
323 cv_timedwait(struct cv *cvp, struct mtx *mp, int timo)
327 WITNESS_SAVE_DECL(mp);
332 if (KTRPOINT(td, KTR_CSW))
335 CV_ASSERT(cvp, mp, td);
336 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, &mp->mtx_object,
337 "Waiting on \"%s\"", cvp->cv_description);
338 WITNESS_SAVE(&mp->mtx_object, mp);
340 if (cold || panicstr) {
342 * After a panic, or during autoconfiguration, just give
343 * interrupts a chance, then just return; don't run any other
344 * thread or panic below, in case this is the idle process and
350 mtx_lock_spin(&sched_lock);
352 CV_WAIT_VALIDATE(cvp, mp);
357 cv_waitq_add(cvp, td);
358 callout_reset(&td->td_slpcallout, timo, cv_timedwait_end, td);
361 if (td->td_flags & TDF_TIMEOUT) {
362 td->td_flags &= ~TDF_TIMEOUT;
364 } else if (td->td_flags & TDF_TIMOFAIL)
365 td->td_flags &= ~TDF_TIMOFAIL;
366 else if (callout_stop(&td->td_slpcallout) == 0) {
368 * Work around race with cv_timedwait_end similar to that
369 * between msleep and endtsleep.
373 td->td_proc->p_stats->p_ru.ru_nivcsw++;
375 td->td_flags &= ~TDF_TIMOFAIL;
378 mtx_unlock_spin(&sched_lock);
380 if (KTRPOINT(td, KTR_CSW))
385 WITNESS_RESTORE(&mp->mtx_object, mp);
391 * Wait on a condition variable for at most timo/hz seconds, allowing
392 * interruption by signals. Returns 0 if the thread was resumed by cv_signal
393 * or cv_broadcast, EWOULDBLOCK if the timeout expires, and EINTR or ERESTART if
394 * a signal was caught.
397 cv_timedwait_sig(struct cv *cvp, struct mtx *mp, int timo)
403 WITNESS_SAVE_DECL(mp);
409 if (KTRPOINT(td, KTR_CSW))
412 CV_ASSERT(cvp, mp, td);
413 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, &mp->mtx_object,
414 "Waiting on \"%s\"", cvp->cv_description);
415 WITNESS_SAVE(&mp->mtx_object, mp);
417 if (cold || panicstr) {
419 * After a panic, or during autoconfiguration, just give
420 * interrupts a chance, then just return; don't run any other
421 * thread or panic below, in case this is the idle process and
427 mtx_lock_spin(&sched_lock);
429 CV_WAIT_VALIDATE(cvp, mp);
434 cv_waitq_add(cvp, td);
435 callout_reset(&td->td_slpcallout, timo, cv_timedwait_end, td);
436 sig = cv_switch_catch(td);
438 if (td->td_flags & TDF_TIMEOUT) {
439 td->td_flags &= ~TDF_TIMEOUT;
441 } else if (td->td_flags & TDF_TIMOFAIL)
442 td->td_flags &= ~TDF_TIMOFAIL;
443 else if (callout_stop(&td->td_slpcallout) == 0) {
445 * Work around race with cv_timedwait_end similar to that
446 * between msleep and endtsleep.
450 td->td_proc->p_stats->p_ru.ru_nivcsw++;
452 td->td_flags &= ~TDF_TIMOFAIL;
454 mtx_unlock_spin(&sched_lock);
457 mtx_lock(&p->p_sigacts->ps_mtx);
460 if (sig == 0 && td->td_flags & TDF_INTERRUPT)
461 rval = td->td_intrval;
464 if (SIGISMEMBER(p->p_sigacts->ps_sigintr, sig))
469 mtx_unlock(&p->p_sigacts->ps_mtx);
470 if (p->p_flag & P_WEXIT)
475 if (KTRPOINT(td, KTR_CSW))
480 WITNESS_RESTORE(&mp->mtx_object, mp);
486 * Common code for signal and broadcast. Assumes waitq is not empty. Must be
487 * called with sched_lock held.
490 cv_wakeup(struct cv *cvp)
494 mtx_assert(&sched_lock, MA_OWNED);
495 td = TAILQ_FIRST(&cvp->cv_waitq);
496 KASSERT(td->td_wchan == cvp, ("%s: bogus wchan", __func__));
497 KASSERT(td->td_flags & TDF_CVWAITQ, ("%s: not on waitq", __func__));
504 * Signal a condition variable, wakes up one waiting thread. Will also wakeup
505 * the swapper if the process is not in memory, so that it can bring the
506 * sleeping process in. Note that this may also result in additional threads
507 * being made runnable. Should be called with the same mutex as was passed to
511 cv_signal(struct cv *cvp)
514 KASSERT(cvp != NULL, ("%s: cvp NULL", __func__));
515 mtx_lock_spin(&sched_lock);
516 if (!TAILQ_EMPTY(&cvp->cv_waitq)) {
517 CV_SIGNAL_VALIDATE(cvp);
520 mtx_unlock_spin(&sched_lock);
524 * Broadcast a signal to a condition variable. Wakes up all waiting threads.
525 * Should be called with the same mutex as was passed to cv_wait held.
528 cv_broadcast(struct cv *cvp)
531 KASSERT(cvp != NULL, ("%s: cvp NULL", __func__));
532 mtx_lock_spin(&sched_lock);
533 CV_SIGNAL_VALIDATE(cvp);
534 while (!TAILQ_EMPTY(&cvp->cv_waitq))
536 mtx_unlock_spin(&sched_lock);
540 * Remove a thread from the wait queue of its condition variable. This may be
544 cv_waitq_remove(struct thread *td)
548 mtx_assert(&sched_lock, MA_OWNED);
549 if ((cvp = td->td_wchan) != NULL && td->td_flags & TDF_CVWAITQ) {
550 TAILQ_REMOVE(&cvp->cv_waitq, td, td_slpq);
551 td->td_flags &= ~TDF_CVWAITQ;
553 TD_CLR_ON_SLEEPQ(td);
558 * Timeout function for cv_timedwait. Put the thread on the runqueue and set
562 cv_timedwait_end(void *arg)
567 CTR3(KTR_PROC, "cv_timedwait_end: thread %p (pid %d, %s)",
568 td, td->td_proc->p_pid, td->td_proc->p_comm);
569 mtx_lock_spin(&sched_lock);
570 if (TD_ON_SLEEPQ(td)) {
572 td->td_flags |= TDF_TIMEOUT;
574 td->td_flags |= TDF_TIMOFAIL;
578 mtx_unlock_spin(&sched_lock);
582 * For now only abort interruptable waits.
583 * The others will have to either complete on their own or have a timeout.
586 cv_abort(struct thread *td)
589 CTR3(KTR_PROC, "cv_abort: thread %p (pid %d, %s)", td,
590 td->td_proc->p_pid, td->td_proc->p_comm);
591 mtx_lock_spin(&sched_lock);
592 if ((td->td_flags & (TDF_SINTR|TDF_TIMEOUT)) == TDF_SINTR) {
593 if (TD_ON_SLEEPQ(td)) {
599 mtx_unlock_spin(&sched_lock);