sys/kern/kern_thread.c

   1 /*-
   2  * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>.
   3  *  All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice(s), this list of conditions and the following disclaimer as
  10  *    the first lines of this file unmodified other than the possible
  11  *    addition of one or more copyright notices.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice(s), this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
  17  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19  * DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
  20  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  21  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  22  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  23  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
  26  * DAMAGE.
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include <sys/param.h>
  33 #include <sys/systm.h>
  34 #include <sys/kernel.h>
  35 #include <sys/lock.h>
  36 #include <sys/mutex.h>
  37 #include <sys/proc.h>
  38 #include <sys/resourcevar.h>
  39 #include <sys/smp.h>
  40 #include <sys/sysctl.h>
  41 #include <sys/sched.h>
  42 #include <sys/sleepqueue.h>
  43 #include <sys/turnstile.h>
  44 #include <sys/ktr.h>
  45 #include <sys/umtx.h>
  46
  47 #include <security/audit/audit.h>
  48
  49 #include <vm/vm.h>
  50 #include <vm/vm_extern.h>
  51 #include <vm/uma.h>
  52
  53 /*
  54  * thread related storage.
  55  */
  56 static uma_zone_t thread_zone;
  57
  58 SYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW, 0, "thread allocation");
  59
  60 int max_threads_per_proc = 1500;
  61 SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_per_proc, CTLFLAG_RW,
  62         &max_threads_per_proc, 0, "Limit on threads per proc");
  63
  64 int max_threads_hits;
  65 SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_hits, CTLFLAG_RD,
  66         &max_threads_hits, 0, "");
  67
  68 #ifdef KSE
  69 int virtual_cpu;
  70
  71 #endif
  72 TAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads);
  73 struct mtx zombie_lock;
  74 MTX_SYSINIT(zombie_lock, &zombie_lock, "zombie lock", MTX_SPIN);
  75
  76 #ifdef KSE
  77 static int
  78 sysctl_kse_virtual_cpu(SYSCTL_HANDLER_ARGS)
  79 {
  80         int error, new_val;
  81         int def_val;
  82
  83         def_val = mp_ncpus;
  84         if (virtual_cpu == 0)
  85                 new_val = def_val;
  86         else
  87                 new_val = virtual_cpu;
  88         error = sysctl_handle_int(oidp, &new_val, 0, req);
  89         if (error != 0 || req->newptr == NULL)
  90                 return (error);
  91         if (new_val < 0)
  92                 return (EINVAL);
  93         virtual_cpu = new_val;
  94         return (0);
  95 }
  96
  97 /* DEBUG ONLY */
  98 SYSCTL_PROC(_kern_threads, OID_AUTO, virtual_cpu, CTLTYPE_INT|CTLFLAG_RW,
  99         0, sizeof(virtual_cpu), sysctl_kse_virtual_cpu, "I",
 100         "debug virtual cpus");
 101 #endif
 102
 103 struct mtx tid_lock;
 104 static struct unrhdr *tid_unrhdr;
 105
 106 /*
 107  * Prepare a thread for use.
 108  */
 109 static int
 110 thread_ctor(void *mem, int size, void *arg, int flags)
 111 {
 112         struct thread   *td;
 113
 114         td = (struct thread *)mem;
 115         td->td_state = TDS_INACTIVE;
 116         td->td_oncpu = NOCPU;
 117
 118         td->td_tid = alloc_unr(tid_unrhdr);
 119         td->td_syscalls = 0;
 120
 121         /*
 122          * Note that td_critnest begins life as 1 because the thread is not
 123          * running and is thereby implicitly waiting to be on the receiving
 124          * end of a context switch.
 125          */
 126         td->td_critnest = 1;
 127
 128 #ifdef AUDIT
 129         audit_thread_alloc(td);
 130 #endif
 131         umtx_thread_alloc(td);
 132         return (0);
 133 }
 134
 135 /*
 136  * Reclaim a thread after use.
 137  */
 138 static void
 139 thread_dtor(void *mem, int size, void *arg)
 140 {
 141         struct thread *td;
 142
 143         td = (struct thread *)mem;
 144
 145 #ifdef INVARIANTS
 146         /* Verify that this thread is in a safe state to free. */
 147         switch (td->td_state) {
 148         case TDS_INHIBITED:
 149         case TDS_RUNNING:
 150         case TDS_CAN_RUN:
 151         case TDS_RUNQ:
 152                 /*
 153                  * We must never unlink a thread that is in one of
 154                  * these states, because it is currently active.
 155                  */
 156                 panic("bad state for thread unlinking");
 157                 /* NOTREACHED */
 158         case TDS_INACTIVE:
 159                 break;
 160         default:
 161                 panic("bad thread state");
 162                 /* NOTREACHED */
 163         }
 164 #endif
 165 #ifdef AUDIT
 166         audit_thread_free(td);
 167 #endif
 168         free_unr(tid_unrhdr, td->td_tid);
 169         sched_newthread(td);
 170 }
 171
 172 /*
 173  * Initialize type-stable parts of a thread (when newly created).
 174  */
 175 static int
 176 thread_init(void *mem, int size, int flags)
 177 {
 178         struct thread *td;
 179
 180         td = (struct thread *)mem;
 181
 182         vm_thread_new(td, 0);
 183         cpu_thread_setup(td);
 184         td->td_sleepqueue = sleepq_alloc();
 185         td->td_turnstile = turnstile_alloc();
 186         td->td_sched = (struct td_sched *)&td[1];
 187         sched_newthread(td);
 188         umtx_thread_init(td);
 189         return (0);
 190 }
 191
 192 /*
 193  * Tear down type-stable parts of a thread (just before being discarded).
 194  */
 195 static void
 196 thread_fini(void *mem, int size)
 197 {
 198         struct thread *td;
 199
 200         td = (struct thread *)mem;
 201         turnstile_free(td->td_turnstile);
 202         sleepq_free(td->td_sleepqueue);
 203         umtx_thread_fini(td);
 204         vm_thread_dispose(td);
 205 }
 206
 207 /*
 208  * For a newly created process,
 209  * link up all the structures and its initial threads etc.
 210  * called from:
 211  * {arch}/{arch}/machdep.c   ia64_init(), init386() etc.
 212  * proc_dtor() (should go away)
 213  * proc_init()
 214  */
 215 void
 216 proc_linkup(struct proc *p, struct thread *td)
 217 {
 218
 219         TAILQ_INIT(&p->p_threads);           /* all threads in proc */
 220         TAILQ_INIT(&p->p_upcalls);           /* upcall list */
 221         sigqueue_init(&p->p_sigqueue, p);
 222         p->p_ksi = ksiginfo_alloc(1);
 223         if (p->p_ksi != NULL) {
 224                 /* XXX p_ksi may be null if ksiginfo zone is not ready */
 225                 p->p_ksi->ksi_flags = KSI_EXT | KSI_INS;
 226         }
 227         LIST_INIT(&p->p_mqnotifier);
 228         p->p_numthreads = 0;
 229         thread_link(td, p);
 230 }
 231
 232 /*
 233  * Initialize global thread allocation resources.
 234  */
 235 void
 236 threadinit(void)
 237 {
 238
 239         mtx_init(&tid_lock, "TID lock", NULL, MTX_DEF);
 240         tid_unrhdr = new_unrhdr(PID_MAX + 1, INT_MAX, &tid_lock);
 241
 242         thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(),
 243             thread_ctor, thread_dtor, thread_init, thread_fini,
 244             16 - 1, 0);
 245 #ifdef KSE
 246         kseinit();      /* set up kse specific stuff  e.g. upcall zone*/
 247 #endif
 248 }
 249
 250 /*
 251  * Stash an embarasingly extra thread into the zombie thread queue.
 252  * Use the slpq as that must be unused by now.
 253  */
 254 void
 255 thread_stash(struct thread *td)
 256 {
 257         mtx_lock_spin(&zombie_lock);
 258         TAILQ_INSERT_HEAD(&zombie_threads, td, td_slpq);
 259         mtx_unlock_spin(&zombie_lock);
 260 }
 261
 262 /*
 263  * Reap zombie kse resource.
 264  */
 265 void
 266 thread_reap(void)
 267 {
 268         struct thread *td_first, *td_next;
 269
 270         /*
 271          * Don't even bother to lock if none at this instant,
 272          * we really don't care about the next instant..
 273          */
 274         if (!TAILQ_EMPTY(&zombie_threads)) {
 275                 mtx_lock_spin(&zombie_lock);
 276                 td_first = TAILQ_FIRST(&zombie_threads);
 277                 if (td_first)
 278                         TAILQ_INIT(&zombie_threads);
 279                 mtx_unlock_spin(&zombie_lock);
 280                 while (td_first) {
 281                         td_next = TAILQ_NEXT(td_first, td_slpq);
 282                         if (td_first->td_ucred)
 283                                 crfree(td_first->td_ucred);
 284                         thread_free(td_first);
 285                         td_first = td_next;
 286                 }
 287         }
 288 }
 289
 290 /*
 291  * Allocate a thread.
 292  */
 293 struct thread *
 294 thread_alloc(void)
 295 {
 296
 297         thread_reap(); /* check if any zombies to get */
 298         return (uma_zalloc(thread_zone, M_WAITOK));
 299 }
 300
 301
 302 /*
 303  * Deallocate a thread.
 304  */
 305 void
 306 thread_free(struct thread *td)
 307 {
 308
 309         cpu_thread_clean(td);
 310         uma_zfree(thread_zone, td);
 311 }
 312
 313 /*
 314  * Discard the current thread and exit from its context.
 315  * Always called with scheduler locked.
 316  *
 317  * Because we can't free a thread while we're operating under its context,
 318  * push the current thread into our CPU's deadthread holder. This means
 319  * we needn't worry about someone else grabbing our context before we
 320  * do a cpu_throw().  This may not be needed now as we are under schedlock.
 321  * Maybe we can just do a thread_stash() as thr_exit1 does.
 322  */
 323 /*  XXX
 324  * libthr expects its thread exit to return for the last
 325  * thread, meaning that the program is back to non-threaded
 326  * mode I guess. Because we do this (cpu_throw) unconditionally
 327  * here, they have their own version of it. (thr_exit1())
 328  * that doesn't do it all if this was the last thread.
 329  * It is also called from thread_suspend_check().
 330  * Of course in the end, they end up coming here through exit1
 331  * anyhow..  After fixing 'thr' to play by the rules we should be able
 332  * to merge these two functions together.
 333  *
 334  * called from:
 335  * exit1()
 336  * kse_exit()
 337  * thr_exit()
 338  * ifdef KSE
 339  * thread_user_enter()
 340  * thread_userret()
 341  * endif
 342  * thread_suspend_check()
 343  */
 344 void
 345 thread_exit(void)
 346 {
 347         uint64_t new_switchtime;
 348         struct thread *td;
 349         struct thread *td2;
 350         struct proc *p;
 351
 352         td = curthread;
 353         p = td->td_proc;
 354
 355         PROC_SLOCK_ASSERT(p, MA_OWNED);
 356         mtx_assert(&Giant, MA_NOTOWNED);
 357
 358         PROC_LOCK_ASSERT(p, MA_OWNED);
 359         KASSERT(p != NULL, ("thread exiting without a process"));
 360         CTR3(KTR_PROC, "thread_exit: thread %p (pid %ld, %s)", td,
 361             (long)p->p_pid, p->p_comm);
 362         KASSERT(TAILQ_EMPTY(&td->td_sigqueue.sq_list), ("signal pending"));
 363
 364 #ifdef AUDIT
 365         AUDIT_SYSCALL_EXIT(0, td);
 366 #endif
 367
 368 #ifdef KSE
 369         if (td->td_standin != NULL) {
 370                 /*
 371                  * Note that we don't need to free the cred here as it
 372                  * is done in thread_reap().
 373                  */
 374                 thread_stash(td->td_standin);
 375                 td->td_standin = NULL;
 376         }
 377 #endif
 378
 379         umtx_thread_exit(td);
 380
 381         /*
 382          * drop FPU & debug register state storage, or any other
 383          * architecture specific resources that
 384          * would not be on a new untouched process.
 385          */
 386         cpu_thread_exit(td);    /* XXXSMP */
 387
 388         /* Do the same timestamp bookkeeping that mi_switch() would do. */
 389         new_switchtime = cpu_ticks();
 390         p->p_rux.rux_runtime += (new_switchtime - PCPU_GET(switchtime));
 391         PCPU_SET(switchtime, new_switchtime);
 392         PCPU_SET(switchticks, ticks);
 393         PCPU_INC(cnt.v_swtch);
 394         /* Add the child usage to our own when the final thread exits. */
 395         if (p->p_numthreads == 1)
 396                 ruadd(p->p_ru, &p->p_rux, &p->p_stats->p_cru, &p->p_crux);
 397         /*
 398          * The last thread is left attached to the process
 399          * So that the whole bundle gets recycled. Skip
 400          * all this stuff if we never had threads.
 401          * EXIT clears all sign of other threads when
 402          * it goes to single threading, so the last thread always
 403          * takes the short path.
 404          */
 405         if (p->p_flag & P_HADTHREADS) {
 406                 if (p->p_numthreads > 1) {
 407                         thread_lock(td);
 408 #ifdef KSE
 409                         kse_unlink(td);
 410 #else
 411                         thread_unlink(td);
 412 #endif
 413                         thread_unlock(td);
 414                         /* Impart our resource usage on another thread */
 415                         td2 = FIRST_THREAD_IN_PROC(p);
 416                         rucollect(&td2->td_ru, &td->td_ru);
 417                         sched_exit_thread(td2, td);
 418
 419                         /*
 420                          * The test below is NOT true if we are the
 421                          * sole exiting thread. P_STOPPED_SNGL is unset
 422                          * in exit1() after it is the only survivor.
 423                          */
 424                         if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
 425                                 if (p->p_numthreads == p->p_suspcount) {
 426                                         thread_lock(p->p_singlethread);
 427                                         thread_unsuspend_one(p->p_singlethread);
 428                                         thread_unlock(p->p_singlethread);
 429                                 }
 430                         }
 431
 432 #ifdef KSE
 433                         /*
 434                          * Because each upcall structure has an owner thread,
 435                          * owner thread exits only when process is in exiting
 436                          * state, so upcall to userland is no longer needed,
 437                          * deleting upcall structure is safe here.
 438                          * So when all threads in a group is exited, all upcalls
 439                          * in the group should be automatically freed.
 440                          *  XXXKSE This is a KSE thing and should be exported
 441                          * there somehow.
 442                          */
 443                         upcall_remove(td);
 444 #endif
 445                         PCPU_SET(deadthread, td);
 446                 } else {
 447                         /*
 448                          * The last thread is exiting.. but not through exit()
 449                          * what should we do?
 450                          * Theoretically this can't happen
 451                          * exit1() - clears threading flags before coming here
 452                          * kse_exit() - treats last thread specially
 453                          * thr_exit() - treats last thread specially
 454                          * ifdef KSE
 455                          * thread_user_enter() - only if more exist
 456                          * thread_userret() - only if more exist
 457                          * endif
 458                          * thread_suspend_check() - only if more exist
 459                          */
 460                         panic ("thread_exit: Last thread exiting on its own");
 461                 }
 462         }
 463         PROC_UNLOCK(p);
 464         thread_lock(td);
 465         /* Aggregate our tick statistics into our parents rux. */
 466         ruxagg(&p->p_rux, td);
 467         PROC_SUNLOCK(p);
 468         td->td_state = TDS_INACTIVE;
 469         CTR1(KTR_PROC, "thread_exit: cpu_throw() thread %p", td);
 470         sched_throw(td);
 471         panic("I'm a teapot!");
 472         /* NOTREACHED */
 473 }
 474
 475 /*
 476  * Do any thread specific cleanups that may be needed in wait()
 477  * called with Giant, proc and schedlock not held.
 478  */
 479 void
 480 thread_wait(struct proc *p)
 481 {
 482         struct thread *td;
 483
 484         mtx_assert(&Giant, MA_NOTOWNED);
 485         KASSERT((p->p_numthreads == 1), ("Multiple threads in wait1()"));
 486         FOREACH_THREAD_IN_PROC(p, td) {
 487 #ifdef KSE
 488                 if (td->td_standin != NULL) {
 489                         if (td->td_standin->td_ucred != NULL) {
 490                                 crfree(td->td_standin->td_ucred);
 491                                 td->td_standin->td_ucred = NULL;
 492                         }
 493                         thread_free(td->td_standin);
 494                         td->td_standin = NULL;
 495                 }
 496 #endif
 497                 cpu_thread_clean(td);
 498                 crfree(td->td_ucred);
 499         }
 500         thread_reap();  /* check for zombie threads etc. */
 501 }
 502
 503 /*
 504  * Link a thread to a process.
 505  * set up anything that needs to be initialized for it to
 506  * be used by the process.
 507  *
 508  * Note that we do not link to the proc's ucred here.
 509  * The thread is linked as if running but no KSE assigned.
 510  * Called from:
 511  *  proc_linkup()
 512  *  thread_schedule_upcall()
 513  *  thr_create()
 514  */
 515 void
 516 thread_link(struct thread *td, struct proc *p)
 517 {
 518
 519         /*
 520          * XXX This can't be enabled because it's called for proc0 before
 521          * it's spinlock has been created.
 522          * PROC_SLOCK_ASSERT(p, MA_OWNED);
 523          */
 524         td->td_state    = TDS_INACTIVE;
 525         td->td_proc     = p;
 526         td->td_flags    = 0;
 527
 528         LIST_INIT(&td->td_contested);
 529         sigqueue_init(&td->td_sigqueue, p);
 530         callout_init(&td->td_slpcallout, CALLOUT_MPSAFE);
 531         TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist);
 532         p->p_numthreads++;
 533 }
 534
 535 /*
 536  * Convert a process with one thread to an unthreaded process.
 537  * Called from:
 538  *  thread_single(exit)  (called from execve and exit)
 539  *  kse_exit()          XXX may need cleaning up wrt KSE stuff
 540  */
 541 void
 542 thread_unthread(struct thread *td)
 543 {
 544         struct proc *p = td->td_proc;
 545
 546         KASSERT((p->p_numthreads == 1), ("Unthreading with >1 threads"));
 547 #ifdef KSE
 548         upcall_remove(td);
 549         p->p_flag &= ~(P_SA|P_HADTHREADS);
 550         td->td_mailbox = NULL;
 551         td->td_pflags &= ~(TDP_SA | TDP_CAN_UNBIND);
 552         if (td->td_standin != NULL) {
 553                 thread_stash(td->td_standin);
 554                 td->td_standin = NULL;
 555         }
 556         sched_set_concurrency(p, 1);
 557 #else
 558         p->p_flag &= ~P_HADTHREADS;
 559 #endif
 560 }
 561
 562 /*
 563  * Called from:
 564  *  thread_exit()
 565  */
 566 void
 567 thread_unlink(struct thread *td)
 568 {
 569         struct proc *p = td->td_proc;
 570
 571         PROC_SLOCK_ASSERT(p, MA_OWNED);
 572         TAILQ_REMOVE(&p->p_threads, td, td_plist);
 573         p->p_numthreads--;
 574         /* could clear a few other things here */
 575         /* Must  NOT clear links to proc! */
 576 }
 577
 578 /*
 579  * Enforce single-threading.
 580  *
 581  * Returns 1 if the caller must abort (another thread is waiting to
 582  * exit the process or similar). Process is locked!
 583  * Returns 0 when you are successfully the only thread running.
 584  * A process has successfully single threaded in the suspend mode when
 585  * There are no threads in user mode. Threads in the kernel must be
 586  * allowed to continue until they get to the user boundary. They may even
 587  * copy out their return values and data before suspending. They may however be
 588  * accelerated in reaching the user boundary as we will wake up
 589  * any sleeping threads that are interruptable. (PCATCH).
 590  */
 591 int
 592 thread_single(int mode)
 593 {
 594         struct thread *td;
 595         struct thread *td2;
 596         struct proc *p;
 597         int remaining;
 598
 599         td = curthread;
 600         p = td->td_proc;
 601         mtx_assert(&Giant, MA_NOTOWNED);
 602         PROC_LOCK_ASSERT(p, MA_OWNED);
 603         KASSERT((td != NULL), ("curthread is NULL"));
 604
 605         if ((p->p_flag & P_HADTHREADS) == 0)
 606                 return (0);
 607
 608         /* Is someone already single threading? */
 609         if (p->p_singlethread != NULL && p->p_singlethread != td)
 610                 return (1);
 611
 612         if (mode == SINGLE_EXIT) {
 613                 p->p_flag |= P_SINGLE_EXIT;
 614                 p->p_flag &= ~P_SINGLE_BOUNDARY;
 615         } else {
 616                 p->p_flag &= ~P_SINGLE_EXIT;
 617                 if (mode == SINGLE_BOUNDARY)
 618                         p->p_flag |= P_SINGLE_BOUNDARY;
 619                 else
 620                         p->p_flag &= ~P_SINGLE_BOUNDARY;
 621         }
 622         p->p_flag |= P_STOPPED_SINGLE;
 623         PROC_SLOCK(p);
 624         p->p_singlethread = td;
 625         if (mode == SINGLE_EXIT)
 626                 remaining = p->p_numthreads;
 627         else if (mode == SINGLE_BOUNDARY)
 628                 remaining = p->p_numthreads - p->p_boundary_count;
 629         else
 630                 remaining = p->p_numthreads - p->p_suspcount;
 631         while (remaining != 1) {
 632                 if (P_SHOULDSTOP(p) != P_STOPPED_SINGLE)
 633                         goto stopme;
 634                 FOREACH_THREAD_IN_PROC(p, td2) {
 635                         if (td2 == td)
 636                                 continue;
 637                         thread_lock(td2);
 638                         td2->td_flags |= TDF_ASTPENDING;
 639                         if (TD_IS_INHIBITED(td2)) {
 640                                 switch (mode) {
 641                                 case SINGLE_EXIT:
 642                                         if (td->td_flags & TDF_DBSUSPEND)
 643                                                 td->td_flags &= ~TDF_DBSUSPEND;
 644                                         if (TD_IS_SUSPENDED(td2))
 645                                                 thread_unsuspend_one(td2);
 646                                         if (TD_ON_SLEEPQ(td2) &&
 647                                             (td2->td_flags & TDF_SINTR))
 648                                                 sleepq_abort(td2, EINTR);
 649                                         break;
 650                                 case SINGLE_BOUNDARY:
 651                                         if (TD_IS_SUSPENDED(td2) &&
 652                                             !(td2->td_flags & TDF_BOUNDARY))
 653                                                 thread_unsuspend_one(td2);
 654                                         if (TD_ON_SLEEPQ(td2) &&
 655                                             (td2->td_flags & TDF_SINTR))
 656                                                 sleepq_abort(td2, ERESTART);
 657                                         break;
 658                                 default:
 659                                         if (TD_IS_SUSPENDED(td2)) {
 660                                                 thread_unlock(td2);
 661                                                 continue;
 662                                         }
 663                                         /*
 664                                          * maybe other inhibited states too?
 665                                          */
 666                                         if ((td2->td_flags & TDF_SINTR) &&
 667                                             (td2->td_inhibitors &
 668                                             (TDI_SLEEPING | TDI_SWAPPED)))
 669                                                 thread_suspend_one(td2);
 670                                         break;
 671                                 }
 672                         }
 673 #ifdef SMP
 674                         else if (TD_IS_RUNNING(td2) && td != td2) {
 675                                 forward_signal(td2);
 676                         }
 677 #endif
 678                         thread_unlock(td2);
 679                 }
 680                 if (mode == SINGLE_EXIT)
 681                         remaining = p->p_numthreads;
 682                 else if (mode == SINGLE_BOUNDARY)
 683                         remaining = p->p_numthreads - p->p_boundary_count;
 684                 else
 685                         remaining = p->p_numthreads - p->p_suspcount;
 686
 687                 /*
 688                  * Maybe we suspended some threads.. was it enough?
 689                  */
 690                 if (remaining == 1)
 691                         break;
 692
 693 stopme:
 694                 /*
 695                  * Wake us up when everyone else has suspended.
 696                  * In the mean time we suspend as well.
 697                  */
 698                 thread_suspend_switch(td);
 699                 if (mode == SINGLE_EXIT)
 700                         remaining = p->p_numthreads;
 701                 else if (mode == SINGLE_BOUNDARY)
 702                         remaining = p->p_numthreads - p->p_boundary_count;
 703                 else
 704                         remaining = p->p_numthreads - p->p_suspcount;
 705         }
 706         if (mode == SINGLE_EXIT) {
 707                 /*
 708                  * We have gotten rid of all the other threads and we
 709                  * are about to either exit or exec. In either case,
 710                  * we try our utmost  to revert to being a non-threaded
 711                  * process.
 712                  */
 713                 p->p_singlethread = NULL;
 714                 p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT);
 715                 thread_unthread(td);
 716         }
 717         PROC_SUNLOCK(p);
 718         return (0);
 719 }
 720
 721 /*
 722  * Called in from locations that can safely check to see
 723  * whether we have to suspend or at least throttle for a
 724  * single-thread event (e.g. fork).
 725  *
 726  * Such locations include userret().
 727  * If the "return_instead" argument is non zero, the thread must be able to
 728  * accept 0 (caller may continue), or 1 (caller must abort) as a result.
 729  *
 730  * The 'return_instead' argument tells the function if it may do a
 731  * thread_exit() or suspend, or whether the caller must abort and back
 732  * out instead.
 733  *
 734  * If the thread that set the single_threading request has set the
 735  * P_SINGLE_EXIT bit in the process flags then this call will never return
 736  * if 'return_instead' is false, but will exit.
 737  *
 738  * P_SINGLE_EXIT | return_instead == 0| return_instead != 0
 739  *---------------+--------------------+---------------------
 740  *       0       | returns 0          |   returns 0 or 1
 741  *               | when ST ends       |   immediatly
 742  *---------------+--------------------+---------------------
 743  *       1       | thread exits       |   returns 1
 744  *               |                    |  immediatly
 745  * 0 = thread_exit() or suspension ok,
 746  * other = return error instead of stopping the thread.
 747  *
 748  * While a full suspension is under effect, even a single threading
 749  * thread would be suspended if it made this call (but it shouldn't).
 750  * This call should only be made from places where
 751  * thread_exit() would be safe as that may be the outcome unless
 752  * return_instead is set.
 753  */
 754 int
 755 thread_suspend_check(int return_instead)
 756 {
 757         struct thread *td;
 758         struct proc *p;
 759
 760         td = curthread;
 761         p = td->td_proc;
 762         mtx_assert(&Giant, MA_NOTOWNED);
 763         PROC_LOCK_ASSERT(p, MA_OWNED);
 764         while (P_SHOULDSTOP(p) ||
 765               ((p->p_flag & P_TRACED) && (td->td_flags & TDF_DBSUSPEND))) {
 766                 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
 767                         KASSERT(p->p_singlethread != NULL,
 768                             ("singlethread not set"));
 769                         /*
 770                          * The only suspension in action is a
 771                          * single-threading. Single threader need not stop.
 772                          * XXX Should be safe to access unlocked
 773                          * as it can only be set to be true by us.
 774                          */
 775                         if (p->p_singlethread == td)
 776                                 return (0);     /* Exempt from stopping. */
 777                 }
 778                 if ((p->p_flag & P_SINGLE_EXIT) && return_instead)
 779                         return (EINTR);
 780
 781                 /* Should we goto user boundary if we didn't come from there? */
 782                 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE &&
 783                     (p->p_flag & P_SINGLE_BOUNDARY) && return_instead)
 784                         return (ERESTART);
 785
 786                 /* If thread will exit, flush its pending signals */
 787                 if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td))
 788                         sigqueue_flush(&td->td_sigqueue);
 789
 790                 PROC_SLOCK(p);
 791                 thread_stopped(p);
 792                 /*
 793                  * If the process is waiting for us to exit,
 794                  * this thread should just suicide.
 795                  * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE.
 796                  */
 797                 if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td))
 798                         thread_exit();
 799                 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
 800                         if (p->p_numthreads == p->p_suspcount + 1) {
 801                                 thread_lock(p->p_singlethread);
 802                                 thread_unsuspend_one(p->p_singlethread);
 803                                 thread_unlock(p->p_singlethread);
 804                         }
 805                 }
 806                 PROC_UNLOCK(p);
 807                 thread_lock(td);
 808                 /*
 809                  * When a thread suspends, it just
 810                  * gets taken off all queues.
 811                  */
 812                 thread_suspend_one(td);
 813                 if (return_instead == 0) {
 814                         p->p_boundary_count++;
 815                         td->td_flags |= TDF_BOUNDARY;
 816                 }
 817                 PROC_SUNLOCK(p);
 818                 mi_switch(SW_INVOL, NULL);
 819                 if (return_instead == 0)
 820                         td->td_flags &= ~TDF_BOUNDARY;
 821                 thread_unlock(td);
 822                 PROC_LOCK(p);
 823                 if (return_instead == 0)
 824                         p->p_boundary_count--;
 825         }
 826         return (0);
 827 }
 828
 829 void
 830 thread_suspend_switch(struct thread *td)
 831 {
 832         struct proc *p;
 833
 834         p = td->td_proc;
 835         KASSERT(!TD_IS_SUSPENDED(td), ("already suspended"));
 836         PROC_LOCK_ASSERT(p, MA_OWNED);
 837         PROC_SLOCK_ASSERT(p, MA_OWNED);
 838         /*
 839          * We implement thread_suspend_one in stages here to avoid
 840          * dropping the proc lock while the thread lock is owned.
 841          */
 842         thread_stopped(p);
 843         p->p_suspcount++;
 844         PROC_UNLOCK(p);
 845         thread_lock(td);
 846         TD_SET_SUSPENDED(td);
 847         PROC_SUNLOCK(p);
 848         DROP_GIANT();
 849         mi_switch(SW_VOL, NULL);
 850         thread_unlock(td);
 851         PICKUP_GIANT();
 852         PROC_LOCK(p);
 853         PROC_SLOCK(p);
 854 }
 855
 856 void
 857 thread_suspend_one(struct thread *td)
 858 {
 859         struct proc *p = td->td_proc;
 860
 861         PROC_SLOCK_ASSERT(p, MA_OWNED);
 862         THREAD_LOCK_ASSERT(td, MA_OWNED);
 863         KASSERT(!TD_IS_SUSPENDED(td), ("already suspended"));
 864         p->p_suspcount++;
 865         TD_SET_SUSPENDED(td);
 866 }
 867
 868 void
 869 thread_unsuspend_one(struct thread *td)
 870 {
 871         struct proc *p = td->td_proc;
 872
 873         PROC_SLOCK_ASSERT(p, MA_OWNED);
 874         THREAD_LOCK_ASSERT(td, MA_OWNED);
 875         KASSERT(TD_IS_SUSPENDED(td), ("Thread not suspended"));
 876         TD_CLR_SUSPENDED(td);
 877         p->p_suspcount--;
 878         setrunnable(td);
 879 }
 880
 881 /*
 882  * Allow all threads blocked by single threading to continue running.
 883  */
 884 void
 885 thread_unsuspend(struct proc *p)
 886 {
 887         struct thread *td;
 888
 889         PROC_LOCK_ASSERT(p, MA_OWNED);
 890         PROC_SLOCK_ASSERT(p, MA_OWNED);
 891         if (!P_SHOULDSTOP(p)) {
 892                 FOREACH_THREAD_IN_PROC(p, td) {
 893                         thread_lock(td);
 894                         if (TD_IS_SUSPENDED(td)) {
 895                                 thread_unsuspend_one(td);
 896                         }
 897                         thread_unlock(td);
 898                 }
 899         } else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) &&
 900             (p->p_numthreads == p->p_suspcount)) {
 901                 /*
 902                  * Stopping everything also did the job for the single
 903                  * threading request. Now we've downgraded to single-threaded,
 904                  * let it continue.
 905                  */
 906                 thread_lock(p->p_singlethread);
 907                 thread_unsuspend_one(p->p_singlethread);
 908                 thread_unlock(p->p_singlethread);
 909         }
 910 }
 911
 912 /*
 913  * End the single threading mode..
 914  */
 915 void
 916 thread_single_end(void)
 917 {
 918         struct thread *td;
 919         struct proc *p;
 920
 921         td = curthread;
 922         p = td->td_proc;
 923         PROC_LOCK_ASSERT(p, MA_OWNED);
 924         p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_SINGLE_BOUNDARY);
 925         PROC_SLOCK(p);
 926         p->p_singlethread = NULL;
 927         /*
 928          * If there are other threads they mey now run,
 929          * unless of course there is a blanket 'stop order'
 930          * on the process. The single threader must be allowed
 931          * to continue however as this is a bad place to stop.
 932          */
 933         if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) {
 934                 FOREACH_THREAD_IN_PROC(p, td) {
 935                         thread_lock(td);
 936                         if (TD_IS_SUSPENDED(td)) {
 937                                 thread_unsuspend_one(td);
 938                         }
 939                         thread_unlock(td);
 940                 }
 941         }
 942         PROC_SUNLOCK(p);
 943 }
 944
 945 struct thread *
 946 thread_find(struct proc *p, lwpid_t tid)
 947 {
 948         struct thread *td;
 949
 950         PROC_LOCK_ASSERT(p, MA_OWNED);
 951         PROC_SLOCK(p);
 952         FOREACH_THREAD_IN_PROC(p, td) {
 953                 if (td->td_tid == tid)
 954                         break;
 955         }
 956         PROC_SUNLOCK(p);
 957         return (td);
 958 }