sys/vm/vm_pageout.c

   1 /*-
   2  * Copyright (c) 1991 Regents of the University of California.
   3  * All rights reserved.
   4  * Copyright (c) 1994 John S. Dyson
   5  * All rights reserved.
   6  * Copyright (c) 1994 David Greenman
   7  * All rights reserved.
   8  * Copyright (c) 2005 Yahoo! Technologies Norway AS
   9  * All rights reserved.
  10  *
  11  * This code is derived from software contributed to Berkeley by
  12  * The Mach Operating System project at Carnegie-Mellon University.
  13  *
  14  * Redistribution and use in source and binary forms, with or without
  15  * modification, are permitted provided that the following conditions
  16  * are met:
  17  * 1. Redistributions of source code must retain the above copyright
  18  *    notice, this list of conditions and the following disclaimer.
  19  * 2. Redistributions in binary form must reproduce the above copyright
  20  *    notice, this list of conditions and the following disclaimer in the
  21  *    documentation and/or other materials provided with the distribution.
  22  * 3. All advertising materials mentioning features or use of this software
  23  *    must display the following acknowledgement:
  24  *      This product includes software developed by the University of
  25  *      California, Berkeley and its contributors.
  26  * 4. Neither the name of the University nor the names of its contributors
  27  *    may be used to endorse or promote products derived from this software
  28  *    without specific prior written permission.
  29  *
  30  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  31  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  32  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  33  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  34  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  35  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  36  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  37  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  38  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  39  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  40  * SUCH DAMAGE.
  41  *
  42  *      from: @(#)vm_pageout.c  7.4 (Berkeley) 5/7/91
  43  *
  44  *
  45  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
  46  * All rights reserved.
  47  *
  48  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
  49  *
  50  * Permission to use, copy, modify and distribute this software and
  51  * its documentation is hereby granted, provided that both the copyright
  52  * notice and this permission notice appear in all copies of the
  53  * software, derivative works or modified versions, and any portions
  54  * thereof, and that both notices appear in supporting documentation.
  55  *
  56  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  57  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
  58  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  59  *
  60  * Carnegie Mellon requests users of this software to return to
  61  *
  62  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  63  *  School of Computer Science
  64  *  Carnegie Mellon University
  65  *  Pittsburgh PA 15213-3890
  66  *
  67  * any improvements or extensions that they make and grant Carnegie the
  68  * rights to redistribute these changes.
  69  */
  70
  71 /*
  72  *      The proverbial page-out daemon.
  73  */
  74
  75 #include <sys/cdefs.h>
  76 __FBSDID("$FreeBSD$");
  77
  78 #include "opt_vm.h"
  79
  80 #include <sys/param.h>
  81 #include <sys/systm.h>
  82 #include <sys/kernel.h>
  83 #include <sys/eventhandler.h>
  84 #include <sys/lock.h>
  85 #include <sys/mutex.h>
  86 #include <sys/proc.h>
  87 #include <sys/kthread.h>
  88 #include <sys/ktr.h>
  89 #include <sys/mount.h>
  90 #include <sys/racct.h>
  91 #include <sys/resourcevar.h>
  92 #include <sys/sched.h>
  93 #include <sys/sdt.h>
  94 #include <sys/signalvar.h>
  95 #include <sys/smp.h>
  96 #include <sys/time.h>
  97 #include <sys/vnode.h>
  98 #include <sys/vmmeter.h>
  99 #include <sys/rwlock.h>
 100 #include <sys/sx.h>
 101 #include <sys/sysctl.h>
 102
 103 #include <vm/vm.h>
 104 #include <vm/vm_param.h>
 105 #include <vm/vm_object.h>
 106 #include <vm/vm_page.h>
 107 #include <vm/vm_map.h>
 108 #include <vm/vm_pageout.h>
 109 #include <vm/vm_pager.h>
 110 #include <vm/vm_phys.h>
 111 #include <vm/swap_pager.h>
 112 #include <vm/vm_extern.h>
 113 #include <vm/uma.h>
 114
 115 /*
 116  * System initialization
 117  */
 118
 119 /* the kernel process "vm_pageout"*/
 120 static void vm_pageout(void);
 121 static void vm_pageout_init(void);
 122 static int vm_pageout_clean(vm_page_t m);
 123 static int vm_pageout_cluster(vm_page_t m);
 124 static void vm_pageout_scan(struct vm_domain *vmd, int pass);
 125 static void vm_pageout_mightbe_oom(struct vm_domain *vmd, int page_shortage,
 126     int starting_page_shortage);
 127
 128 SYSINIT(pagedaemon_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_FIRST, vm_pageout_init,
 129     NULL);
 130
 131 struct proc *pageproc;
 132
 133 static struct kproc_desc page_kp = {
 134         "pagedaemon",
 135         vm_pageout,
 136         &pageproc
 137 };
 138 SYSINIT(pagedaemon, SI_SUB_KTHREAD_PAGE, SI_ORDER_SECOND, kproc_start,
 139     &page_kp);
 140
 141 SDT_PROVIDER_DEFINE(vm);
 142 SDT_PROBE_DEFINE(vm, , , vm__lowmem_scan);
 143
 144 #if !defined(NO_SWAPPING)
 145 /* the kernel process "vm_daemon"*/
 146 static void vm_daemon(void);
 147 static struct   proc *vmproc;
 148
 149 static struct kproc_desc vm_kp = {
 150         "vmdaemon",
 151         vm_daemon,
 152         &vmproc
 153 };
 154 SYSINIT(vmdaemon, SI_SUB_KTHREAD_VM, SI_ORDER_FIRST, kproc_start, &vm_kp);
 155 #endif
 156
 157
 158 int vm_pageout_deficit;         /* Estimated number of pages deficit */
 159 u_int vm_pageout_wakeup_thresh;
 160 static int vm_pageout_oom_seq = 12;
 161 bool vm_pageout_wanted;         /* Event on which pageout daemon sleeps */
 162 bool vm_pages_needed;           /* Are threads waiting for free pages? */
 163
 164 #if !defined(NO_SWAPPING)
 165 static int vm_pageout_req_swapout;      /* XXX */
 166 static int vm_daemon_needed;
 167 static struct mtx vm_daemon_mtx;
 168 /* Allow for use by vm_pageout before vm_daemon is initialized. */
 169 MTX_SYSINIT(vm_daemon, &vm_daemon_mtx, "vm daemon", MTX_DEF);
 170 #endif
 171 static int vm_max_launder = 32;
 172 static int vm_pageout_update_period;
 173 static int defer_swap_pageouts;
 174 static int disable_swap_pageouts;
 175 static int lowmem_period = 10;
 176 static time_t lowmem_uptime;
 177
 178 #if defined(NO_SWAPPING)
 179 static int vm_swap_enabled = 0;
 180 static int vm_swap_idle_enabled = 0;
 181 #else
 182 static int vm_swap_enabled = 1;
 183 static int vm_swap_idle_enabled = 0;
 184 #endif
 185
 186 static int vm_panic_on_oom = 0;
 187
 188 SYSCTL_INT(_vm, OID_AUTO, panic_on_oom,
 189         CTLFLAG_RWTUN, &vm_panic_on_oom, 0,
 190         "panic on out of memory instead of killing the largest process");
 191
 192 SYSCTL_INT(_vm, OID_AUTO, pageout_wakeup_thresh,
 193         CTLFLAG_RW, &vm_pageout_wakeup_thresh, 0,
 194         "free page threshold for waking up the pageout daemon");
 195
 196 SYSCTL_INT(_vm, OID_AUTO, max_launder,
 197         CTLFLAG_RW, &vm_max_launder, 0, "Limit dirty flushes in pageout");
 198
 199 SYSCTL_INT(_vm, OID_AUTO, pageout_update_period,
 200         CTLFLAG_RW, &vm_pageout_update_period, 0,
 201         "Maximum active LRU update period");
 202
 203 SYSCTL_INT(_vm, OID_AUTO, lowmem_period, CTLFLAG_RW, &lowmem_period, 0,
 204         "Low memory callback period");
 205
 206 #if defined(NO_SWAPPING)
 207 SYSCTL_INT(_vm, VM_SWAPPING_ENABLED, swap_enabled,
 208         CTLFLAG_RD, &vm_swap_enabled, 0, "Enable entire process swapout");
 209 SYSCTL_INT(_vm, OID_AUTO, swap_idle_enabled,
 210         CTLFLAG_RD, &vm_swap_idle_enabled, 0, "Allow swapout on idle criteria");
 211 #else
 212 SYSCTL_INT(_vm, VM_SWAPPING_ENABLED, swap_enabled,
 213         CTLFLAG_RW, &vm_swap_enabled, 0, "Enable entire process swapout");
 214 SYSCTL_INT(_vm, OID_AUTO, swap_idle_enabled,
 215         CTLFLAG_RW, &vm_swap_idle_enabled, 0, "Allow swapout on idle criteria");
 216 #endif
 217
 218 SYSCTL_INT(_vm, OID_AUTO, defer_swapspace_pageouts,
 219         CTLFLAG_RW, &defer_swap_pageouts, 0, "Give preference to dirty pages in mem");
 220
 221 SYSCTL_INT(_vm, OID_AUTO, disable_swapspace_pageouts,
 222         CTLFLAG_RW, &disable_swap_pageouts, 0, "Disallow swapout of dirty pages");
 223
 224 static int pageout_lock_miss;
 225 SYSCTL_INT(_vm, OID_AUTO, pageout_lock_miss,
 226         CTLFLAG_RD, &pageout_lock_miss, 0, "vget() lock misses during pageout");
 227
 228 SYSCTL_INT(_vm, OID_AUTO, pageout_oom_seq,
 229         CTLFLAG_RW, &vm_pageout_oom_seq, 0,
 230         "back-to-back calls to oom detector to start OOM");
 231
 232 #define VM_PAGEOUT_PAGE_COUNT 16
 233 int vm_pageout_page_count = VM_PAGEOUT_PAGE_COUNT;
 234
 235 int vm_page_max_wired;          /* XXX max # of wired pages system-wide */
 236 SYSCTL_INT(_vm, OID_AUTO, max_wired,
 237         CTLFLAG_RW, &vm_page_max_wired, 0, "System-wide limit to wired page count");
 238
 239 static boolean_t vm_pageout_fallback_object_lock(vm_page_t, vm_page_t *);
 240 #if !defined(NO_SWAPPING)
 241 static void vm_pageout_map_deactivate_pages(vm_map_t, long);
 242 static void vm_pageout_object_deactivate_pages(pmap_t, vm_object_t, long);
 243 static void vm_req_vmdaemon(int req);
 244 #endif
 245 static boolean_t vm_pageout_page_lock(vm_page_t, vm_page_t *);
 246
 247 /*
 248  * Initialize a dummy page for marking the caller's place in the specified
 249  * paging queue.  In principle, this function only needs to set the flag
 250  * PG_MARKER.  Nonetheless, it write busies and initializes the hold count
 251  * to one as safety precautions.
 252  */
 253 static void
 254 vm_pageout_init_marker(vm_page_t marker, u_short queue)
 255 {
 256
 257         bzero(marker, sizeof(*marker));
 258         marker->flags = PG_MARKER;
 259         marker->busy_lock = VPB_SINGLE_EXCLUSIVER;
 260         marker->queue = queue;
 261         marker->hold_count = 1;
 262 }
 263
 264 /*
 265  * vm_pageout_fallback_object_lock:
 266  *
 267  * Lock vm object currently associated with `m'. VM_OBJECT_TRYWLOCK is
 268  * known to have failed and page queue must be either PQ_ACTIVE or
 269  * PQ_INACTIVE.  To avoid lock order violation, unlock the page queue
 270  * while locking the vm object.  Use marker page to detect page queue
 271  * changes and maintain notion of next page on page queue.  Return
 272  * TRUE if no changes were detected, FALSE otherwise.  vm object is
 273  * locked on return.
 274  *
 275  * This function depends on both the lock portion of struct vm_object
 276  * and normal struct vm_page being type stable.
 277  */
 278 static boolean_t
 279 vm_pageout_fallback_object_lock(vm_page_t m, vm_page_t *next)
 280 {
 281         struct vm_page marker;
 282         struct vm_pagequeue *pq;
 283         boolean_t unchanged;
 284         u_short queue;
 285         vm_object_t object;
 286
 287         queue = m->queue;
 288         vm_pageout_init_marker(&marker, queue);
 289         pq = vm_page_pagequeue(m);
 290         object = m->object;
 291
 292         TAILQ_INSERT_AFTER(&pq->pq_pl, m, &marker, plinks.q);
 293         vm_pagequeue_unlock(pq);
 294         vm_page_unlock(m);
 295         VM_OBJECT_WLOCK(object);
 296         vm_page_lock(m);
 297         vm_pagequeue_lock(pq);
 298
 299         /*
 300          * The page's object might have changed, and/or the page might
 301          * have moved from its original position in the queue.  If the
 302          * page's object has changed, then the caller should abandon
 303          * processing the page because the wrong object lock was
 304          * acquired.  Use the marker's plinks.q, not the page's, to
 305          * determine if the page has been moved.  The state of the
 306          * page's plinks.q can be indeterminate; whereas, the marker's
 307          * plinks.q must be valid.
 308          */
 309         *next = TAILQ_NEXT(&marker, plinks.q);
 310         unchanged = m->object == object &&
 311             m == TAILQ_PREV(&marker, pglist, plinks.q);
 312         KASSERT(!unchanged || m->queue == queue,
 313             ("page %p queue %d %d", m, queue, m->queue));
 314         TAILQ_REMOVE(&pq->pq_pl, &marker, plinks.q);
 315         return (unchanged);
 316 }
 317
 318 /*
 319  * Lock the page while holding the page queue lock.  Use marker page
 320  * to detect page queue changes and maintain notion of next page on
 321  * page queue.  Return TRUE if no changes were detected, FALSE
 322  * otherwise.  The page is locked on return. The page queue lock might
 323  * be dropped and reacquired.
 324  *
 325  * This function depends on normal struct vm_page being type stable.
 326  */
 327 static boolean_t
 328 vm_pageout_page_lock(vm_page_t m, vm_page_t *next)
 329 {
 330         struct vm_page marker;
 331         struct vm_pagequeue *pq;
 332         boolean_t unchanged;
 333         u_short queue;
 334
 335         vm_page_lock_assert(m, MA_NOTOWNED);
 336         if (vm_page_trylock(m))
 337                 return (TRUE);
 338
 339         queue = m->queue;
 340         vm_pageout_init_marker(&marker, queue);
 341         pq = vm_page_pagequeue(m);
 342
 343         TAILQ_INSERT_AFTER(&pq->pq_pl, m, &marker, plinks.q);
 344         vm_pagequeue_unlock(pq);
 345         vm_page_lock(m);
 346         vm_pagequeue_lock(pq);
 347
 348         /* Page queue might have changed. */
 349         *next = TAILQ_NEXT(&marker, plinks.q);
 350         unchanged = m == TAILQ_PREV(&marker, pglist, plinks.q);
 351         KASSERT(!unchanged || m->queue == queue,
 352             ("page %p queue %d %d", m, queue, m->queue));
 353         TAILQ_REMOVE(&pq->pq_pl, &marker, plinks.q);
 354         return (unchanged);
 355 }
 356
 357 /*
 358  * Scan for pages at adjacent offsets within the given page's object that are
 359  * eligible for laundering, form a cluster of these pages and the given page,
 360  * and launder that cluster.
 361  */
 362 static int
 363 vm_pageout_cluster(vm_page_t m)
 364 {
 365         vm_object_t object;
 366         vm_page_t mc[2 * vm_pageout_page_count], p, pb, ps;
 367         vm_pindex_t pindex;
 368         int ib, is, page_base, pageout_count;
 369
 370         vm_page_assert_locked(m);
 371         object = m->object;
 372         VM_OBJECT_ASSERT_WLOCKED(object);
 373         pindex = m->pindex;
 374
 375         /*
 376          * We can't clean the page if it is busy or held.
 377          */
 378         vm_page_assert_unbusied(m);
 379         KASSERT(m->hold_count == 0, ("page %p is held", m));
 380         vm_page_unlock(m);
 381
 382         mc[vm_pageout_page_count] = pb = ps = m;
 383         pageout_count = 1;
 384         page_base = vm_pageout_page_count;
 385         ib = 1;
 386         is = 1;
 387
 388         /*
 389          * We can cluster only if the page is not clean, busy, or held, and
 390          * the page is inactive.
 391          *
 392          * During heavy mmap/modification loads the pageout
 393          * daemon can really fragment the underlying file
 394          * due to flushing pages out of order and not trying to
 395          * align the clusters (which leaves sporadic out-of-order
 396          * holes).  To solve this problem we do the reverse scan
 397          * first and attempt to align our cluster, then do a
 398          * forward scan if room remains.
 399          */
 400 more:
 401         while (ib != 0 && pageout_count < vm_pageout_page_count) {
 402                 if (ib > pindex) {
 403                         ib = 0;
 404                         break;
 405                 }
 406                 if ((p = vm_page_prev(pb)) == NULL || vm_page_busied(p)) {
 407                         ib = 0;
 408                         break;
 409                 }
 410                 vm_page_test_dirty(p);
 411                 if (p->dirty == 0) {
 412                         ib = 0;
 413                         break;
 414                 }
 415                 vm_page_lock(p);
 416                 if (p->queue != PQ_INACTIVE ||
 417                     p->hold_count != 0) {       /* may be undergoing I/O */
 418                         vm_page_unlock(p);
 419                         ib = 0;
 420                         break;
 421                 }
 422                 vm_page_unlock(p);
 423                 mc[--page_base] = pb = p;
 424                 ++pageout_count;
 425                 ++ib;
 426
 427                 /*
 428                  * We are at an alignment boundary.  Stop here, and switch
 429                  * directions.  Do not clear ib.
 430                  */
 431                 if ((pindex - (ib - 1)) % vm_pageout_page_count == 0)
 432                         break;
 433         }
 434         while (pageout_count < vm_pageout_page_count &&
 435             pindex + is < object->size) {
 436                 if ((p = vm_page_next(ps)) == NULL || vm_page_busied(p))
 437                         break;
 438                 vm_page_test_dirty(p);
 439                 if (p->dirty == 0)
 440                         break;
 441                 vm_page_lock(p);
 442                 if (p->queue != PQ_INACTIVE ||
 443                     p->hold_count != 0) {       /* may be undergoing I/O */
 444                         vm_page_unlock(p);
 445                         break;
 446                 }
 447                 vm_page_unlock(p);
 448                 mc[page_base + pageout_count] = ps = p;
 449                 ++pageout_count;
 450                 ++is;
 451         }
 452
 453         /*
 454          * If we exhausted our forward scan, continue with the reverse scan
 455          * when possible, even past an alignment boundary.  This catches
 456          * boundary conditions.
 457          */
 458         if (ib != 0 && pageout_count < vm_pageout_page_count)
 459                 goto more;
 460
 461         return (vm_pageout_flush(&mc[page_base], pageout_count, 0, 0, NULL,
 462             NULL));
 463 }
 464
 465 /*
 466  * vm_pageout_flush() - launder the given pages
 467  *
 468  *      The given pages are laundered.  Note that we setup for the start of
 469  *      I/O ( i.e. busy the page ), mark it read-only, and bump the object
 470  *      reference count all in here rather then in the parent.  If we want
 471  *      the parent to do more sophisticated things we may have to change
 472  *      the ordering.
 473  *
 474  *      Returned runlen is the count of pages between mreq and first
 475  *      page after mreq with status VM_PAGER_AGAIN.
 476  *      *eio is set to TRUE if pager returned VM_PAGER_ERROR or VM_PAGER_FAIL
 477  *      for any page in runlen set.
 478  */
 479 int
 480 vm_pageout_flush(vm_page_t *mc, int count, int flags, int mreq, int *prunlen,
 481     boolean_t *eio)
 482 {
 483         vm_object_t object = mc[0]->object;
 484         int pageout_status[count];
 485         int numpagedout = 0;
 486         int i, runlen;
 487
 488         VM_OBJECT_ASSERT_WLOCKED(object);
 489
 490         /*
 491          * Initiate I/O.  Bump the vm_page_t->busy counter and
 492          * mark the pages read-only.
 493          *
 494          * We do not have to fixup the clean/dirty bits here... we can
 495          * allow the pager to do it after the I/O completes.
 496          *
 497          * NOTE! mc[i]->dirty may be partial or fragmented due to an
 498          * edge case with file fragments.
 499          */
 500         for (i = 0; i < count; i++) {
 501                 KASSERT(mc[i]->valid == VM_PAGE_BITS_ALL,
 502                     ("vm_pageout_flush: partially invalid page %p index %d/%d",
 503                         mc[i], i, count));
 504                 vm_page_sbusy(mc[i]);
 505                 pmap_remove_write(mc[i]);
 506         }
 507         vm_object_pip_add(object, count);
 508
 509         vm_pager_put_pages(object, mc, count, flags, pageout_status);
 510
 511         runlen = count - mreq;
 512         if (eio != NULL)
 513                 *eio = FALSE;
 514         for (i = 0; i < count; i++) {
 515                 vm_page_t mt = mc[i];
 516
 517                 KASSERT(pageout_status[i] == VM_PAGER_PEND ||
 518                     !pmap_page_is_write_mapped(mt),
 519                     ("vm_pageout_flush: page %p is not write protected", mt));
 520                 switch (pageout_status[i]) {
 521                 case VM_PAGER_OK:
 522                 case VM_PAGER_PEND:
 523                         numpagedout++;
 524                         break;
 525                 case VM_PAGER_BAD:
 526                         /*
 527                          * Page outside of range of object. Right now we
 528                          * essentially lose the changes by pretending it
 529                          * worked.
 530                          */
 531                         vm_page_undirty(mt);
 532                         break;
 533                 case VM_PAGER_ERROR:
 534                 case VM_PAGER_FAIL:
 535                         /*
 536                          * If page couldn't be paged out, then reactivate the
 537                          * page so it doesn't clog the inactive list.  (We
 538                          * will try paging out it again later).
 539                          */
 540                         vm_page_lock(mt);
 541                         vm_page_activate(mt);
 542                         vm_page_unlock(mt);
 543                         if (eio != NULL && i >= mreq && i - mreq < runlen)
 544                                 *eio = TRUE;
 545                         break;
 546                 case VM_PAGER_AGAIN:
 547                         if (i >= mreq && i - mreq < runlen)
 548                                 runlen = i - mreq;
 549                         break;
 550                 }
 551
 552                 /*
 553                  * If the operation is still going, leave the page busy to
 554                  * block all other accesses. Also, leave the paging in
 555                  * progress indicator set so that we don't attempt an object
 556                  * collapse.
 557                  */
 558                 if (pageout_status[i] != VM_PAGER_PEND) {
 559                         vm_object_pip_wakeup(object);
 560                         vm_page_sunbusy(mt);
 561                 }
 562         }
 563         if (prunlen != NULL)
 564                 *prunlen = runlen;
 565         return (numpagedout);
 566 }
 567
 568 #if !defined(NO_SWAPPING)
 569 /*
 570  *      vm_pageout_object_deactivate_pages
 571  *
 572  *      Deactivate enough pages to satisfy the inactive target
 573  *      requirements.
 574  *
 575  *      The object and map must be locked.
 576  */
 577 static void
 578 vm_pageout_object_deactivate_pages(pmap_t pmap, vm_object_t first_object,
 579     long desired)
 580 {
 581         vm_object_t backing_object, object;
 582         vm_page_t p;
 583         int act_delta, remove_mode;
 584
 585         VM_OBJECT_ASSERT_LOCKED(first_object);
 586         if ((first_object->flags & OBJ_FICTITIOUS) != 0)
 587                 return;
 588         for (object = first_object;; object = backing_object) {
 589                 if (pmap_resident_count(pmap) <= desired)
 590                         goto unlock_return;
 591                 VM_OBJECT_ASSERT_LOCKED(object);
 592                 if ((object->flags & OBJ_UNMANAGED) != 0 ||
 593                     object->paging_in_progress != 0)
 594                         goto unlock_return;
 595
 596                 remove_mode = 0;
 597                 if (object->shadow_count > 1)
 598                         remove_mode = 1;
 599                 /*
 600                  * Scan the object's entire memory queue.
 601                  */
 602                 TAILQ_FOREACH(p, &object->memq, listq) {
 603                         if (pmap_resident_count(pmap) <= desired)
 604                                 goto unlock_return;
 605                         if (vm_page_busied(p))
 606                                 continue;
 607                         PCPU_INC(cnt.v_pdpages);
 608                         vm_page_lock(p);
 609                         if (p->wire_count != 0 || p->hold_count != 0 ||
 610                             !pmap_page_exists_quick(pmap, p)) {
 611                                 vm_page_unlock(p);
 612                                 continue;
 613                         }
 614                         act_delta = pmap_ts_referenced(p);
 615                         if ((p->aflags & PGA_REFERENCED) != 0) {
 616                                 if (act_delta == 0)
 617                                         act_delta = 1;
 618                                 vm_page_aflag_clear(p, PGA_REFERENCED);
 619                         }
 620                         if (p->queue != PQ_ACTIVE && act_delta != 0) {
 621                                 vm_page_activate(p);
 622                                 p->act_count += act_delta;
 623                         } else if (p->queue == PQ_ACTIVE) {
 624                                 if (act_delta == 0) {
 625                                         p->act_count -= min(p->act_count,
 626                                             ACT_DECLINE);
 627                                         if (!remove_mode && p->act_count == 0) {
 628                                                 pmap_remove_all(p);
 629                                                 vm_page_deactivate(p);
 630                                         } else
 631                                                 vm_page_requeue(p);
 632                                 } else {
 633                                         vm_page_activate(p);
 634                                         if (p->act_count < ACT_MAX -
 635                                             ACT_ADVANCE)
 636                                                 p->act_count += ACT_ADVANCE;
 637                                         vm_page_requeue(p);
 638                                 }
 639                         } else if (p->queue == PQ_INACTIVE)
 640                                 pmap_remove_all(p);
 641                         vm_page_unlock(p);
 642                 }
 643                 if ((backing_object = object->backing_object) == NULL)
 644                         goto unlock_return;
 645                 VM_OBJECT_RLOCK(backing_object);
 646                 if (object != first_object)
 647                         VM_OBJECT_RUNLOCK(object);
 648         }
 649 unlock_return:
 650         if (object != first_object)
 651                 VM_OBJECT_RUNLOCK(object);
 652 }
 653
 654 /*
 655  * deactivate some number of pages in a map, try to do it fairly, but
 656  * that is really hard to do.
 657  */
 658 static void
 659 vm_pageout_map_deactivate_pages(map, desired)
 660         vm_map_t map;
 661         long desired;
 662 {
 663         vm_map_entry_t tmpe;
 664         vm_object_t obj, bigobj;
 665         int nothingwired;
 666
 667         if (!vm_map_trylock(map))
 668                 return;
 669
 670         bigobj = NULL;
 671         nothingwired = TRUE;
 672
 673         /*
 674          * first, search out the biggest object, and try to free pages from
 675          * that.
 676          */
 677         tmpe = map->header.next;
 678         while (tmpe != &map->header) {
 679                 if ((tmpe->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
 680                         obj = tmpe->object.vm_object;
 681                         if (obj != NULL && VM_OBJECT_TRYRLOCK(obj)) {
 682                                 if (obj->shadow_count <= 1 &&
 683                                     (bigobj == NULL ||
 684                                      bigobj->resident_page_count < obj->resident_page_count)) {
 685                                         if (bigobj != NULL)
 686                                                 VM_OBJECT_RUNLOCK(bigobj);
 687                                         bigobj = obj;
 688                                 } else
 689                                         VM_OBJECT_RUNLOCK(obj);
 690                         }
 691                 }
 692                 if (tmpe->wired_count > 0)
 693                         nothingwired = FALSE;
 694                 tmpe = tmpe->next;
 695         }
 696
 697         if (bigobj != NULL) {
 698                 vm_pageout_object_deactivate_pages(map->pmap, bigobj, desired);
 699                 VM_OBJECT_RUNLOCK(bigobj);
 700         }
 701         /*
 702          * Next, hunt around for other pages to deactivate.  We actually
 703          * do this search sort of wrong -- .text first is not the best idea.
 704          */
 705         tmpe = map->header.next;
 706         while (tmpe != &map->header) {
 707                 if (pmap_resident_count(vm_map_pmap(map)) <= desired)
 708                         break;
 709                 if ((tmpe->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
 710                         obj = tmpe->object.vm_object;
 711                         if (obj != NULL) {
 712                                 VM_OBJECT_RLOCK(obj);
 713                                 vm_pageout_object_deactivate_pages(map->pmap, obj, desired);
 714                                 VM_OBJECT_RUNLOCK(obj);
 715                         }
 716                 }
 717                 tmpe = tmpe->next;
 718         }
 719
 720         /*
 721          * Remove all mappings if a process is swapped out, this will free page
 722          * table pages.
 723          */
 724         if (desired == 0 && nothingwired) {
 725                 pmap_remove(vm_map_pmap(map), vm_map_min(map),
 726                     vm_map_max(map));
 727         }
 728
 729         vm_map_unlock(map);
 730 }
 731 #endif          /* !defined(NO_SWAPPING) */
 732
 733 /*
 734  * Attempt to acquire all of the necessary locks to launder a page and
 735  * then call through the clustering layer to PUTPAGES.  Wait a short
 736  * time for a vnode lock.
 737  *
 738  * Requires the page and object lock on entry, releases both before return.
 739  * Returns 0 on success and an errno otherwise.
 740  */
 741 static int
 742 vm_pageout_clean(vm_page_t m)
 743 {
 744         struct vnode *vp;
 745         struct mount *mp;
 746         vm_object_t object;
 747         vm_pindex_t pindex;
 748         int error, lockmode;
 749
 750         vm_page_assert_locked(m);
 751         object = m->object;
 752         VM_OBJECT_ASSERT_WLOCKED(object);
 753         error = 0;
 754         vp = NULL;
 755         mp = NULL;
 756
 757         /*
 758          * The object is already known NOT to be dead.   It
 759          * is possible for the vget() to block the whole
 760          * pageout daemon, but the new low-memory handling
 761          * code should prevent it.
 762          *
 763          * We can't wait forever for the vnode lock, we might
 764          * deadlock due to a vn_read() getting stuck in
 765          * vm_wait while holding this vnode.  We skip the
 766          * vnode if we can't get it in a reasonable amount
 767          * of time.
 768          */
 769         if (object->type == OBJT_VNODE) {
 770                 vm_page_unlock(m);
 771                 vp = object->handle;
 772                 if (vp->v_type == VREG &&
 773                     vn_start_write(vp, &mp, V_NOWAIT) != 0) {
 774                         mp = NULL;
 775                         error = EDEADLK;
 776                         goto unlock_all;
 777                 }
 778                 KASSERT(mp != NULL,
 779                     ("vp %p with NULL v_mount", vp));
 780                 vm_object_reference_locked(object);
 781                 pindex = m->pindex;
 782                 VM_OBJECT_WUNLOCK(object);
 783                 lockmode = MNT_SHARED_WRITES(vp->v_mount) ?
 784                     LK_SHARED : LK_EXCLUSIVE;
 785                 if (vget(vp, lockmode | LK_TIMELOCK, curthread)) {
 786                         vp = NULL;
 787                         error = EDEADLK;
 788                         goto unlock_mp;
 789                 }
 790                 VM_OBJECT_WLOCK(object);
 791                 vm_page_lock(m);
 792                 /*
 793                  * While the object and page were unlocked, the page
 794                  * may have been:
 795                  * (1) moved to a different queue,
 796                  * (2) reallocated to a different object,
 797                  * (3) reallocated to a different offset, or
 798                  * (4) cleaned.
 799                  */
 800                 if (m->queue != PQ_INACTIVE || m->object != object ||
 801                     m->pindex != pindex || m->dirty == 0) {
 802                         vm_page_unlock(m);
 803                         error = ENXIO;
 804                         goto unlock_all;
 805                 }
 806
 807                 /*
 808                  * The page may have been busied or held while the object
 809                  * and page locks were released.
 810                  */
 811                 if (vm_page_busied(m) || m->hold_count != 0) {
 812                         vm_page_unlock(m);
 813                         error = EBUSY;
 814                         goto unlock_all;
 815                 }
 816         }
 817
 818         /*
 819          * If a page is dirty, then it is either being washed
 820          * (but not yet cleaned) or it is still in the
 821          * laundry.  If it is still in the laundry, then we
 822          * start the cleaning operation.
 823          */
 824         if (vm_pageout_cluster(m) == 0)
 825                 error = EIO;
 826
 827 unlock_all:
 828         VM_OBJECT_WUNLOCK(object);
 829
 830 unlock_mp:
 831         vm_page_lock_assert(m, MA_NOTOWNED);
 832         if (mp != NULL) {
 833                 if (vp != NULL)
 834                         vput(vp);
 835                 vm_object_deallocate(object);
 836                 vn_finished_write(mp);
 837         }
 838
 839         return (error);
 840 }
 841
 842 /*
 843  *      vm_pageout_scan does the dirty work for the pageout daemon.
 844  *
 845  *      pass 0 - Update active LRU/deactivate pages
 846  *      pass 1 - Free inactive pages
 847  *      pass 2 - Launder dirty pages
 848  */
 849 static void
 850 vm_pageout_scan(struct vm_domain *vmd, int pass)
 851 {
 852         vm_page_t m, next;
 853         struct vm_pagequeue *pq;
 854         vm_object_t object;
 855         long min_scan;
 856         int act_delta, addl_page_shortage, deficit, error, maxlaunder, maxscan;
 857         int page_shortage, scan_tick, scanned, starting_page_shortage;
 858         int vnodes_skipped;
 859         boolean_t pageout_ok, queue_locked;
 860
 861         /*
 862          * If we need to reclaim memory ask kernel caches to return
 863          * some.  We rate limit to avoid thrashing.
 864          */
 865         if (vmd == &vm_dom[0] && pass > 0 &&
 866             (time_uptime - lowmem_uptime) >= lowmem_period) {
 867                 /*
 868                  * Decrease registered cache sizes.
 869                  */
 870                 SDT_PROBE0(vm, , , vm__lowmem_scan);
 871                 EVENTHANDLER_INVOKE(vm_lowmem, 0);
 872                 /*
 873                  * We do this explicitly after the caches have been
 874                  * drained above.
 875                  */
 876                 uma_reclaim();
 877                 lowmem_uptime = time_uptime;
 878         }
 879
 880         /*
 881          * The addl_page_shortage is the number of temporarily
 882          * stuck pages in the inactive queue.  In other words, the
 883          * number of pages from the inactive count that should be
 884          * discounted in setting the target for the active queue scan.
 885          */
 886         addl_page_shortage = 0;
 887
 888         /*
 889          * Calculate the number of pages that we want to free.
 890          */
 891         if (pass > 0) {
 892                 deficit = atomic_readandclear_int(&vm_pageout_deficit);
 893                 page_shortage = vm_paging_target() + deficit;
 894         } else
 895                 page_shortage = deficit = 0;
 896         starting_page_shortage = page_shortage;
 897
 898         /*
 899          * maxlaunder limits the number of dirty pages we flush per scan.
 900          * For most systems a smaller value (16 or 32) is more robust under
 901          * extreme memory and disk pressure because any unnecessary writes
 902          * to disk can result in extreme performance degredation.  However,
 903          * systems with excessive dirty pages (especially when MAP_NOSYNC is
 904          * used) will die horribly with limited laundering.  If the pageout
 905          * daemon cannot clean enough pages in the first pass, we let it go
 906          * all out in succeeding passes.
 907          */
 908         if ((maxlaunder = vm_max_launder) <= 1)
 909                 maxlaunder = 1;
 910         if (pass > 1)
 911                 maxlaunder = 10000;
 912
 913         vnodes_skipped = 0;
 914
 915         /*
 916          * Start scanning the inactive queue for pages that we can free.  The
 917          * scan will stop when we reach the target or we have scanned the
 918          * entire queue.  (Note that m->act_count is not used to make
 919          * decisions for the inactive queue, only for the active queue.)
 920          */
 921         pq = &vmd->vmd_pagequeues[PQ_INACTIVE];
 922         maxscan = pq->pq_cnt;
 923         vm_pagequeue_lock(pq);
 924         queue_locked = TRUE;
 925         for (m = TAILQ_FIRST(&pq->pq_pl);
 926              m != NULL && maxscan-- > 0 && page_shortage > 0;
 927              m = next) {
 928                 vm_pagequeue_assert_locked(pq);
 929                 KASSERT(queue_locked, ("unlocked inactive queue"));
 930                 KASSERT(m->queue == PQ_INACTIVE, ("Inactive queue %p", m));
 931
 932                 PCPU_INC(cnt.v_pdpages);
 933                 next = TAILQ_NEXT(m, plinks.q);
 934
 935                 /*
 936                  * skip marker pages
 937                  */
 938                 if (m->flags & PG_MARKER)
 939                         continue;
 940
 941                 KASSERT((m->flags & PG_FICTITIOUS) == 0,
 942                     ("Fictitious page %p cannot be in inactive queue", m));
 943                 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 944                     ("Unmanaged page %p cannot be in inactive queue", m));
 945
 946                 /*
 947                  * The page or object lock acquisitions fail if the
 948                  * page was removed from the queue or moved to a
 949                  * different position within the queue.  In either
 950                  * case, addl_page_shortage should not be incremented.
 951                  */
 952                 if (!vm_pageout_page_lock(m, &next))
 953                         goto unlock_page;
 954                 else if (m->hold_count != 0) {
 955                         /*
 956                          * Held pages are essentially stuck in the
 957                          * queue.  So, they ought to be discounted
 958                          * from the inactive count.  See the
 959                          * calculation of the page_shortage for the
 960                          * loop over the active queue below.
 961                          */
 962                         addl_page_shortage++;
 963                         goto unlock_page;
 964                 }
 965                 object = m->object;
 966                 if (!VM_OBJECT_TRYWLOCK(object)) {
 967                         if (!vm_pageout_fallback_object_lock(m, &next))
 968                                 goto unlock_object;
 969                         else if (m->hold_count != 0) {
 970                                 addl_page_shortage++;
 971                                 goto unlock_object;
 972                         }
 973                 }
 974                 if (vm_page_busied(m)) {
 975                         /*
 976                          * Don't mess with busy pages.  Leave them at
 977                          * the front of the queue.  Most likely, they
 978                          * are being paged out and will leave the
 979                          * queue shortly after the scan finishes.  So,
 980                          * they ought to be discounted from the
 981                          * inactive count.
 982                          */
 983                         addl_page_shortage++;
 984 unlock_object:
 985                         VM_OBJECT_WUNLOCK(object);
 986 unlock_page:
 987                         vm_page_unlock(m);
 988                         continue;
 989                 }
 990                 KASSERT(m->hold_count == 0, ("Held page %p", m));
 991
 992                 /*
 993                  * We unlock the inactive page queue, invalidating the
 994                  * 'next' pointer.  Use our marker to remember our
 995                  * place.
 996                  */
 997                 TAILQ_INSERT_AFTER(&pq->pq_pl, m, &vmd->vmd_marker, plinks.q);
 998                 vm_pagequeue_unlock(pq);
 999                 queue_locked = FALSE;
1000
1001                 /*
1002                  * Invalid pages can be easily freed. They cannot be
1003                  * mapped, vm_page_free() asserts this.
1004                  */
1005                 if (m->valid == 0)
1006                         goto free_page;
1007
1008                 /*
1009                  * If the page has been referenced and the object is not dead,
1010                  * reactivate or requeue the page depending on whether the
1011                  * object is mapped.
1012                  */
1013                 if ((m->aflags & PGA_REFERENCED) != 0) {
1014                         vm_page_aflag_clear(m, PGA_REFERENCED);
1015                         act_delta = 1;
1016                 } else
1017                         act_delta = 0;
1018                 if (object->ref_count != 0) {
1019                         act_delta += pmap_ts_referenced(m);
1020                 } else {
1021                         KASSERT(!pmap_page_is_mapped(m),
1022                             ("vm_pageout_scan: page %p is mapped", m));
1023                 }
1024                 if (act_delta != 0) {
1025                         if (object->ref_count != 0) {
1026                                 vm_page_activate(m);
1027
1028                                 /*
1029                                  * Increase the activation count if the page
1030                                  * was referenced while in the inactive queue.
1031                                  * This makes it less likely that the page will
1032                                  * be returned prematurely to the inactive
1033                                  * queue.
1034                                  */
1035                                 m->act_count += act_delta + ACT_ADVANCE;
1036                                 goto drop_page;
1037                         } else if ((object->flags & OBJ_DEAD) == 0)
1038                                 goto requeue_page;
1039                 }
1040
1041                 /*
1042                  * If the page appears to be clean at the machine-independent
1043                  * layer, then remove all of its mappings from the pmap in
1044                  * anticipation of freeing it.  If, however, any of the page's
1045                  * mappings allow write access, then the page may still be
1046                  * modified until the last of those mappings are removed.
1047                  */
1048                 if (object->ref_count != 0) {
1049                         vm_page_test_dirty(m);
1050                         if (m->dirty == 0)
1051                                 pmap_remove_all(m);
1052                 }
1053
1054                 if (m->dirty == 0) {
1055                         /*
1056                          * Clean pages can be freed.
1057                          */
1058 free_page:
1059                         vm_page_free(m);
1060                         PCPU_INC(cnt.v_dfree);
1061                         --page_shortage;
1062                 } else if ((object->flags & OBJ_DEAD) != 0) {
1063                         /*
1064                          * Leave dirty pages from dead objects at the front of
1065                          * the queue.  They are being paged out and freed by
1066                          * the thread that destroyed the object.  They will
1067                          * leave the queue shortly after the scan finishes, so
1068                          * they should be discounted from the inactive count.
1069                          */
1070                         addl_page_shortage++;
1071                 } else if ((m->flags & PG_WINATCFLS) == 0 && pass < 2) {
1072                         /*
1073                          * Dirty pages need to be paged out, but flushing
1074                          * a page is extremely expensive versus freeing
1075                          * a clean page.  Rather then artificially limiting
1076                          * the number of pages we can flush, we instead give
1077                          * dirty pages extra priority on the inactive queue
1078                          * by forcing them to be cycled through the queue
1079                          * twice before being flushed, after which the
1080                          * (now clean) page will cycle through once more
1081                          * before being freed.  This significantly extends
1082                          * the thrash point for a heavily loaded machine.
1083                          */
1084                         m->flags |= PG_WINATCFLS;
1085 requeue_page:
1086                         vm_pagequeue_lock(pq);
1087                         queue_locked = TRUE;
1088                         vm_page_requeue_locked(m);
1089                 } else if (maxlaunder > 0) {
1090                         /*
1091                          * We always want to try to flush some dirty pages if
1092                          * we encounter them, to keep the system stable.
1093                          * Normally this number is small, but under extreme
1094                          * pressure where there are insufficient clean pages
1095                          * on the inactive queue, we may have to go all out.
1096                          */
1097
1098                         if (object->type != OBJT_SWAP &&
1099                             object->type != OBJT_DEFAULT)
1100                                 pageout_ok = TRUE;
1101                         else if (disable_swap_pageouts)
1102                                 pageout_ok = FALSE;
1103                         else if (defer_swap_pageouts)
1104                                 pageout_ok = vm_page_count_min();
1105                         else
1106                                 pageout_ok = TRUE;
1107                         if (!pageout_ok)
1108                                 goto requeue_page;
1109                         error = vm_pageout_clean(m);
1110                         /*
1111                          * Decrement page_shortage on success to account for
1112                          * the (future) cleaned page.  Otherwise we could wind
1113                          * up laundering or cleaning too many pages.
1114                          */
1115                         if (error == 0) {
1116                                 page_shortage--;
1117                                 maxlaunder--;
1118                         } else if (error == EDEADLK) {
1119                                 pageout_lock_miss++;
1120                                 vnodes_skipped++;
1121                         } else if (error == EBUSY) {
1122                                 addl_page_shortage++;
1123                         }
1124                         vm_page_lock_assert(m, MA_NOTOWNED);
1125                         goto relock_queue;
1126                 }
1127 drop_page:
1128                 vm_page_unlock(m);
1129                 VM_OBJECT_WUNLOCK(object);
1130 relock_queue:
1131                 if (!queue_locked) {
1132                         vm_pagequeue_lock(pq);
1133                         queue_locked = TRUE;
1134                 }
1135                 next = TAILQ_NEXT(&vmd->vmd_marker, plinks.q);
1136                 TAILQ_REMOVE(&pq->pq_pl, &vmd->vmd_marker, plinks.q);
1137         }
1138         vm_pagequeue_unlock(pq);
1139
1140 #if !defined(NO_SWAPPING)
1141         /*
1142          * Wakeup the swapout daemon if we didn't free the targeted number of
1143          * pages.
1144          */
1145         if (vm_swap_enabled && page_shortage > 0)
1146                 vm_req_vmdaemon(VM_SWAP_NORMAL);
1147 #endif
1148
1149         /*
1150          * Wakeup the sync daemon if we skipped a vnode in a writeable object
1151          * and we didn't free enough pages.
1152          */
1153         if (vnodes_skipped > 0 && page_shortage > vm_cnt.v_free_target -
1154             vm_cnt.v_free_min)
1155                 (void)speedup_syncer();
1156
1157         /*
1158          * If the inactive queue scan fails repeatedly to meet its
1159          * target, kill the largest process.
1160          */
1161         vm_pageout_mightbe_oom(vmd, page_shortage, starting_page_shortage);
1162
1163         /*
1164          * Compute the number of pages we want to try to move from the
1165          * active queue to the inactive queue.
1166          */
1167         page_shortage = vm_cnt.v_inactive_target - vm_cnt.v_inactive_count +
1168             vm_paging_target() + deficit + addl_page_shortage;
1169
1170         pq = &vmd->vmd_pagequeues[PQ_ACTIVE];
1171         vm_pagequeue_lock(pq);
1172         maxscan = pq->pq_cnt;
1173
1174         /*
1175          * If we're just idle polling attempt to visit every
1176          * active page within 'update_period' seconds.
1177          */
1178         scan_tick = ticks;
1179         if (vm_pageout_update_period != 0) {
1180                 min_scan = pq->pq_cnt;
1181                 min_scan *= scan_tick - vmd->vmd_last_active_scan;
1182                 min_scan /= hz * vm_pageout_update_period;
1183         } else
1184                 min_scan = 0;
1185         if (min_scan > 0 || (page_shortage > 0 && maxscan > 0))
1186                 vmd->vmd_last_active_scan = scan_tick;
1187
1188         /*
1189          * Scan the active queue for pages that can be deactivated.  Update
1190          * the per-page activity counter and use it to identify deactivation
1191          * candidates.  Held pages may be deactivated.
1192          */
1193         for (m = TAILQ_FIRST(&pq->pq_pl), scanned = 0; m != NULL && (scanned <
1194             min_scan || (page_shortage > 0 && scanned < maxscan)); m = next,
1195             scanned++) {
1196                 KASSERT(m->queue == PQ_ACTIVE,
1197                     ("vm_pageout_scan: page %p isn't active", m));
1198                 next = TAILQ_NEXT(m, plinks.q);
1199                 if ((m->flags & PG_MARKER) != 0)
1200                         continue;
1201                 KASSERT((m->flags & PG_FICTITIOUS) == 0,
1202                     ("Fictitious page %p cannot be in active queue", m));
1203                 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
1204                     ("Unmanaged page %p cannot be in active queue", m));
1205                 if (!vm_pageout_page_lock(m, &next)) {
1206                         vm_page_unlock(m);
1207                         continue;
1208                 }
1209
1210                 /*
1211                  * The count for page daemon pages is updated after checking
1212                  * the page for eligibility.
1213                  */
1214                 PCPU_INC(cnt.v_pdpages);
1215
1216                 /*
1217                  * Check to see "how much" the page has been used.
1218                  */
1219                 if ((m->aflags & PGA_REFERENCED) != 0) {
1220                         vm_page_aflag_clear(m, PGA_REFERENCED);
1221                         act_delta = 1;
1222                 } else
1223                         act_delta = 0;
1224
1225                 /*
1226                  * Perform an unsynchronized object ref count check.  While
1227                  * the page lock ensures that the page is not reallocated to
1228                  * another object, in particular, one with unmanaged mappings
1229                  * that cannot support pmap_ts_referenced(), two races are,
1230                  * nonetheless, possible:
1231                  * 1) The count was transitioning to zero, but we saw a non-
1232                  *    zero value.  pmap_ts_referenced() will return zero
1233                  *    because the page is not mapped.
1234                  * 2) The count was transitioning to one, but we saw zero.
1235                  *    This race delays the detection of a new reference.  At
1236                  *    worst, we will deactivate and reactivate the page.
1237                  */
1238                 if (m->object->ref_count != 0)
1239                         act_delta += pmap_ts_referenced(m);
1240
1241                 /*
1242                  * Advance or decay the act_count based on recent usage.
1243                  */
1244                 if (act_delta != 0) {
1245                         m->act_count += ACT_ADVANCE + act_delta;
1246                         if (m->act_count > ACT_MAX)
1247                                 m->act_count = ACT_MAX;
1248                 } else
1249                         m->act_count -= min(m->act_count, ACT_DECLINE);
1250
1251                 /*
1252                  * Move this page to the tail of the active or inactive
1253                  * queue depending on usage.
1254                  */
1255                 if (m->act_count == 0) {
1256                         /* Dequeue to avoid later lock recursion. */
1257                         vm_page_dequeue_locked(m);
1258                         vm_page_deactivate(m);
1259                         page_shortage--;
1260                 } else
1261                         vm_page_requeue_locked(m);
1262                 vm_page_unlock(m);
1263         }
1264         vm_pagequeue_unlock(pq);
1265 #if !defined(NO_SWAPPING)
1266         /*
1267          * Idle process swapout -- run once per second when we are reclaiming
1268          * pages.
1269          */
1270         if (vm_swap_idle_enabled && pass > 0) {
1271                 static long lsec;
1272                 if (time_second != lsec) {
1273                         vm_req_vmdaemon(VM_SWAP_IDLE);
1274                         lsec = time_second;
1275                 }
1276         }
1277 #endif
1278 }
1279
1280 static int vm_pageout_oom_vote;
1281
1282 /*
1283  * The pagedaemon threads randlomly select one to perform the
1284  * OOM.  Trying to kill processes before all pagedaemons
1285  * failed to reach free target is premature.
1286  */
1287 static void
1288 vm_pageout_mightbe_oom(struct vm_domain *vmd, int page_shortage,
1289     int starting_page_shortage)
1290 {
1291         int old_vote;
1292
1293         if (starting_page_shortage <= 0 || starting_page_shortage !=
1294             page_shortage)
1295                 vmd->vmd_oom_seq = 0;
1296         else
1297                 vmd->vmd_oom_seq++;
1298         if (vmd->vmd_oom_seq < vm_pageout_oom_seq) {
1299                 if (vmd->vmd_oom) {
1300                         vmd->vmd_oom = FALSE;
1301                         atomic_subtract_int(&vm_pageout_oom_vote, 1);
1302                 }
1303                 return;
1304         }
1305
1306         /*
1307          * Do not follow the call sequence until OOM condition is
1308          * cleared.
1309          */
1310         vmd->vmd_oom_seq = 0;
1311
1312         if (vmd->vmd_oom)
1313                 return;
1314
1315         vmd->vmd_oom = TRUE;
1316         old_vote = atomic_fetchadd_int(&vm_pageout_oom_vote, 1);
1317         if (old_vote != vm_ndomains - 1)
1318                 return;
1319
1320         /*
1321          * The current pagedaemon thread is the last in the quorum to
1322          * start OOM.  Initiate the selection and signaling of the
1323          * victim.
1324          */
1325         vm_pageout_oom(VM_OOM_MEM);
1326
1327         /*
1328          * After one round of OOM terror, recall our vote.  On the
1329          * next pass, current pagedaemon would vote again if the low
1330          * memory condition is still there, due to vmd_oom being
1331          * false.
1332          */
1333         vmd->vmd_oom = FALSE;
1334         atomic_subtract_int(&vm_pageout_oom_vote, 1);
1335 }
1336
1337 /*
1338  * The OOM killer is the page daemon's action of last resort when
1339  * memory allocation requests have been stalled for a prolonged period
1340  * of time because it cannot reclaim memory.  This function computes
1341  * the approximate number of physical pages that could be reclaimed if
1342  * the specified address space is destroyed.
1343  *
1344  * Private, anonymous memory owned by the address space is the
1345  * principal resource that we expect to recover after an OOM kill.
1346  * Since the physical pages mapped by the address space's COW entries
1347  * are typically shared pages, they are unlikely to be released and so
1348  * they are not counted.
1349  *
1350  * To get to the point where the page daemon runs the OOM killer, its
1351  * efforts to write-back vnode-backed pages may have stalled.  This
1352  * could be caused by a memory allocation deadlock in the write path
1353  * that might be resolved by an OOM kill.  Therefore, physical pages
1354  * belonging to vnode-backed objects are counted, because they might
1355  * be freed without being written out first if the address space holds
1356  * the last reference to an unlinked vnode.
1357  *
1358  * Similarly, physical pages belonging to OBJT_PHYS objects are
1359  * counted because the address space might hold the last reference to
1360  * the object.
1361  */
1362 static long
1363 vm_pageout_oom_pagecount(struct vmspace *vmspace)
1364 {
1365         vm_map_t map;
1366         vm_map_entry_t entry;
1367         vm_object_t obj;
1368         long res;
1369
1370         map = &vmspace->vm_map;
1371         KASSERT(!map->system_map, ("system map"));
1372         sx_assert(&map->lock, SA_LOCKED);
1373         res = 0;
1374         for (entry = map->header.next; entry != &map->header;
1375             entry = entry->next) {
1376                 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0)
1377                         continue;
1378                 obj = entry->object.vm_object;
1379                 if (obj == NULL)
1380                         continue;
1381                 if ((entry->eflags & MAP_ENTRY_NEEDS_COPY) != 0 &&
1382                     obj->ref_count != 1)
1383                         continue;
1384                 switch (obj->type) {
1385                 case OBJT_DEFAULT:
1386                 case OBJT_SWAP:
1387                 case OBJT_PHYS:
1388                 case OBJT_VNODE:
1389                         res += obj->resident_page_count;
1390                         break;
1391                 }
1392         }
1393         return (res);
1394 }
1395
1396 void
1397 vm_pageout_oom(int shortage)
1398 {
1399         struct proc *p, *bigproc;
1400         vm_offset_t size, bigsize;
1401         struct thread *td;
1402         struct vmspace *vm;
1403
1404         /*
1405          * We keep the process bigproc locked once we find it to keep anyone
1406          * from messing with it; however, there is a possibility of
1407          * deadlock if process B is bigproc and one of it's child processes
1408          * attempts to propagate a signal to B while we are waiting for A's
1409          * lock while walking this list.  To avoid this, we don't block on
1410          * the process lock but just skip a process if it is already locked.
1411          */
1412         bigproc = NULL;
1413         bigsize = 0;
1414         sx_slock(&allproc_lock);
1415         FOREACH_PROC_IN_SYSTEM(p) {
1416                 int breakout;
1417
1418                 PROC_LOCK(p);
1419
1420                 /*
1421                  * If this is a system, protected or killed process, skip it.
1422                  */
1423                 if (p->p_state != PRS_NORMAL || (p->p_flag & (P_INEXEC |
1424                     P_PROTECTED | P_SYSTEM | P_WEXIT)) != 0 ||
1425                     p->p_pid == 1 || P_KILLED(p) ||
1426                     (p->p_pid < 48 && swap_pager_avail != 0)) {
1427                         PROC_UNLOCK(p);
1428                         continue;
1429                 }
1430                 /*
1431                  * If the process is in a non-running type state,
1432                  * don't touch it.  Check all the threads individually.
1433                  */
1434                 breakout = 0;
1435                 FOREACH_THREAD_IN_PROC(p, td) {
1436                         thread_lock(td);
1437                         if (!TD_ON_RUNQ(td) &&
1438                             !TD_IS_RUNNING(td) &&
1439                             !TD_IS_SLEEPING(td) &&
1440                             !TD_IS_SUSPENDED(td) &&
1441                             !TD_IS_SWAPPED(td)) {
1442                                 thread_unlock(td);
1443                                 breakout = 1;
1444                                 break;
1445                         }
1446                         thread_unlock(td);
1447                 }
1448                 if (breakout) {
1449                         PROC_UNLOCK(p);
1450                         continue;
1451                 }
1452                 /*
1453                  * get the process size
1454                  */
1455                 vm = vmspace_acquire_ref(p);
1456                 if (vm == NULL) {
1457                         PROC_UNLOCK(p);
1458                         continue;
1459                 }
1460                 _PHOLD_LITE(p);
1461                 PROC_UNLOCK(p);
1462                 sx_sunlock(&allproc_lock);
1463                 if (!vm_map_trylock_read(&vm->vm_map)) {
1464                         vmspace_free(vm);
1465                         sx_slock(&allproc_lock);
1466                         PRELE(p);
1467                         continue;
1468                 }
1469                 size = vmspace_swap_count(vm);
1470                 if (shortage == VM_OOM_MEM)
1471                         size += vm_pageout_oom_pagecount(vm);
1472                 vm_map_unlock_read(&vm->vm_map);
1473                 vmspace_free(vm);
1474                 sx_slock(&allproc_lock);
1475
1476                 /*
1477                  * If this process is bigger than the biggest one,
1478                  * remember it.
1479                  */
1480                 if (size > bigsize) {
1481                         if (bigproc != NULL)
1482                                 PRELE(bigproc);
1483                         bigproc = p;
1484                         bigsize = size;
1485                 } else {
1486                         PRELE(p);
1487                 }
1488         }
1489         sx_sunlock(&allproc_lock);
1490         if (bigproc != NULL) {
1491                 if (vm_panic_on_oom != 0)
1492                         panic("out of swap space");
1493                 PROC_LOCK(bigproc);
1494                 killproc(bigproc, "out of swap space");
1495                 sched_nice(bigproc, PRIO_MIN);
1496                 _PRELE(bigproc);
1497                 PROC_UNLOCK(bigproc);
1498                 wakeup(&vm_cnt.v_free_count);
1499         }
1500 }
1501
1502 static void
1503 vm_pageout_worker(void *arg)
1504 {
1505         struct vm_domain *domain;
1506         int domidx;
1507
1508         domidx = (uintptr_t)arg;
1509         domain = &vm_dom[domidx];
1510
1511         /*
1512          * XXXKIB It could be useful to bind pageout daemon threads to
1513          * the cores belonging to the domain, from which vm_page_array
1514          * is allocated.
1515          */
1516
1517         KASSERT(domain->vmd_segs != 0, ("domain without segments"));
1518         domain->vmd_last_active_scan = ticks;
1519         vm_pageout_init_marker(&domain->vmd_marker, PQ_INACTIVE);
1520         vm_pageout_init_marker(&domain->vmd_inacthead, PQ_INACTIVE);
1521         TAILQ_INSERT_HEAD(&domain->vmd_pagequeues[PQ_INACTIVE].pq_pl,
1522             &domain->vmd_inacthead, plinks.q);
1523
1524         /*
1525          * The pageout daemon worker is never done, so loop forever.
1526          */
1527         while (TRUE) {
1528                 mtx_lock(&vm_page_queue_free_mtx);
1529
1530                 /*
1531                  * Generally, after a level >= 1 scan, if there are enough
1532                  * free pages to wakeup the waiters, then they are already
1533                  * awake.  A call to vm_page_free() during the scan awakened
1534                  * them.  However, in the following case, this wakeup serves
1535                  * to bound the amount of time that a thread might wait.
1536                  * Suppose a thread's call to vm_page_alloc() fails, but
1537                  * before that thread calls VM_WAIT, enough pages are freed by
1538                  * other threads to alleviate the free page shortage.  The
1539                  * thread will, nonetheless, wait until another page is freed
1540                  * or this wakeup is performed.
1541                  */
1542                 if (vm_pages_needed && !vm_page_count_min()) {
1543                         vm_pages_needed = false;
1544                         wakeup(&vm_cnt.v_free_count);
1545                 }
1546
1547                 /*
1548                  * Do not clear vm_pageout_wanted until we reach our target.
1549                  * Otherwise, we may be awakened over and over again, wasting
1550                  * CPU time.
1551                  */
1552                 if (vm_pageout_wanted && !vm_paging_needed())
1553                         vm_pageout_wanted = false;
1554
1555                 /*
1556                  * Might the page daemon receive a wakeup call?
1557                  */
1558                 if (vm_pageout_wanted) {
1559                         /*
1560                          * No.  Either vm_pageout_wanted was set by another
1561                          * thread during the previous scan, which must have
1562                          * been a level 0 scan, or vm_pageout_wanted was
1563                          * already set and the scan failed to free enough
1564                          * pages.  If we haven't yet performed a level >= 2
1565                          * scan (unlimited dirty cleaning), then upgrade the
1566                          * level and scan again now.  Otherwise, sleep a bit
1567                          * and try again later.
1568                          */
1569                         mtx_unlock(&vm_page_queue_free_mtx);
1570                         if (domain->vmd_pass > 1)
1571                                 pause("psleep", hz / 2);
1572                         domain->vmd_pass++;
1573                 } else {
1574                         /*
1575                          * Yes.  Sleep until pages need to be reclaimed or
1576                          * have their reference stats updated.
1577                          */
1578                         if (mtx_sleep(&vm_pageout_wanted,
1579                             &vm_page_queue_free_mtx, PDROP | PVM, "psleep",
1580                             hz) == 0) {
1581                                 PCPU_INC(cnt.v_pdwakeups);
1582                                 domain->vmd_pass = 1;
1583                         } else
1584                                 domain->vmd_pass = 0;
1585                 }
1586
1587                 vm_pageout_scan(domain, domain->vmd_pass);
1588         }
1589 }
1590
1591 /*
1592  *      vm_pageout_init initialises basic pageout daemon settings.
1593  */
1594 static void
1595 vm_pageout_init(void)
1596 {
1597         /*
1598          * Initialize some paging parameters.
1599          */
1600         vm_cnt.v_interrupt_free_min = 2;
1601         if (vm_cnt.v_page_count < 2000)
1602                 vm_pageout_page_count = 8;
1603
1604         /*
1605          * v_free_reserved needs to include enough for the largest
1606          * swap pager structures plus enough for any pv_entry structs
1607          * when paging.
1608          */
1609         if (vm_cnt.v_page_count > 1024)
1610                 vm_cnt.v_free_min = 4 + (vm_cnt.v_page_count - 1024) / 200;
1611         else
1612                 vm_cnt.v_free_min = 4;
1613         vm_cnt.v_pageout_free_min = (2*MAXBSIZE)/PAGE_SIZE +
1614             vm_cnt.v_interrupt_free_min;
1615         vm_cnt.v_free_reserved = vm_pageout_page_count +
1616             vm_cnt.v_pageout_free_min + (vm_cnt.v_page_count / 768);
1617         vm_cnt.v_free_severe = vm_cnt.v_free_min / 2;
1618         vm_cnt.v_free_target = 4 * vm_cnt.v_free_min + vm_cnt.v_free_reserved;
1619         vm_cnt.v_free_min += vm_cnt.v_free_reserved;
1620         vm_cnt.v_free_severe += vm_cnt.v_free_reserved;
1621         vm_cnt.v_inactive_target = (3 * vm_cnt.v_free_target) / 2;
1622         if (vm_cnt.v_inactive_target > vm_cnt.v_free_count / 3)
1623                 vm_cnt.v_inactive_target = vm_cnt.v_free_count / 3;
1624
1625         /*
1626          * Set the default wakeup threshold to be 10% above the minimum
1627          * page limit.  This keeps the steady state out of shortfall.
1628          */
1629         vm_pageout_wakeup_thresh = (vm_cnt.v_free_min / 10) * 11;
1630
1631         /*
1632          * Set interval in seconds for active scan.  We want to visit each
1633          * page at least once every ten minutes.  This is to prevent worst
1634          * case paging behaviors with stale active LRU.
1635          */
1636         if (vm_pageout_update_period == 0)
1637                 vm_pageout_update_period = 600;
1638
1639         /* XXX does not really belong here */
1640         if (vm_page_max_wired == 0)
1641                 vm_page_max_wired = vm_cnt.v_free_count / 3;
1642 }
1643
1644 /*
1645  *     vm_pageout is the high level pageout daemon.
1646  */
1647 static void
1648 vm_pageout(void)
1649 {
1650         int error;
1651 #ifdef VM_NUMA_ALLOC
1652         int i;
1653 #endif
1654
1655         swap_pager_swap_init();
1656 #ifdef VM_NUMA_ALLOC
1657         for (i = 1; i < vm_ndomains; i++) {
1658                 error = kthread_add(vm_pageout_worker, (void *)(uintptr_t)i,
1659                     curproc, NULL, 0, 0, "dom%d", i);
1660                 if (error != 0) {
1661                         panic("starting pageout for domain %d, error %d\n",
1662                             i, error);
1663                 }
1664         }
1665 #endif
1666         error = kthread_add(uma_reclaim_worker, NULL, curproc, NULL,
1667             0, 0, "uma");
1668         if (error != 0)
1669                 panic("starting uma_reclaim helper, error %d\n", error);
1670         vm_pageout_worker((void *)(uintptr_t)0);
1671 }
1672
1673 /*
1674  * Unless the free page queue lock is held by the caller, this function
1675  * should be regarded as advisory.  Specifically, the caller should
1676  * not msleep() on &vm_cnt.v_free_count following this function unless
1677  * the free page queue lock is held until the msleep() is performed.
1678  */
1679 void
1680 pagedaemon_wakeup(void)
1681 {
1682
1683         if (!vm_pageout_wanted && curthread->td_proc != pageproc) {
1684                 vm_pageout_wanted = true;
1685                 wakeup(&vm_pageout_wanted);
1686         }
1687 }
1688
1689 #if !defined(NO_SWAPPING)
1690 static void
1691 vm_req_vmdaemon(int req)
1692 {
1693         static int lastrun = 0;
1694
1695         mtx_lock(&vm_daemon_mtx);
1696         vm_pageout_req_swapout |= req;
1697         if ((ticks > (lastrun + hz)) || (ticks < lastrun)) {
1698                 wakeup(&vm_daemon_needed);
1699                 lastrun = ticks;
1700         }
1701         mtx_unlock(&vm_daemon_mtx);
1702 }
1703
1704 static void
1705 vm_daemon(void)
1706 {
1707         struct rlimit rsslim;
1708         struct proc *p;
1709         struct thread *td;
1710         struct vmspace *vm;
1711         int breakout, swapout_flags, tryagain, attempts;
1712 #ifdef RACCT
1713         uint64_t rsize, ravailable;
1714 #endif
1715
1716         while (TRUE) {
1717                 mtx_lock(&vm_daemon_mtx);
1718                 msleep(&vm_daemon_needed, &vm_daemon_mtx, PPAUSE, "psleep",
1719 #ifdef RACCT
1720                     racct_enable ? hz : 0
1721 #else
1722                     0
1723 #endif
1724                 );
1725                 swapout_flags = vm_pageout_req_swapout;
1726                 vm_pageout_req_swapout = 0;
1727                 mtx_unlock(&vm_daemon_mtx);
1728                 if (swapout_flags)
1729                         swapout_procs(swapout_flags);
1730
1731                 /*
1732                  * scan the processes for exceeding their rlimits or if
1733                  * process is swapped out -- deactivate pages
1734                  */
1735                 tryagain = 0;
1736                 attempts = 0;
1737 again:
1738                 attempts++;
1739                 sx_slock(&allproc_lock);
1740                 FOREACH_PROC_IN_SYSTEM(p) {
1741                         vm_pindex_t limit, size;
1742
1743                         /*
1744                          * if this is a system process or if we have already
1745                          * looked at this process, skip it.
1746                          */
1747                         PROC_LOCK(p);
1748                         if (p->p_state != PRS_NORMAL ||
1749                             p->p_flag & (P_INEXEC | P_SYSTEM | P_WEXIT)) {
1750                                 PROC_UNLOCK(p);
1751                                 continue;
1752                         }
1753                         /*
1754                          * if the process is in a non-running type state,
1755                          * don't touch it.
1756                          */
1757                         breakout = 0;
1758                         FOREACH_THREAD_IN_PROC(p, td) {
1759                                 thread_lock(td);
1760                                 if (!TD_ON_RUNQ(td) &&
1761                                     !TD_IS_RUNNING(td) &&
1762                                     !TD_IS_SLEEPING(td) &&
1763                                     !TD_IS_SUSPENDED(td)) {
1764                                         thread_unlock(td);
1765                                         breakout = 1;
1766                                         break;
1767                                 }
1768                                 thread_unlock(td);
1769                         }
1770                         if (breakout) {
1771                                 PROC_UNLOCK(p);
1772                                 continue;
1773                         }
1774                         /*
1775                          * get a limit
1776                          */
1777                         lim_rlimit_proc(p, RLIMIT_RSS, &rsslim);
1778                         limit = OFF_TO_IDX(
1779                             qmin(rsslim.rlim_cur, rsslim.rlim_max));
1780
1781                         /*
1782                          * let processes that are swapped out really be
1783                          * swapped out set the limit to nothing (will force a
1784                          * swap-out.)
1785                          */
1786                         if ((p->p_flag & P_INMEM) == 0)
1787                                 limit = 0;      /* XXX */
1788                         vm = vmspace_acquire_ref(p);
1789                         _PHOLD_LITE(p);
1790                         PROC_UNLOCK(p);
1791                         if (vm == NULL) {
1792                                 PRELE(p);
1793                                 continue;
1794                         }
1795                         sx_sunlock(&allproc_lock);
1796
1797                         size = vmspace_resident_count(vm);
1798                         if (size >= limit) {
1799                                 vm_pageout_map_deactivate_pages(
1800                                     &vm->vm_map, limit);
1801                         }
1802 #ifdef RACCT
1803                         if (racct_enable) {
1804                                 rsize = IDX_TO_OFF(size);
1805                                 PROC_LOCK(p);
1806                                 racct_set(p, RACCT_RSS, rsize);
1807                                 ravailable = racct_get_available(p, RACCT_RSS);
1808                                 PROC_UNLOCK(p);
1809                                 if (rsize > ravailable) {
1810                                         /*
1811                                          * Don't be overly aggressive; this
1812                                          * might be an innocent process,
1813                                          * and the limit could've been exceeded
1814                                          * by some memory hog.  Don't try
1815                                          * to deactivate more than 1/4th
1816                                          * of process' resident set size.
1817                                          */
1818                                         if (attempts <= 8) {
1819                                                 if (ravailable < rsize -
1820                                                     (rsize / 4)) {
1821                                                         ravailable = rsize -
1822                                                             (rsize / 4);
1823                                                 }
1824                                         }
1825                                         vm_pageout_map_deactivate_pages(
1826                                             &vm->vm_map,
1827                                             OFF_TO_IDX(ravailable));
1828                                         /* Update RSS usage after paging out. */
1829                                         size = vmspace_resident_count(vm);
1830                                         rsize = IDX_TO_OFF(size);
1831                                         PROC_LOCK(p);
1832                                         racct_set(p, RACCT_RSS, rsize);
1833                                         PROC_UNLOCK(p);
1834                                         if (rsize > ravailable)
1835                                                 tryagain = 1;
1836                                 }
1837                         }
1838 #endif
1839                         vmspace_free(vm);
1840                         sx_slock(&allproc_lock);
1841                         PRELE(p);
1842                 }
1843                 sx_sunlock(&allproc_lock);
1844                 if (tryagain != 0 && attempts <= 10)
1845                         goto again;
1846         }
1847 }
1848 #endif                  /* !defined(NO_SWAPPING) */