sys/vm/vm_pageout.c

   1 /*-
   2  * Copyright (c) 1991 Regents of the University of California.
   3  * All rights reserved.
   4  * Copyright (c) 1994 John S. Dyson
   5  * All rights reserved.
   6  * Copyright (c) 1994 David Greenman
   7  * All rights reserved.
   8  * Copyright (c) 2005 Yahoo! Technologies Norway AS
   9  * All rights reserved.
  10  *
  11  * This code is derived from software contributed to Berkeley by
  12  * The Mach Operating System project at Carnegie-Mellon University.
  13  *
  14  * Redistribution and use in source and binary forms, with or without
  15  * modification, are permitted provided that the following conditions
  16  * are met:
  17  * 1. Redistributions of source code must retain the above copyright
  18  *    notice, this list of conditions and the following disclaimer.
  19  * 2. Redistributions in binary form must reproduce the above copyright
  20  *    notice, this list of conditions and the following disclaimer in the
  21  *    documentation and/or other materials provided with the distribution.
  22  * 3. All advertising materials mentioning features or use of this software
  23  *    must display the following acknowledgement:
  24  *      This product includes software developed by the University of
  25  *      California, Berkeley and its contributors.
  26  * 4. Neither the name of the University nor the names of its contributors
  27  *    may be used to endorse or promote products derived from this software
  28  *    without specific prior written permission.
  29  *
  30  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  31  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  32  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  33  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  34  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  35  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  36  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  37  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  38  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  39  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  40  * SUCH DAMAGE.
  41  *
  42  *      from: @(#)vm_pageout.c  7.4 (Berkeley) 5/7/91
  43  *
  44  *
  45  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
  46  * All rights reserved.
  47  *
  48  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
  49  *
  50  * Permission to use, copy, modify and distribute this software and
  51  * its documentation is hereby granted, provided that both the copyright
  52  * notice and this permission notice appear in all copies of the
  53  * software, derivative works or modified versions, and any portions
  54  * thereof, and that both notices appear in supporting documentation.
  55  *
  56  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  57  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
  58  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  59  *
  60  * Carnegie Mellon requests users of this software to return to
  61  *
  62  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  63  *  School of Computer Science
  64  *  Carnegie Mellon University
  65  *  Pittsburgh PA 15213-3890
  66  *
  67  * any improvements or extensions that they make and grant Carnegie the
  68  * rights to redistribute these changes.
  69  */
  70
  71 /*
  72  *      The proverbial page-out daemon.
  73  */
  74
  75 #include <sys/cdefs.h>
  76 __FBSDID("$FreeBSD$");
  77
  78 #include "opt_vm.h"
  79
  80 #include <sys/param.h>
  81 #include <sys/systm.h>
  82 #include <sys/kernel.h>
  83 #include <sys/eventhandler.h>
  84 #include <sys/lock.h>
  85 #include <sys/mutex.h>
  86 #include <sys/proc.h>
  87 #include <sys/kthread.h>
  88 #include <sys/ktr.h>
  89 #include <sys/mount.h>
  90 #include <sys/racct.h>
  91 #include <sys/resourcevar.h>
  92 #include <sys/sched.h>
  93 #include <sys/sdt.h>
  94 #include <sys/signalvar.h>
  95 #include <sys/smp.h>
  96 #include <sys/time.h>
  97 #include <sys/vnode.h>
  98 #include <sys/vmmeter.h>
  99 #include <sys/rwlock.h>
 100 #include <sys/sx.h>
 101 #include <sys/sysctl.h>
 102
 103 #include <vm/vm.h>
 104 #include <vm/vm_param.h>
 105 #include <vm/vm_object.h>
 106 #include <vm/vm_page.h>
 107 #include <vm/vm_map.h>
 108 #include <vm/vm_pageout.h>
 109 #include <vm/vm_pager.h>
 110 #include <vm/vm_phys.h>
 111 #include <vm/swap_pager.h>
 112 #include <vm/vm_extern.h>
 113 #include <vm/uma.h>
 114
 115 /*
 116  * System initialization
 117  */
 118
 119 /* the kernel process "vm_pageout"*/
 120 static void vm_pageout(void);
 121 static void vm_pageout_init(void);
 122 static int vm_pageout_clean(vm_page_t m);
 123 static int vm_pageout_cluster(vm_page_t m);
 124 static void vm_pageout_scan(struct vm_domain *vmd, int pass);
 125 static void vm_pageout_mightbe_oom(struct vm_domain *vmd, int page_shortage,
 126     int starting_page_shortage);
 127
 128 SYSINIT(pagedaemon_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_FIRST, vm_pageout_init,
 129     NULL);
 130
 131 struct proc *pageproc;
 132
 133 static struct kproc_desc page_kp = {
 134         "pagedaemon",
 135         vm_pageout,
 136         &pageproc
 137 };
 138 SYSINIT(pagedaemon, SI_SUB_KTHREAD_PAGE, SI_ORDER_SECOND, kproc_start,
 139     &page_kp);
 140
 141 SDT_PROVIDER_DEFINE(vm);
 142 SDT_PROBE_DEFINE(vm, , , vm__lowmem_cache);
 143 SDT_PROBE_DEFINE(vm, , , vm__lowmem_scan);
 144
 145 #if !defined(NO_SWAPPING)
 146 /* the kernel process "vm_daemon"*/
 147 static void vm_daemon(void);
 148 static struct   proc *vmproc;
 149
 150 static struct kproc_desc vm_kp = {
 151         "vmdaemon",
 152         vm_daemon,
 153         &vmproc
 154 };
 155 SYSINIT(vmdaemon, SI_SUB_KTHREAD_VM, SI_ORDER_FIRST, kproc_start, &vm_kp);
 156 #endif
 157
 158
 159 int vm_pageout_deficit;         /* Estimated number of pages deficit */
 160 int vm_pageout_wakeup_thresh;
 161 static int vm_pageout_oom_seq = 12;
 162 bool vm_pageout_wanted;         /* Event on which pageout daemon sleeps */
 163 bool vm_pages_needed;           /* Are threads waiting for free pages? */
 164
 165 #if !defined(NO_SWAPPING)
 166 static int vm_pageout_req_swapout;      /* XXX */
 167 static int vm_daemon_needed;
 168 static struct mtx vm_daemon_mtx;
 169 /* Allow for use by vm_pageout before vm_daemon is initialized. */
 170 MTX_SYSINIT(vm_daemon, &vm_daemon_mtx, "vm daemon", MTX_DEF);
 171 #endif
 172 static int vm_max_launder = 32;
 173 static int vm_pageout_update_period;
 174 static int defer_swap_pageouts;
 175 static int disable_swap_pageouts;
 176 static int lowmem_period = 10;
 177 static time_t lowmem_uptime;
 178
 179 #if defined(NO_SWAPPING)
 180 static int vm_swap_enabled = 0;
 181 static int vm_swap_idle_enabled = 0;
 182 #else
 183 static int vm_swap_enabled = 1;
 184 static int vm_swap_idle_enabled = 0;
 185 #endif
 186
 187 static int vm_panic_on_oom = 0;
 188
 189 SYSCTL_INT(_vm, OID_AUTO, panic_on_oom,
 190         CTLFLAG_RWTUN, &vm_panic_on_oom, 0,
 191         "panic on out of memory instead of killing the largest process");
 192
 193 SYSCTL_INT(_vm, OID_AUTO, pageout_wakeup_thresh,
 194         CTLFLAG_RW, &vm_pageout_wakeup_thresh, 0,
 195         "free page threshold for waking up the pageout daemon");
 196
 197 SYSCTL_INT(_vm, OID_AUTO, max_launder,
 198         CTLFLAG_RW, &vm_max_launder, 0, "Limit dirty flushes in pageout");
 199
 200 SYSCTL_INT(_vm, OID_AUTO, pageout_update_period,
 201         CTLFLAG_RW, &vm_pageout_update_period, 0,
 202         "Maximum active LRU update period");
 203
 204 SYSCTL_INT(_vm, OID_AUTO, lowmem_period, CTLFLAG_RW, &lowmem_period, 0,
 205         "Low memory callback period");
 206
 207 #if defined(NO_SWAPPING)
 208 SYSCTL_INT(_vm, VM_SWAPPING_ENABLED, swap_enabled,
 209         CTLFLAG_RD, &vm_swap_enabled, 0, "Enable entire process swapout");
 210 SYSCTL_INT(_vm, OID_AUTO, swap_idle_enabled,
 211         CTLFLAG_RD, &vm_swap_idle_enabled, 0, "Allow swapout on idle criteria");
 212 #else
 213 SYSCTL_INT(_vm, VM_SWAPPING_ENABLED, swap_enabled,
 214         CTLFLAG_RW, &vm_swap_enabled, 0, "Enable entire process swapout");
 215 SYSCTL_INT(_vm, OID_AUTO, swap_idle_enabled,
 216         CTLFLAG_RW, &vm_swap_idle_enabled, 0, "Allow swapout on idle criteria");
 217 #endif
 218
 219 SYSCTL_INT(_vm, OID_AUTO, defer_swapspace_pageouts,
 220         CTLFLAG_RW, &defer_swap_pageouts, 0, "Give preference to dirty pages in mem");
 221
 222 SYSCTL_INT(_vm, OID_AUTO, disable_swapspace_pageouts,
 223         CTLFLAG_RW, &disable_swap_pageouts, 0, "Disallow swapout of dirty pages");
 224
 225 static int pageout_lock_miss;
 226 SYSCTL_INT(_vm, OID_AUTO, pageout_lock_miss,
 227         CTLFLAG_RD, &pageout_lock_miss, 0, "vget() lock misses during pageout");
 228
 229 SYSCTL_INT(_vm, OID_AUTO, pageout_oom_seq,
 230         CTLFLAG_RW, &vm_pageout_oom_seq, 0,
 231         "back-to-back calls to oom detector to start OOM");
 232
 233 #define VM_PAGEOUT_PAGE_COUNT 16
 234 int vm_pageout_page_count = VM_PAGEOUT_PAGE_COUNT;
 235
 236 int vm_page_max_wired;          /* XXX max # of wired pages system-wide */
 237 SYSCTL_INT(_vm, OID_AUTO, max_wired,
 238         CTLFLAG_RW, &vm_page_max_wired, 0, "System-wide limit to wired page count");
 239
 240 static boolean_t vm_pageout_fallback_object_lock(vm_page_t, vm_page_t *);
 241 #if !defined(NO_SWAPPING)
 242 static void vm_pageout_map_deactivate_pages(vm_map_t, long);
 243 static void vm_pageout_object_deactivate_pages(pmap_t, vm_object_t, long);
 244 static void vm_req_vmdaemon(int req);
 245 #endif
 246 static boolean_t vm_pageout_page_lock(vm_page_t, vm_page_t *);
 247
 248 /*
 249  * Initialize a dummy page for marking the caller's place in the specified
 250  * paging queue.  In principle, this function only needs to set the flag
 251  * PG_MARKER.  Nonetheless, it wirte busies and initializes the hold count
 252  * to one as safety precautions.
 253  */
 254 static void
 255 vm_pageout_init_marker(vm_page_t marker, u_short queue)
 256 {
 257
 258         bzero(marker, sizeof(*marker));
 259         marker->flags = PG_MARKER;
 260         marker->busy_lock = VPB_SINGLE_EXCLUSIVER;
 261         marker->queue = queue;
 262         marker->hold_count = 1;
 263 }
 264
 265 /*
 266  * vm_pageout_fallback_object_lock:
 267  *
 268  * Lock vm object currently associated with `m'. VM_OBJECT_TRYWLOCK is
 269  * known to have failed and page queue must be either PQ_ACTIVE or
 270  * PQ_INACTIVE.  To avoid lock order violation, unlock the page queues
 271  * while locking the vm object.  Use marker page to detect page queue
 272  * changes and maintain notion of next page on page queue.  Return
 273  * TRUE if no changes were detected, FALSE otherwise.  vm object is
 274  * locked on return.
 275  *
 276  * This function depends on both the lock portion of struct vm_object
 277  * and normal struct vm_page being type stable.
 278  */
 279 static boolean_t
 280 vm_pageout_fallback_object_lock(vm_page_t m, vm_page_t *next)
 281 {
 282         struct vm_page marker;
 283         struct vm_pagequeue *pq;
 284         boolean_t unchanged;
 285         u_short queue;
 286         vm_object_t object;
 287
 288         queue = m->queue;
 289         vm_pageout_init_marker(&marker, queue);
 290         pq = vm_page_pagequeue(m);
 291         object = m->object;
 292
 293         TAILQ_INSERT_AFTER(&pq->pq_pl, m, &marker, plinks.q);
 294         vm_pagequeue_unlock(pq);
 295         vm_page_unlock(m);
 296         VM_OBJECT_WLOCK(object);
 297         vm_page_lock(m);
 298         vm_pagequeue_lock(pq);
 299
 300         /*
 301          * The page's object might have changed, and/or the page might
 302          * have moved from its original position in the queue.  If the
 303          * page's object has changed, then the caller should abandon
 304          * processing the page because the wrong object lock was
 305          * acquired.  Use the marker's plinks.q, not the page's, to
 306          * determine if the page has been moved.  The state of the
 307          * page's plinks.q can be indeterminate; whereas, the marker's
 308          * plinks.q must be valid.
 309          */
 310         *next = TAILQ_NEXT(&marker, plinks.q);
 311         unchanged = m->object == object &&
 312             m == TAILQ_PREV(&marker, pglist, plinks.q);
 313         KASSERT(!unchanged || m->queue == queue,
 314             ("page %p queue %d %d", m, queue, m->queue));
 315         TAILQ_REMOVE(&pq->pq_pl, &marker, plinks.q);
 316         return (unchanged);
 317 }
 318
 319 /*
 320  * Lock the page while holding the page queue lock.  Use marker page
 321  * to detect page queue changes and maintain notion of next page on
 322  * page queue.  Return TRUE if no changes were detected, FALSE
 323  * otherwise.  The page is locked on return. The page queue lock might
 324  * be dropped and reacquired.
 325  *
 326  * This function depends on normal struct vm_page being type stable.
 327  */
 328 static boolean_t
 329 vm_pageout_page_lock(vm_page_t m, vm_page_t *next)
 330 {
 331         struct vm_page marker;
 332         struct vm_pagequeue *pq;
 333         boolean_t unchanged;
 334         u_short queue;
 335
 336         vm_page_lock_assert(m, MA_NOTOWNED);
 337         if (vm_page_trylock(m))
 338                 return (TRUE);
 339
 340         queue = m->queue;
 341         vm_pageout_init_marker(&marker, queue);
 342         pq = vm_page_pagequeue(m);
 343
 344         TAILQ_INSERT_AFTER(&pq->pq_pl, m, &marker, plinks.q);
 345         vm_pagequeue_unlock(pq);
 346         vm_page_lock(m);
 347         vm_pagequeue_lock(pq);
 348
 349         /* Page queue might have changed. */
 350         *next = TAILQ_NEXT(&marker, plinks.q);
 351         unchanged = m == TAILQ_PREV(&marker, pglist, plinks.q);
 352         KASSERT(!unchanged || m->queue == queue,
 353             ("page %p queue %d %d", m, queue, m->queue));
 354         TAILQ_REMOVE(&pq->pq_pl, &marker, plinks.q);
 355         return (unchanged);
 356 }
 357
 358 /*
 359  * vm_pageout_clean:
 360  *
 361  * Clean the page and remove it from the laundry.
 362  *
 363  * We set the busy bit to cause potential page faults on this page to
 364  * block.  Note the careful timing, however, the busy bit isn't set till
 365  * late and we cannot do anything that will mess with the page.
 366  */
 367 static int
 368 vm_pageout_cluster(vm_page_t m)
 369 {
 370         vm_object_t object;
 371         vm_page_t mc[2*vm_pageout_page_count], pb, ps;
 372         int pageout_count;
 373         int ib, is, page_base;
 374         vm_pindex_t pindex = m->pindex;
 375
 376         vm_page_lock_assert(m, MA_OWNED);
 377         object = m->object;
 378         VM_OBJECT_ASSERT_WLOCKED(object);
 379
 380         /*
 381          * It doesn't cost us anything to pageout OBJT_DEFAULT or OBJT_SWAP
 382          * with the new swapper, but we could have serious problems paging
 383          * out other object types if there is insufficient memory.
 384          *
 385          * Unfortunately, checking free memory here is far too late, so the
 386          * check has been moved up a procedural level.
 387          */
 388
 389         /*
 390          * Can't clean the page if it's busy or held.
 391          */
 392         vm_page_assert_unbusied(m);
 393         KASSERT(m->hold_count == 0, ("vm_pageout_clean: page %p is held", m));
 394         vm_page_unlock(m);
 395
 396         mc[vm_pageout_page_count] = pb = ps = m;
 397         pageout_count = 1;
 398         page_base = vm_pageout_page_count;
 399         ib = 1;
 400         is = 1;
 401
 402         /*
 403          * Scan object for clusterable pages.
 404          *
 405          * We can cluster ONLY if: ->> the page is NOT
 406          * clean, wired, busy, held, or mapped into a
 407          * buffer, and one of the following:
 408          * 1) The page is inactive, or a seldom used
 409          *    active page.
 410          * -or-
 411          * 2) we force the issue.
 412          *
 413          * During heavy mmap/modification loads the pageout
 414          * daemon can really fragment the underlying file
 415          * due to flushing pages out of order and not trying
 416          * align the clusters (which leave sporatic out-of-order
 417          * holes).  To solve this problem we do the reverse scan
 418          * first and attempt to align our cluster, then do a
 419          * forward scan if room remains.
 420          */
 421 more:
 422         while (ib && pageout_count < vm_pageout_page_count) {
 423                 vm_page_t p;
 424
 425                 if (ib > pindex) {
 426                         ib = 0;
 427                         break;
 428                 }
 429
 430                 if ((p = vm_page_prev(pb)) == NULL || vm_page_busied(p)) {
 431                         ib = 0;
 432                         break;
 433                 }
 434                 vm_page_test_dirty(p);
 435                 if (p->dirty == 0) {
 436                         ib = 0;
 437                         break;
 438                 }
 439                 vm_page_lock(p);
 440                 if (p->queue != PQ_INACTIVE ||
 441                     p->hold_count != 0) {       /* may be undergoing I/O */
 442                         vm_page_unlock(p);
 443                         ib = 0;
 444                         break;
 445                 }
 446                 vm_page_unlock(p);
 447                 mc[--page_base] = pb = p;
 448                 ++pageout_count;
 449                 ++ib;
 450                 /*
 451                  * alignment boundary, stop here and switch directions.  Do
 452                  * not clear ib.
 453                  */
 454                 if ((pindex - (ib - 1)) % vm_pageout_page_count == 0)
 455                         break;
 456         }
 457
 458         while (pageout_count < vm_pageout_page_count &&
 459             pindex + is < object->size) {
 460                 vm_page_t p;
 461
 462                 if ((p = vm_page_next(ps)) == NULL || vm_page_busied(p))
 463                         break;
 464                 vm_page_test_dirty(p);
 465                 if (p->dirty == 0)
 466                         break;
 467                 vm_page_lock(p);
 468                 if (p->queue != PQ_INACTIVE ||
 469                     p->hold_count != 0) {       /* may be undergoing I/O */
 470                         vm_page_unlock(p);
 471                         break;
 472                 }
 473                 vm_page_unlock(p);
 474                 mc[page_base + pageout_count] = ps = p;
 475                 ++pageout_count;
 476                 ++is;
 477         }
 478
 479         /*
 480          * If we exhausted our forward scan, continue with the reverse scan
 481          * when possible, even past a page boundary.  This catches boundary
 482          * conditions.
 483          */
 484         if (ib && pageout_count < vm_pageout_page_count)
 485                 goto more;
 486
 487         /*
 488          * we allow reads during pageouts...
 489          */
 490         return (vm_pageout_flush(&mc[page_base], pageout_count, 0, 0, NULL,
 491             NULL));
 492 }
 493
 494 /*
 495  * vm_pageout_flush() - launder the given pages
 496  *
 497  *      The given pages are laundered.  Note that we setup for the start of
 498  *      I/O ( i.e. busy the page ), mark it read-only, and bump the object
 499  *      reference count all in here rather then in the parent.  If we want
 500  *      the parent to do more sophisticated things we may have to change
 501  *      the ordering.
 502  *
 503  *      Returned runlen is the count of pages between mreq and first
 504  *      page after mreq with status VM_PAGER_AGAIN.
 505  *      *eio is set to TRUE if pager returned VM_PAGER_ERROR or VM_PAGER_FAIL
 506  *      for any page in runlen set.
 507  */
 508 int
 509 vm_pageout_flush(vm_page_t *mc, int count, int flags, int mreq, int *prunlen,
 510     boolean_t *eio)
 511 {
 512         vm_object_t object = mc[0]->object;
 513         int pageout_status[count];
 514         int numpagedout = 0;
 515         int i, runlen;
 516
 517         VM_OBJECT_ASSERT_WLOCKED(object);
 518
 519         /*
 520          * Initiate I/O.  Bump the vm_page_t->busy counter and
 521          * mark the pages read-only.
 522          *
 523          * We do not have to fixup the clean/dirty bits here... we can
 524          * allow the pager to do it after the I/O completes.
 525          *
 526          * NOTE! mc[i]->dirty may be partial or fragmented due to an
 527          * edge case with file fragments.
 528          */
 529         for (i = 0; i < count; i++) {
 530                 KASSERT(mc[i]->valid == VM_PAGE_BITS_ALL,
 531                     ("vm_pageout_flush: partially invalid page %p index %d/%d",
 532                         mc[i], i, count));
 533                 vm_page_sbusy(mc[i]);
 534                 pmap_remove_write(mc[i]);
 535         }
 536         vm_object_pip_add(object, count);
 537
 538         vm_pager_put_pages(object, mc, count, flags, pageout_status);
 539
 540         runlen = count - mreq;
 541         if (eio != NULL)
 542                 *eio = FALSE;
 543         for (i = 0; i < count; i++) {
 544                 vm_page_t mt = mc[i];
 545
 546                 KASSERT(pageout_status[i] == VM_PAGER_PEND ||
 547                     !pmap_page_is_write_mapped(mt),
 548                     ("vm_pageout_flush: page %p is not write protected", mt));
 549                 switch (pageout_status[i]) {
 550                 case VM_PAGER_OK:
 551                 case VM_PAGER_PEND:
 552                         numpagedout++;
 553                         break;
 554                 case VM_PAGER_BAD:
 555                         /*
 556                          * Page outside of range of object. Right now we
 557                          * essentially lose the changes by pretending it
 558                          * worked.
 559                          */
 560                         vm_page_undirty(mt);
 561                         break;
 562                 case VM_PAGER_ERROR:
 563                 case VM_PAGER_FAIL:
 564                         /*
 565                          * If page couldn't be paged out, then reactivate the
 566                          * page so it doesn't clog the inactive list.  (We
 567                          * will try paging out it again later).
 568                          */
 569                         vm_page_lock(mt);
 570                         vm_page_activate(mt);
 571                         vm_page_unlock(mt);
 572                         if (eio != NULL && i >= mreq && i - mreq < runlen)
 573                                 *eio = TRUE;
 574                         break;
 575                 case VM_PAGER_AGAIN:
 576                         if (i >= mreq && i - mreq < runlen)
 577                                 runlen = i - mreq;
 578                         break;
 579                 }
 580
 581                 /*
 582                  * If the operation is still going, leave the page busy to
 583                  * block all other accesses. Also, leave the paging in
 584                  * progress indicator set so that we don't attempt an object
 585                  * collapse.
 586                  */
 587                 if (pageout_status[i] != VM_PAGER_PEND) {
 588                         vm_object_pip_wakeup(object);
 589                         vm_page_sunbusy(mt);
 590                 }
 591         }
 592         if (prunlen != NULL)
 593                 *prunlen = runlen;
 594         return (numpagedout);
 595 }
 596
 597 #if !defined(NO_SWAPPING)
 598 /*
 599  *      vm_pageout_object_deactivate_pages
 600  *
 601  *      Deactivate enough pages to satisfy the inactive target
 602  *      requirements.
 603  *
 604  *      The object and map must be locked.
 605  */
 606 static void
 607 vm_pageout_object_deactivate_pages(pmap_t pmap, vm_object_t first_object,
 608     long desired)
 609 {
 610         vm_object_t backing_object, object;
 611         vm_page_t p;
 612         int act_delta, remove_mode;
 613
 614         VM_OBJECT_ASSERT_LOCKED(first_object);
 615         if ((first_object->flags & OBJ_FICTITIOUS) != 0)
 616                 return;
 617         for (object = first_object;; object = backing_object) {
 618                 if (pmap_resident_count(pmap) <= desired)
 619                         goto unlock_return;
 620                 VM_OBJECT_ASSERT_LOCKED(object);
 621                 if ((object->flags & OBJ_UNMANAGED) != 0 ||
 622                     object->paging_in_progress != 0)
 623                         goto unlock_return;
 624
 625                 remove_mode = 0;
 626                 if (object->shadow_count > 1)
 627                         remove_mode = 1;
 628                 /*
 629                  * Scan the object's entire memory queue.
 630                  */
 631                 TAILQ_FOREACH(p, &object->memq, listq) {
 632                         if (pmap_resident_count(pmap) <= desired)
 633                                 goto unlock_return;
 634                         if (vm_page_busied(p))
 635                                 continue;
 636                         PCPU_INC(cnt.v_pdpages);
 637                         vm_page_lock(p);
 638                         if (p->wire_count != 0 || p->hold_count != 0 ||
 639                             !pmap_page_exists_quick(pmap, p)) {
 640                                 vm_page_unlock(p);
 641                                 continue;
 642                         }
 643                         act_delta = pmap_ts_referenced(p);
 644                         if ((p->aflags & PGA_REFERENCED) != 0) {
 645                                 if (act_delta == 0)
 646                                         act_delta = 1;
 647                                 vm_page_aflag_clear(p, PGA_REFERENCED);
 648                         }
 649                         if (p->queue != PQ_ACTIVE && act_delta != 0) {
 650                                 vm_page_activate(p);
 651                                 p->act_count += act_delta;
 652                         } else if (p->queue == PQ_ACTIVE) {
 653                                 if (act_delta == 0) {
 654                                         p->act_count -= min(p->act_count,
 655                                             ACT_DECLINE);
 656                                         if (!remove_mode && p->act_count == 0) {
 657                                                 pmap_remove_all(p);
 658                                                 vm_page_deactivate(p);
 659                                         } else
 660                                                 vm_page_requeue(p);
 661                                 } else {
 662                                         vm_page_activate(p);
 663                                         if (p->act_count < ACT_MAX -
 664                                             ACT_ADVANCE)
 665                                                 p->act_count += ACT_ADVANCE;
 666                                         vm_page_requeue(p);
 667                                 }
 668                         } else if (p->queue == PQ_INACTIVE)
 669                                 pmap_remove_all(p);
 670                         vm_page_unlock(p);
 671                 }
 672                 if ((backing_object = object->backing_object) == NULL)
 673                         goto unlock_return;
 674                 VM_OBJECT_RLOCK(backing_object);
 675                 if (object != first_object)
 676                         VM_OBJECT_RUNLOCK(object);
 677         }
 678 unlock_return:
 679         if (object != first_object)
 680                 VM_OBJECT_RUNLOCK(object);
 681 }
 682
 683 /*
 684  * deactivate some number of pages in a map, try to do it fairly, but
 685  * that is really hard to do.
 686  */
 687 static void
 688 vm_pageout_map_deactivate_pages(map, desired)
 689         vm_map_t map;
 690         long desired;
 691 {
 692         vm_map_entry_t tmpe;
 693         vm_object_t obj, bigobj;
 694         int nothingwired;
 695
 696         if (!vm_map_trylock(map))
 697                 return;
 698
 699         bigobj = NULL;
 700         nothingwired = TRUE;
 701
 702         /*
 703          * first, search out the biggest object, and try to free pages from
 704          * that.
 705          */
 706         tmpe = map->header.next;
 707         while (tmpe != &map->header) {
 708                 if ((tmpe->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
 709                         obj = tmpe->object.vm_object;
 710                         if (obj != NULL && VM_OBJECT_TRYRLOCK(obj)) {
 711                                 if (obj->shadow_count <= 1 &&
 712                                     (bigobj == NULL ||
 713                                      bigobj->resident_page_count < obj->resident_page_count)) {
 714                                         if (bigobj != NULL)
 715                                                 VM_OBJECT_RUNLOCK(bigobj);
 716                                         bigobj = obj;
 717                                 } else
 718                                         VM_OBJECT_RUNLOCK(obj);
 719                         }
 720                 }
 721                 if (tmpe->wired_count > 0)
 722                         nothingwired = FALSE;
 723                 tmpe = tmpe->next;
 724         }
 725
 726         if (bigobj != NULL) {
 727                 vm_pageout_object_deactivate_pages(map->pmap, bigobj, desired);
 728                 VM_OBJECT_RUNLOCK(bigobj);
 729         }
 730         /*
 731          * Next, hunt around for other pages to deactivate.  We actually
 732          * do this search sort of wrong -- .text first is not the best idea.
 733          */
 734         tmpe = map->header.next;
 735         while (tmpe != &map->header) {
 736                 if (pmap_resident_count(vm_map_pmap(map)) <= desired)
 737                         break;
 738                 if ((tmpe->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
 739                         obj = tmpe->object.vm_object;
 740                         if (obj != NULL) {
 741                                 VM_OBJECT_RLOCK(obj);
 742                                 vm_pageout_object_deactivate_pages(map->pmap, obj, desired);
 743                                 VM_OBJECT_RUNLOCK(obj);
 744                         }
 745                 }
 746                 tmpe = tmpe->next;
 747         }
 748
 749         /*
 750          * Remove all mappings if a process is swapped out, this will free page
 751          * table pages.
 752          */
 753         if (desired == 0 && nothingwired) {
 754                 pmap_remove(vm_map_pmap(map), vm_map_min(map),
 755                     vm_map_max(map));
 756         }
 757
 758         vm_map_unlock(map);
 759 }
 760 #endif          /* !defined(NO_SWAPPING) */
 761
 762 /*
 763  * Attempt to acquire all of the necessary locks to launder a page and
 764  * then call through the clustering layer to PUTPAGES.  Wait a short
 765  * time for a vnode lock.
 766  *
 767  * Requires the page and object lock on entry, releases both before return.
 768  * Returns 0 on success and an errno otherwise.
 769  */
 770 static int
 771 vm_pageout_clean(vm_page_t m)
 772 {
 773         struct vnode *vp;
 774         struct mount *mp;
 775         vm_object_t object;
 776         vm_pindex_t pindex;
 777         int error, lockmode;
 778
 779         vm_page_assert_locked(m);
 780         object = m->object;
 781         VM_OBJECT_ASSERT_WLOCKED(object);
 782         error = 0;
 783         vp = NULL;
 784         mp = NULL;
 785
 786         /*
 787          * The object is already known NOT to be dead.   It
 788          * is possible for the vget() to block the whole
 789          * pageout daemon, but the new low-memory handling
 790          * code should prevent it.
 791          *
 792          * We can't wait forever for the vnode lock, we might
 793          * deadlock due to a vn_read() getting stuck in
 794          * vm_wait while holding this vnode.  We skip the
 795          * vnode if we can't get it in a reasonable amount
 796          * of time.
 797          */
 798         if (object->type == OBJT_VNODE) {
 799                 vm_page_unlock(m);
 800                 vp = object->handle;
 801                 if (vp->v_type == VREG &&
 802                     vn_start_write(vp, &mp, V_NOWAIT) != 0) {
 803                         mp = NULL;
 804                         error = EDEADLK;
 805                         goto unlock_all;
 806                 }
 807                 KASSERT(mp != NULL,
 808                     ("vp %p with NULL v_mount", vp));
 809                 vm_object_reference_locked(object);
 810                 pindex = m->pindex;
 811                 VM_OBJECT_WUNLOCK(object);
 812                 lockmode = MNT_SHARED_WRITES(vp->v_mount) ?
 813                     LK_SHARED : LK_EXCLUSIVE;
 814                 if (vget(vp, lockmode | LK_TIMELOCK, curthread)) {
 815                         vp = NULL;
 816                         error = EDEADLK;
 817                         goto unlock_mp;
 818                 }
 819                 VM_OBJECT_WLOCK(object);
 820                 vm_page_lock(m);
 821                 /*
 822                  * While the object and page were unlocked, the page
 823                  * may have been:
 824                  * (1) moved to a different queue,
 825                  * (2) reallocated to a different object,
 826                  * (3) reallocated to a different offset, or
 827                  * (4) cleaned.
 828                  */
 829                 if (m->queue != PQ_INACTIVE || m->object != object ||
 830                     m->pindex != pindex || m->dirty == 0) {
 831                         vm_page_unlock(m);
 832                         error = ENXIO;
 833                         goto unlock_all;
 834                 }
 835
 836                 /*
 837                  * The page may have been busied or held while the object
 838                  * and page locks were released.
 839                  */
 840                 if (vm_page_busied(m) || m->hold_count != 0) {
 841                         vm_page_unlock(m);
 842                         error = EBUSY;
 843                         goto unlock_all;
 844                 }
 845         }
 846
 847         /*
 848          * If a page is dirty, then it is either being washed
 849          * (but not yet cleaned) or it is still in the
 850          * laundry.  If it is still in the laundry, then we
 851          * start the cleaning operation.
 852          */
 853         if (vm_pageout_cluster(m) == 0)
 854                 error = EIO;
 855
 856 unlock_all:
 857         VM_OBJECT_WUNLOCK(object);
 858
 859 unlock_mp:
 860         vm_page_lock_assert(m, MA_NOTOWNED);
 861         if (mp != NULL) {
 862                 if (vp != NULL)
 863                         vput(vp);
 864                 vm_object_deallocate(object);
 865                 vn_finished_write(mp);
 866         }
 867
 868         return (error);
 869 }
 870
 871 /*
 872  *      vm_pageout_scan does the dirty work for the pageout daemon.
 873  *
 874  *      pass 0 - Update active LRU/deactivate pages
 875  *      pass 1 - Move inactive to cache or free
 876  *      pass 2 - Launder dirty pages
 877  */
 878 static void
 879 vm_pageout_scan(struct vm_domain *vmd, int pass)
 880 {
 881         vm_page_t m, next;
 882         struct vm_pagequeue *pq;
 883         vm_object_t object;
 884         long min_scan;
 885         int act_delta, addl_page_shortage, deficit, error, maxlaunder, maxscan;
 886         int page_shortage, scan_tick, scanned, starting_page_shortage;
 887         int vnodes_skipped;
 888         boolean_t pageout_ok, queues_locked;
 889
 890         /*
 891          * If we need to reclaim memory ask kernel caches to return
 892          * some.  We rate limit to avoid thrashing.
 893          */
 894         if (vmd == &vm_dom[0] && pass > 0 &&
 895             (time_uptime - lowmem_uptime) >= lowmem_period) {
 896                 /*
 897                  * Decrease registered cache sizes.
 898                  */
 899                 SDT_PROBE0(vm, , , vm__lowmem_scan);
 900                 EVENTHANDLER_INVOKE(vm_lowmem, 0);
 901                 /*
 902                  * We do this explicitly after the caches have been
 903                  * drained above.
 904                  */
 905                 uma_reclaim();
 906                 lowmem_uptime = time_uptime;
 907         }
 908
 909         /*
 910          * The addl_page_shortage is the number of temporarily
 911          * stuck pages in the inactive queue.  In other words, the
 912          * number of pages from the inactive count that should be
 913          * discounted in setting the target for the active queue scan.
 914          */
 915         addl_page_shortage = 0;
 916
 917         /*
 918          * Calculate the number of pages we want to either free or move
 919          * to the cache.
 920          */
 921         if (pass > 0) {
 922                 deficit = atomic_readandclear_int(&vm_pageout_deficit);
 923                 page_shortage = vm_paging_target() + deficit;
 924         } else
 925                 page_shortage = deficit = 0;
 926         starting_page_shortage = page_shortage;
 927
 928         /*
 929          * maxlaunder limits the number of dirty pages we flush per scan.
 930          * For most systems a smaller value (16 or 32) is more robust under
 931          * extreme memory and disk pressure because any unnecessary writes
 932          * to disk can result in extreme performance degredation.  However,
 933          * systems with excessive dirty pages (especially when MAP_NOSYNC is
 934          * used) will die horribly with limited laundering.  If the pageout
 935          * daemon cannot clean enough pages in the first pass, we let it go
 936          * all out in succeeding passes.
 937          */
 938         if ((maxlaunder = vm_max_launder) <= 1)
 939                 maxlaunder = 1;
 940         if (pass > 1)
 941                 maxlaunder = 10000;
 942
 943         vnodes_skipped = 0;
 944
 945         /*
 946          * Start scanning the inactive queue for pages we can move to the
 947          * cache or free.  The scan will stop when the target is reached or
 948          * we have scanned the entire inactive queue.  Note that m->act_count
 949          * is not used to form decisions for the inactive queue, only for the
 950          * active queue.
 951          */
 952         pq = &vmd->vmd_pagequeues[PQ_INACTIVE];
 953         maxscan = pq->pq_cnt;
 954         vm_pagequeue_lock(pq);
 955         queues_locked = TRUE;
 956         for (m = TAILQ_FIRST(&pq->pq_pl);
 957              m != NULL && maxscan-- > 0 && page_shortage > 0;
 958              m = next) {
 959                 vm_pagequeue_assert_locked(pq);
 960                 KASSERT(queues_locked, ("unlocked queues"));
 961                 KASSERT(m->queue == PQ_INACTIVE, ("Inactive queue %p", m));
 962
 963                 PCPU_INC(cnt.v_pdpages);
 964                 next = TAILQ_NEXT(m, plinks.q);
 965
 966                 /*
 967                  * skip marker pages
 968                  */
 969                 if (m->flags & PG_MARKER)
 970                         continue;
 971
 972                 KASSERT((m->flags & PG_FICTITIOUS) == 0,
 973                     ("Fictitious page %p cannot be in inactive queue", m));
 974                 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 975                     ("Unmanaged page %p cannot be in inactive queue", m));
 976
 977                 /*
 978                  * The page or object lock acquisitions fail if the
 979                  * page was removed from the queue or moved to a
 980                  * different position within the queue.  In either
 981                  * case, addl_page_shortage should not be incremented.
 982                  */
 983                 if (!vm_pageout_page_lock(m, &next))
 984                         goto unlock_page;
 985                 else if (m->hold_count != 0) {
 986                         /*
 987                          * Held pages are essentially stuck in the
 988                          * queue.  So, they ought to be discounted
 989                          * from the inactive count.  See the
 990                          * calculation of the page_shortage for the
 991                          * loop over the active queue below.
 992                          */
 993                         addl_page_shortage++;
 994                         goto unlock_page;
 995                 }
 996                 object = m->object;
 997                 if (!VM_OBJECT_TRYWLOCK(object)) {
 998                         if (!vm_pageout_fallback_object_lock(m, &next))
 999                                 goto unlock_object;
1000                         else if (m->hold_count != 0) {
1001                                 addl_page_shortage++;
1002                                 goto unlock_object;
1003                         }
1004                 }
1005                 if (vm_page_busied(m)) {
1006                         /*
1007                          * Don't mess with busy pages.  Leave them at
1008                          * the front of the queue.  Most likely, they
1009                          * are being paged out and will leave the
1010                          * queue shortly after the scan finishes.  So,
1011                          * they ought to be discounted from the
1012                          * inactive count.
1013                          */
1014                         addl_page_shortage++;
1015 unlock_object:
1016                         VM_OBJECT_WUNLOCK(object);
1017 unlock_page:
1018                         vm_page_unlock(m);
1019                         continue;
1020                 }
1021                 KASSERT(m->hold_count == 0, ("Held page %p", m));
1022
1023                 /*
1024                  * We unlock the inactive page queue, invalidating the
1025                  * 'next' pointer.  Use our marker to remember our
1026                  * place.
1027                  */
1028                 TAILQ_INSERT_AFTER(&pq->pq_pl, m, &vmd->vmd_marker, plinks.q);
1029                 vm_pagequeue_unlock(pq);
1030                 queues_locked = FALSE;
1031
1032                 /*
1033                  * Invalid pages can be easily freed. They cannot be
1034                  * mapped, vm_page_free() asserts this.
1035                  */
1036                 if (m->valid == 0)
1037                         goto free_page;
1038
1039                 /*
1040                  * If the page has been referenced and the object is not dead,
1041                  * reactivate or requeue the page depending on whether the
1042                  * object is mapped.
1043                  */
1044                 if ((m->aflags & PGA_REFERENCED) != 0) {
1045                         vm_page_aflag_clear(m, PGA_REFERENCED);
1046                         act_delta = 1;
1047                 } else
1048                         act_delta = 0;
1049                 if (object->ref_count != 0) {
1050                         act_delta += pmap_ts_referenced(m);
1051                 } else {
1052                         KASSERT(!pmap_page_is_mapped(m),
1053                             ("vm_pageout_scan: page %p is mapped", m));
1054                 }
1055                 if (act_delta != 0) {
1056                         if (object->ref_count != 0) {
1057                                 vm_page_activate(m);
1058
1059                                 /*
1060                                  * Increase the activation count if the page
1061                                  * was referenced while in the inactive queue.
1062                                  * This makes it less likely that the page will
1063                                  * be returned prematurely to the inactive
1064                                  * queue.
1065                                  */
1066                                 m->act_count += act_delta + ACT_ADVANCE;
1067                                 goto drop_page;
1068                         } else if ((object->flags & OBJ_DEAD) == 0)
1069                                 goto requeue_page;
1070                 }
1071
1072                 /*
1073                  * If the page appears to be clean at the machine-independent
1074                  * layer, then remove all of its mappings from the pmap in
1075                  * anticipation of placing it onto the cache queue.  If,
1076                  * however, any of the page's mappings allow write access,
1077                  * then the page may still be modified until the last of those
1078                  * mappings are removed.
1079                  */
1080                 if (object->ref_count != 0) {
1081                         vm_page_test_dirty(m);
1082                         if (m->dirty == 0)
1083                                 pmap_remove_all(m);
1084                 }
1085
1086                 if (m->dirty == 0) {
1087                         /*
1088                          * Clean pages can be freed.
1089                          */
1090 free_page:
1091                         vm_page_free(m);
1092                         PCPU_INC(cnt.v_dfree);
1093                         --page_shortage;
1094                 } else if ((object->flags & OBJ_DEAD) != 0) {
1095                         /*
1096                          * Leave dirty pages from dead objects at the front of
1097                          * the queue.  They are being paged out and freed by
1098                          * the thread that destroyed the object.  They will
1099                          * leave the queue shortly after the scan finishes, so
1100                          * they should be discounted from the inactive count.
1101                          */
1102                         addl_page_shortage++;
1103                 } else if ((m->flags & PG_WINATCFLS) == 0 && pass < 2) {
1104                         /*
1105                          * Dirty pages need to be paged out, but flushing
1106                          * a page is extremely expensive versus freeing
1107                          * a clean page.  Rather then artificially limiting
1108                          * the number of pages we can flush, we instead give
1109                          * dirty pages extra priority on the inactive queue
1110                          * by forcing them to be cycled through the queue
1111                          * twice before being flushed, after which the
1112                          * (now clean) page will cycle through once more
1113                          * before being freed.  This significantly extends
1114                          * the thrash point for a heavily loaded machine.
1115                          */
1116                         m->flags |= PG_WINATCFLS;
1117 requeue_page:
1118                         vm_pagequeue_lock(pq);
1119                         queues_locked = TRUE;
1120                         vm_page_requeue_locked(m);
1121                 } else if (maxlaunder > 0) {
1122                         /*
1123                          * We always want to try to flush some dirty pages if
1124                          * we encounter them, to keep the system stable.
1125                          * Normally this number is small, but under extreme
1126                          * pressure where there are insufficient clean pages
1127                          * on the inactive queue, we may have to go all out.
1128                          */
1129
1130                         if (object->type != OBJT_SWAP &&
1131                             object->type != OBJT_DEFAULT)
1132                                 pageout_ok = TRUE;
1133                         else if (disable_swap_pageouts)
1134                                 pageout_ok = FALSE;
1135                         else if (defer_swap_pageouts)
1136                                 pageout_ok = vm_page_count_min();
1137                         else
1138                                 pageout_ok = TRUE;
1139                         if (!pageout_ok)
1140                                 goto requeue_page;
1141                         error = vm_pageout_clean(m);
1142                         /*
1143                          * Decrement page_shortage on success to account for
1144                          * the (future) cleaned page.  Otherwise we could wind
1145                          * up laundering or cleaning too many pages.
1146                          */
1147                         if (error == 0) {
1148                                 page_shortage--;
1149                                 maxlaunder--;
1150                         } else if (error == EDEADLK) {
1151                                 pageout_lock_miss++;
1152                                 vnodes_skipped++;
1153                         } else if (error == EBUSY) {
1154                                 addl_page_shortage++;
1155                         }
1156                         vm_page_lock_assert(m, MA_NOTOWNED);
1157                         goto relock_queues;
1158                 }
1159 drop_page:
1160                 vm_page_unlock(m);
1161                 VM_OBJECT_WUNLOCK(object);
1162 relock_queues:
1163                 if (!queues_locked) {
1164                         vm_pagequeue_lock(pq);
1165                         queues_locked = TRUE;
1166                 }
1167                 next = TAILQ_NEXT(&vmd->vmd_marker, plinks.q);
1168                 TAILQ_REMOVE(&pq->pq_pl, &vmd->vmd_marker, plinks.q);
1169         }
1170         vm_pagequeue_unlock(pq);
1171
1172 #if !defined(NO_SWAPPING)
1173         /*
1174          * Wakeup the swapout daemon if we didn't cache or free the targeted
1175          * number of pages.
1176          */
1177         if (vm_swap_enabled && page_shortage > 0)
1178                 vm_req_vmdaemon(VM_SWAP_NORMAL);
1179 #endif
1180
1181         /*
1182          * Wakeup the sync daemon if we skipped a vnode in a writeable object
1183          * and we didn't cache or free enough pages.
1184          */
1185         if (vnodes_skipped > 0 && page_shortage > vm_cnt.v_free_target -
1186             vm_cnt.v_free_min)
1187                 (void)speedup_syncer();
1188
1189         /*
1190          * If the inactive queue scan fails repeatedly to meet its
1191          * target, kill the largest process.
1192          */
1193         vm_pageout_mightbe_oom(vmd, page_shortage, starting_page_shortage);
1194
1195         /*
1196          * Compute the number of pages we want to try to move from the
1197          * active queue to the inactive queue.
1198          */
1199         page_shortage = vm_cnt.v_inactive_target - vm_cnt.v_inactive_count +
1200             vm_paging_target() + deficit + addl_page_shortage;
1201
1202         pq = &vmd->vmd_pagequeues[PQ_ACTIVE];
1203         vm_pagequeue_lock(pq);
1204         maxscan = pq->pq_cnt;
1205
1206         /*
1207          * If we're just idle polling attempt to visit every
1208          * active page within 'update_period' seconds.
1209          */
1210         scan_tick = ticks;
1211         if (vm_pageout_update_period != 0) {
1212                 min_scan = pq->pq_cnt;
1213                 min_scan *= scan_tick - vmd->vmd_last_active_scan;
1214                 min_scan /= hz * vm_pageout_update_period;
1215         } else
1216                 min_scan = 0;
1217         if (min_scan > 0 || (page_shortage > 0 && maxscan > 0))
1218                 vmd->vmd_last_active_scan = scan_tick;
1219
1220         /*
1221          * Scan the active queue for pages that can be deactivated.  Update
1222          * the per-page activity counter and use it to identify deactivation
1223          * candidates.
1224          */
1225         for (m = TAILQ_FIRST(&pq->pq_pl), scanned = 0; m != NULL && (scanned <
1226             min_scan || (page_shortage > 0 && scanned < maxscan)); m = next,
1227             scanned++) {
1228
1229                 KASSERT(m->queue == PQ_ACTIVE,
1230                     ("vm_pageout_scan: page %p isn't active", m));
1231
1232                 next = TAILQ_NEXT(m, plinks.q);
1233                 if ((m->flags & PG_MARKER) != 0)
1234                         continue;
1235                 KASSERT((m->flags & PG_FICTITIOUS) == 0,
1236                     ("Fictitious page %p cannot be in active queue", m));
1237                 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
1238                     ("Unmanaged page %p cannot be in active queue", m));
1239                 if (!vm_pageout_page_lock(m, &next)) {
1240                         vm_page_unlock(m);
1241                         continue;
1242                 }
1243
1244                 /*
1245                  * The count for pagedaemon pages is done after checking the
1246                  * page for eligibility...
1247                  */
1248                 PCPU_INC(cnt.v_pdpages);
1249
1250                 /*
1251                  * Check to see "how much" the page has been used.
1252                  */
1253                 if ((m->aflags & PGA_REFERENCED) != 0) {
1254                         vm_page_aflag_clear(m, PGA_REFERENCED);
1255                         act_delta = 1;
1256                 } else
1257                         act_delta = 0;
1258
1259                 /*
1260                  * Unlocked object ref count check.  Two races are possible.
1261                  * 1) The ref was transitioning to zero and we saw non-zero,
1262                  *    the pmap bits will be checked unnecessarily.
1263                  * 2) The ref was transitioning to one and we saw zero.
1264                  *    The page lock prevents a new reference to this page so
1265                  *    we need not check the reference bits.
1266                  */
1267                 if (m->object->ref_count != 0)
1268                         act_delta += pmap_ts_referenced(m);
1269
1270                 /*
1271                  * Advance or decay the act_count based on recent usage.
1272                  */
1273                 if (act_delta != 0) {
1274                         m->act_count += ACT_ADVANCE + act_delta;
1275                         if (m->act_count > ACT_MAX)
1276                                 m->act_count = ACT_MAX;
1277                 } else
1278                         m->act_count -= min(m->act_count, ACT_DECLINE);
1279
1280                 /*
1281                  * Move this page to the tail of the active or inactive
1282                  * queue depending on usage.
1283                  */
1284                 if (m->act_count == 0) {
1285                         /* Dequeue to avoid later lock recursion. */
1286                         vm_page_dequeue_locked(m);
1287                         vm_page_deactivate(m);
1288                         page_shortage--;
1289                 } else
1290                         vm_page_requeue_locked(m);
1291                 vm_page_unlock(m);
1292         }
1293         vm_pagequeue_unlock(pq);
1294 #if !defined(NO_SWAPPING)
1295         /*
1296          * Idle process swapout -- run once per second.
1297          */
1298         if (vm_swap_idle_enabled) {
1299                 static long lsec;
1300                 if (time_second != lsec) {
1301                         vm_req_vmdaemon(VM_SWAP_IDLE);
1302                         lsec = time_second;
1303                 }
1304         }
1305 #endif
1306 }
1307
1308 static int vm_pageout_oom_vote;
1309
1310 /*
1311  * The pagedaemon threads randlomly select one to perform the
1312  * OOM.  Trying to kill processes before all pagedaemons
1313  * failed to reach free target is premature.
1314  */
1315 static void
1316 vm_pageout_mightbe_oom(struct vm_domain *vmd, int page_shortage,
1317     int starting_page_shortage)
1318 {
1319         int old_vote;
1320
1321         if (starting_page_shortage <= 0 || starting_page_shortage !=
1322             page_shortage)
1323                 vmd->vmd_oom_seq = 0;
1324         else
1325                 vmd->vmd_oom_seq++;
1326         if (vmd->vmd_oom_seq < vm_pageout_oom_seq) {
1327                 if (vmd->vmd_oom) {
1328                         vmd->vmd_oom = FALSE;
1329                         atomic_subtract_int(&vm_pageout_oom_vote, 1);
1330                 }
1331                 return;
1332         }
1333
1334         /*
1335          * Do not follow the call sequence until OOM condition is
1336          * cleared.
1337          */
1338         vmd->vmd_oom_seq = 0;
1339
1340         if (vmd->vmd_oom)
1341                 return;
1342
1343         vmd->vmd_oom = TRUE;
1344         old_vote = atomic_fetchadd_int(&vm_pageout_oom_vote, 1);
1345         if (old_vote != vm_ndomains - 1)
1346                 return;
1347
1348         /*
1349          * The current pagedaemon thread is the last in the quorum to
1350          * start OOM.  Initiate the selection and signaling of the
1351          * victim.
1352          */
1353         vm_pageout_oom(VM_OOM_MEM);
1354
1355         /*
1356          * After one round of OOM terror, recall our vote.  On the
1357          * next pass, current pagedaemon would vote again if the low
1358          * memory condition is still there, due to vmd_oom being
1359          * false.
1360          */
1361         vmd->vmd_oom = FALSE;
1362         atomic_subtract_int(&vm_pageout_oom_vote, 1);
1363 }
1364
1365 /*
1366  * The OOM killer is the page daemon's action of last resort when
1367  * memory allocation requests have been stalled for a prolonged period
1368  * of time because it cannot reclaim memory.  This function computes
1369  * the approximate number of physical pages that could be reclaimed if
1370  * the specified address space is destroyed.
1371  *
1372  * Private, anonymous memory owned by the address space is the
1373  * principal resource that we expect to recover after an OOM kill.
1374  * Since the physical pages mapped by the address space's COW entries
1375  * are typically shared pages, they are unlikely to be released and so
1376  * they are not counted.
1377  *
1378  * To get to the point where the page daemon runs the OOM killer, its
1379  * efforts to write-back vnode-backed pages may have stalled.  This
1380  * could be caused by a memory allocation deadlock in the write path
1381  * that might be resolved by an OOM kill.  Therefore, physical pages
1382  * belonging to vnode-backed objects are counted, because they might
1383  * be freed without being written out first if the address space holds
1384  * the last reference to an unlinked vnode.
1385  *
1386  * Similarly, physical pages belonging to OBJT_PHYS objects are
1387  * counted because the address space might hold the last reference to
1388  * the object.
1389  */
1390 static long
1391 vm_pageout_oom_pagecount(struct vmspace *vmspace)
1392 {
1393         vm_map_t map;
1394         vm_map_entry_t entry;
1395         vm_object_t obj;
1396         long res;
1397
1398         map = &vmspace->vm_map;
1399         KASSERT(!map->system_map, ("system map"));
1400         sx_assert(&map->lock, SA_LOCKED);
1401         res = 0;
1402         for (entry = map->header.next; entry != &map->header;
1403             entry = entry->next) {
1404                 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0)
1405                         continue;
1406                 obj = entry->object.vm_object;
1407                 if (obj == NULL)
1408                         continue;
1409                 if ((entry->eflags & MAP_ENTRY_NEEDS_COPY) != 0 &&
1410                     obj->ref_count != 1)
1411                         continue;
1412                 switch (obj->type) {
1413                 case OBJT_DEFAULT:
1414                 case OBJT_SWAP:
1415                 case OBJT_PHYS:
1416                 case OBJT_VNODE:
1417                         res += obj->resident_page_count;
1418                         break;
1419                 }
1420         }
1421         return (res);
1422 }
1423
1424 void
1425 vm_pageout_oom(int shortage)
1426 {
1427         struct proc *p, *bigproc;
1428         vm_offset_t size, bigsize;
1429         struct thread *td;
1430         struct vmspace *vm;
1431
1432         /*
1433          * We keep the process bigproc locked once we find it to keep anyone
1434          * from messing with it; however, there is a possibility of
1435          * deadlock if process B is bigproc and one of it's child processes
1436          * attempts to propagate a signal to B while we are waiting for A's
1437          * lock while walking this list.  To avoid this, we don't block on
1438          * the process lock but just skip a process if it is already locked.
1439          */
1440         bigproc = NULL;
1441         bigsize = 0;
1442         sx_slock(&allproc_lock);
1443         FOREACH_PROC_IN_SYSTEM(p) {
1444                 int breakout;
1445
1446                 PROC_LOCK(p);
1447
1448                 /*
1449                  * If this is a system, protected or killed process, skip it.
1450                  */
1451                 if (p->p_state != PRS_NORMAL || (p->p_flag & (P_INEXEC |
1452                     P_PROTECTED | P_SYSTEM | P_WEXIT)) != 0 ||
1453                     p->p_pid == 1 || P_KILLED(p) ||
1454                     (p->p_pid < 48 && swap_pager_avail != 0)) {
1455                         PROC_UNLOCK(p);
1456                         continue;
1457                 }
1458                 /*
1459                  * If the process is in a non-running type state,
1460                  * don't touch it.  Check all the threads individually.
1461                  */
1462                 breakout = 0;
1463                 FOREACH_THREAD_IN_PROC(p, td) {
1464                         thread_lock(td);
1465                         if (!TD_ON_RUNQ(td) &&
1466                             !TD_IS_RUNNING(td) &&
1467                             !TD_IS_SLEEPING(td) &&
1468                             !TD_IS_SUSPENDED(td) &&
1469                             !TD_IS_SWAPPED(td)) {
1470                                 thread_unlock(td);
1471                                 breakout = 1;
1472                                 break;
1473                         }
1474                         thread_unlock(td);
1475                 }
1476                 if (breakout) {
1477                         PROC_UNLOCK(p);
1478                         continue;
1479                 }
1480                 /*
1481                  * get the process size
1482                  */
1483                 vm = vmspace_acquire_ref(p);
1484                 if (vm == NULL) {
1485                         PROC_UNLOCK(p);
1486                         continue;
1487                 }
1488                 _PHOLD(p);
1489                 if (!vm_map_trylock_read(&vm->vm_map)) {
1490                         _PRELE(p);
1491                         PROC_UNLOCK(p);
1492                         vmspace_free(vm);
1493                         continue;
1494                 }
1495                 PROC_UNLOCK(p);
1496                 size = vmspace_swap_count(vm);
1497                 if (shortage == VM_OOM_MEM)
1498                         size += vm_pageout_oom_pagecount(vm);
1499                 vm_map_unlock_read(&vm->vm_map);
1500                 vmspace_free(vm);
1501
1502                 /*
1503                  * If this process is bigger than the biggest one,
1504                  * remember it.
1505                  */
1506                 if (size > bigsize) {
1507                         if (bigproc != NULL)
1508                                 PRELE(bigproc);
1509                         bigproc = p;
1510                         bigsize = size;
1511                 } else {
1512                         PRELE(p);
1513                 }
1514         }
1515         sx_sunlock(&allproc_lock);
1516         if (bigproc != NULL) {
1517                 if (vm_panic_on_oom != 0)
1518                         panic("out of swap space");
1519                 PROC_LOCK(bigproc);
1520                 killproc(bigproc, "out of swap space");
1521                 sched_nice(bigproc, PRIO_MIN);
1522                 _PRELE(bigproc);
1523                 PROC_UNLOCK(bigproc);
1524                 wakeup(&vm_cnt.v_free_count);
1525         }
1526 }
1527
1528 static void
1529 vm_pageout_worker(void *arg)
1530 {
1531         struct vm_domain *domain;
1532         int domidx;
1533
1534         domidx = (uintptr_t)arg;
1535         domain = &vm_dom[domidx];
1536
1537         /*
1538          * XXXKIB It could be useful to bind pageout daemon threads to
1539          * the cores belonging to the domain, from which vm_page_array
1540          * is allocated.
1541          */
1542
1543         KASSERT(domain->vmd_segs != 0, ("domain without segments"));
1544         domain->vmd_last_active_scan = ticks;
1545         vm_pageout_init_marker(&domain->vmd_marker, PQ_INACTIVE);
1546         vm_pageout_init_marker(&domain->vmd_inacthead, PQ_INACTIVE);
1547         TAILQ_INSERT_HEAD(&domain->vmd_pagequeues[PQ_INACTIVE].pq_pl,
1548             &domain->vmd_inacthead, plinks.q);
1549
1550         /*
1551          * The pageout daemon worker is never done, so loop forever.
1552          */
1553         while (TRUE) {
1554                 mtx_lock(&vm_page_queue_free_mtx);
1555
1556                 /*
1557                  * Generally, after a level >= 1 scan, if there are enough
1558                  * free pages to wakeup the waiters, then they are already
1559                  * awake.  A call to vm_page_free() during the scan awakened
1560                  * them.  However, in the following case, this wakeup serves
1561                  * to bound the amount of time that a thread might wait.
1562                  * Suppose a thread's call to vm_page_alloc() fails, but
1563                  * before that thread calls VM_WAIT, enough pages are freed by
1564                  * other threads to alleviate the free page shortage.  The
1565                  * thread will, nonetheless, wait until another page is freed
1566                  * or this wakeup is performed.
1567                  */
1568                 if (vm_pages_needed && !vm_page_count_min()) {
1569                         vm_pages_needed = false;
1570                         wakeup(&vm_cnt.v_free_count);
1571                 }
1572
1573                 /*
1574                  * Do not clear vm_pageout_wanted until we reach our target.
1575                  * Otherwise, we may be awakened over and over again, wasting
1576                  * CPU time.
1577                  */
1578                 if (vm_pageout_wanted && !vm_paging_needed())
1579                         vm_pageout_wanted = false;
1580
1581                 /*
1582                  * Might the page daemon receive a wakeup call?
1583                  */
1584                 if (vm_pageout_wanted) {
1585                         /*
1586                          * No.  Either vm_pageout_wanted was set by another
1587                          * thread during the previous scan, which must have
1588                          * been a level 0 scan, or vm_pageout_wanted was
1589                          * already set and the scan failed to free enough
1590                          * pages.  If we haven't yet performed a level >= 2
1591                          * scan (unlimited dirty cleaning), then upgrade the
1592                          * level and scan again now.  Otherwise, sleep a bit
1593                          * and try again later.
1594                          */
1595                         mtx_unlock(&vm_page_queue_free_mtx);
1596                         if (domain->vmd_pass > 1)
1597                                 pause("psleep", hz / 2);
1598                         domain->vmd_pass++;
1599                 } else {
1600                         /*
1601                          * Yes.  Sleep until pages need to be reclaimed or
1602                          * have their reference stats updated.
1603                          */
1604                         if (mtx_sleep(&vm_pageout_wanted,
1605                             &vm_page_queue_free_mtx, PDROP | PVM, "psleep",
1606                             hz) == 0) {
1607                                 PCPU_INC(cnt.v_pdwakeups);
1608                                 domain->vmd_pass = 1;
1609                         } else
1610                                 domain->vmd_pass = 0;
1611                 }
1612
1613                 vm_pageout_scan(domain, domain->vmd_pass);
1614         }
1615 }
1616
1617 /*
1618  *      vm_pageout_init initialises basic pageout daemon settings.
1619  */
1620 static void
1621 vm_pageout_init(void)
1622 {
1623         /*
1624          * Initialize some paging parameters.
1625          */
1626         vm_cnt.v_interrupt_free_min = 2;
1627         if (vm_cnt.v_page_count < 2000)
1628                 vm_pageout_page_count = 8;
1629
1630         /*
1631          * v_free_reserved needs to include enough for the largest
1632          * swap pager structures plus enough for any pv_entry structs
1633          * when paging.
1634          */
1635         if (vm_cnt.v_page_count > 1024)
1636                 vm_cnt.v_free_min = 4 + (vm_cnt.v_page_count - 1024) / 200;
1637         else
1638                 vm_cnt.v_free_min = 4;
1639         vm_cnt.v_pageout_free_min = (2*MAXBSIZE)/PAGE_SIZE +
1640             vm_cnt.v_interrupt_free_min;
1641         vm_cnt.v_free_reserved = vm_pageout_page_count +
1642             vm_cnt.v_pageout_free_min + (vm_cnt.v_page_count / 768);
1643         vm_cnt.v_free_severe = vm_cnt.v_free_min / 2;
1644         vm_cnt.v_free_target = 4 * vm_cnt.v_free_min + vm_cnt.v_free_reserved;
1645         vm_cnt.v_free_min += vm_cnt.v_free_reserved;
1646         vm_cnt.v_free_severe += vm_cnt.v_free_reserved;
1647         vm_cnt.v_inactive_target = (3 * vm_cnt.v_free_target) / 2;
1648         if (vm_cnt.v_inactive_target > vm_cnt.v_free_count / 3)
1649                 vm_cnt.v_inactive_target = vm_cnt.v_free_count / 3;
1650
1651         /*
1652          * Set the default wakeup threshold to be 10% above the minimum
1653          * page limit.  This keeps the steady state out of shortfall.
1654          */
1655         vm_pageout_wakeup_thresh = (vm_cnt.v_free_min / 10) * 11;
1656
1657         /*
1658          * Set interval in seconds for active scan.  We want to visit each
1659          * page at least once every ten minutes.  This is to prevent worst
1660          * case paging behaviors with stale active LRU.
1661          */
1662         if (vm_pageout_update_period == 0)
1663                 vm_pageout_update_period = 600;
1664
1665         /* XXX does not really belong here */
1666         if (vm_page_max_wired == 0)
1667                 vm_page_max_wired = vm_cnt.v_free_count / 3;
1668 }
1669
1670 /*
1671  *     vm_pageout is the high level pageout daemon.
1672  */
1673 static void
1674 vm_pageout(void)
1675 {
1676         int error;
1677 #ifdef VM_NUMA_ALLOC
1678         int i;
1679 #endif
1680
1681         swap_pager_swap_init();
1682 #ifdef VM_NUMA_ALLOC
1683         for (i = 1; i < vm_ndomains; i++) {
1684                 error = kthread_add(vm_pageout_worker, (void *)(uintptr_t)i,
1685                     curproc, NULL, 0, 0, "dom%d", i);
1686                 if (error != 0) {
1687                         panic("starting pageout for domain %d, error %d\n",
1688                             i, error);
1689                 }
1690         }
1691 #endif
1692         error = kthread_add(uma_reclaim_worker, NULL, curproc, NULL,
1693             0, 0, "uma");
1694         if (error != 0)
1695                 panic("starting uma_reclaim helper, error %d\n", error);
1696         vm_pageout_worker((void *)(uintptr_t)0);
1697 }
1698
1699 /*
1700  * Unless the free page queue lock is held by the caller, this function
1701  * should be regarded as advisory.  Specifically, the caller should
1702  * not msleep() on &vm_cnt.v_free_count following this function unless
1703  * the free page queue lock is held until the msleep() is performed.
1704  */
1705 void
1706 pagedaemon_wakeup(void)
1707 {
1708
1709         if (!vm_pageout_wanted && curthread->td_proc != pageproc) {
1710                 vm_pageout_wanted = true;
1711                 wakeup(&vm_pageout_wanted);
1712         }
1713 }
1714
1715 #if !defined(NO_SWAPPING)
1716 static void
1717 vm_req_vmdaemon(int req)
1718 {
1719         static int lastrun = 0;
1720
1721         mtx_lock(&vm_daemon_mtx);
1722         vm_pageout_req_swapout |= req;
1723         if ((ticks > (lastrun + hz)) || (ticks < lastrun)) {
1724                 wakeup(&vm_daemon_needed);
1725                 lastrun = ticks;
1726         }
1727         mtx_unlock(&vm_daemon_mtx);
1728 }
1729
1730 static void
1731 vm_daemon(void)
1732 {
1733         struct rlimit rsslim;
1734         struct proc *p;
1735         struct thread *td;
1736         struct vmspace *vm;
1737         int breakout, swapout_flags, tryagain, attempts;
1738 #ifdef RACCT
1739         uint64_t rsize, ravailable;
1740 #endif
1741
1742         while (TRUE) {
1743                 mtx_lock(&vm_daemon_mtx);
1744                 msleep(&vm_daemon_needed, &vm_daemon_mtx, PPAUSE, "psleep",
1745 #ifdef RACCT
1746                     racct_enable ? hz : 0
1747 #else
1748                     0
1749 #endif
1750                 );
1751                 swapout_flags = vm_pageout_req_swapout;
1752                 vm_pageout_req_swapout = 0;
1753                 mtx_unlock(&vm_daemon_mtx);
1754                 if (swapout_flags)
1755                         swapout_procs(swapout_flags);
1756
1757                 /*
1758                  * scan the processes for exceeding their rlimits or if
1759                  * process is swapped out -- deactivate pages
1760                  */
1761                 tryagain = 0;
1762                 attempts = 0;
1763 again:
1764                 attempts++;
1765                 sx_slock(&allproc_lock);
1766                 FOREACH_PROC_IN_SYSTEM(p) {
1767                         vm_pindex_t limit, size;
1768
1769                         /*
1770                          * if this is a system process or if we have already
1771                          * looked at this process, skip it.
1772                          */
1773                         PROC_LOCK(p);
1774                         if (p->p_state != PRS_NORMAL ||
1775                             p->p_flag & (P_INEXEC | P_SYSTEM | P_WEXIT)) {
1776                                 PROC_UNLOCK(p);
1777                                 continue;
1778                         }
1779                         /*
1780                          * if the process is in a non-running type state,
1781                          * don't touch it.
1782                          */
1783                         breakout = 0;
1784                         FOREACH_THREAD_IN_PROC(p, td) {
1785                                 thread_lock(td);
1786                                 if (!TD_ON_RUNQ(td) &&
1787                                     !TD_IS_RUNNING(td) &&
1788                                     !TD_IS_SLEEPING(td) &&
1789                                     !TD_IS_SUSPENDED(td)) {
1790                                         thread_unlock(td);
1791                                         breakout = 1;
1792                                         break;
1793                                 }
1794                                 thread_unlock(td);
1795                         }
1796                         if (breakout) {
1797                                 PROC_UNLOCK(p);
1798                                 continue;
1799                         }
1800                         /*
1801                          * get a limit
1802                          */
1803                         lim_rlimit_proc(p, RLIMIT_RSS, &rsslim);
1804                         limit = OFF_TO_IDX(
1805                             qmin(rsslim.rlim_cur, rsslim.rlim_max));
1806
1807                         /*
1808                          * let processes that are swapped out really be
1809                          * swapped out set the limit to nothing (will force a
1810                          * swap-out.)
1811                          */
1812                         if ((p->p_flag & P_INMEM) == 0)
1813                                 limit = 0;      /* XXX */
1814                         vm = vmspace_acquire_ref(p);
1815                         PROC_UNLOCK(p);
1816                         if (vm == NULL)
1817                                 continue;
1818
1819                         size = vmspace_resident_count(vm);
1820                         if (size >= limit) {
1821                                 vm_pageout_map_deactivate_pages(
1822                                     &vm->vm_map, limit);
1823                         }
1824 #ifdef RACCT
1825                         if (racct_enable) {
1826                                 rsize = IDX_TO_OFF(size);
1827                                 PROC_LOCK(p);
1828                                 racct_set(p, RACCT_RSS, rsize);
1829                                 ravailable = racct_get_available(p, RACCT_RSS);
1830                                 PROC_UNLOCK(p);
1831                                 if (rsize > ravailable) {
1832                                         /*
1833                                          * Don't be overly aggressive; this
1834                                          * might be an innocent process,
1835                                          * and the limit could've been exceeded
1836                                          * by some memory hog.  Don't try
1837                                          * to deactivate more than 1/4th
1838                                          * of process' resident set size.
1839                                          */
1840                                         if (attempts <= 8) {
1841                                                 if (ravailable < rsize -
1842                                                     (rsize / 4)) {
1843                                                         ravailable = rsize -
1844                                                             (rsize / 4);
1845                                                 }
1846                                         }
1847                                         vm_pageout_map_deactivate_pages(
1848                                             &vm->vm_map,
1849                                             OFF_TO_IDX(ravailable));
1850                                         /* Update RSS usage after paging out. */
1851                                         size = vmspace_resident_count(vm);
1852                                         rsize = IDX_TO_OFF(size);
1853                                         PROC_LOCK(p);
1854                                         racct_set(p, RACCT_RSS, rsize);
1855                                         PROC_UNLOCK(p);
1856                                         if (rsize > ravailable)
1857                                                 tryagain = 1;
1858                                 }
1859                         }
1860 #endif
1861                         vmspace_free(vm);
1862                 }
1863                 sx_sunlock(&allproc_lock);
1864                 if (tryagain != 0 && attempts <= 10)
1865                         goto again;
1866         }
1867 }
1868 #endif                  /* !defined(NO_SWAPPING) */