sys/vm/vm_pageout.c

   1 /*-
   2  * Copyright (c) 1991 Regents of the University of California.
   3  * All rights reserved.
   4  * Copyright (c) 1994 John S. Dyson
   5  * All rights reserved.
   6  * Copyright (c) 1994 David Greenman
   7  * All rights reserved.
   8  * Copyright (c) 2005 Yahoo! Technologies Norway AS
   9  * All rights reserved.
  10  *
  11  * This code is derived from software contributed to Berkeley by
  12  * The Mach Operating System project at Carnegie-Mellon University.
  13  *
  14  * Redistribution and use in source and binary forms, with or without
  15  * modification, are permitted provided that the following conditions
  16  * are met:
  17  * 1. Redistributions of source code must retain the above copyright
  18  *    notice, this list of conditions and the following disclaimer.
  19  * 2. Redistributions in binary form must reproduce the above copyright
  20  *    notice, this list of conditions and the following disclaimer in the
  21  *    documentation and/or other materials provided with the distribution.
  22  * 3. All advertising materials mentioning features or use of this software
  23  *    must display the following acknowledgement:
  24  *      This product includes software developed by the University of
  25  *      California, Berkeley and its contributors.
  26  * 4. Neither the name of the University nor the names of its contributors
  27  *    may be used to endorse or promote products derived from this software
  28  *    without specific prior written permission.
  29  *
  30  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  31  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  32  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  33  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  34  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  35  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  36  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  37  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  38  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  39  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  40  * SUCH DAMAGE.
  41  *
  42  *      from: @(#)vm_pageout.c  7.4 (Berkeley) 5/7/91
  43  *
  44  *
  45  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
  46  * All rights reserved.
  47  *
  48  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
  49  *
  50  * Permission to use, copy, modify and distribute this software and
  51  * its documentation is hereby granted, provided that both the copyright
  52  * notice and this permission notice appear in all copies of the
  53  * software, derivative works or modified versions, and any portions
  54  * thereof, and that both notices appear in supporting documentation.
  55  *
  56  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  57  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
  58  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  59  *
  60  * Carnegie Mellon requests users of this software to return to
  61  *
  62  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  63  *  School of Computer Science
  64  *  Carnegie Mellon University
  65  *  Pittsburgh PA 15213-3890
  66  *
  67  * any improvements or extensions that they make and grant Carnegie the
  68  * rights to redistribute these changes.
  69  */
  70
  71 /*
  72  *      The proverbial page-out daemon.
  73  */
  74
  75 #include <sys/cdefs.h>
  76 __FBSDID("$FreeBSD$");
  77
  78 #include "opt_vm.h"
  79
  80 #include <sys/param.h>
  81 #include <sys/systm.h>
  82 #include <sys/kernel.h>
  83 #include <sys/eventhandler.h>
  84 #include <sys/lock.h>
  85 #include <sys/mutex.h>
  86 #include <sys/proc.h>
  87 #include <sys/kthread.h>
  88 #include <sys/ktr.h>
  89 #include <sys/mount.h>
  90 #include <sys/racct.h>
  91 #include <sys/resourcevar.h>
  92 #include <sys/sched.h>
  93 #include <sys/sdt.h>
  94 #include <sys/signalvar.h>
  95 #include <sys/smp.h>
  96 #include <sys/time.h>
  97 #include <sys/vnode.h>
  98 #include <sys/vmmeter.h>
  99 #include <sys/rwlock.h>
 100 #include <sys/sx.h>
 101 #include <sys/sysctl.h>
 102
 103 #include <vm/vm.h>
 104 #include <vm/vm_param.h>
 105 #include <vm/vm_object.h>
 106 #include <vm/vm_page.h>
 107 #include <vm/vm_map.h>
 108 #include <vm/vm_pageout.h>
 109 #include <vm/vm_pager.h>
 110 #include <vm/vm_phys.h>
 111 #include <vm/swap_pager.h>
 112 #include <vm/vm_extern.h>
 113 #include <vm/uma.h>
 114
 115 /*
 116  * System initialization
 117  */
 118
 119 /* the kernel process "vm_pageout"*/
 120 static void vm_pageout(void);
 121 static void vm_pageout_init(void);
 122 static int vm_pageout_clean(vm_page_t m);
 123 static int vm_pageout_cluster(vm_page_t m);
 124 static bool vm_pageout_scan(struct vm_domain *vmd, int pass);
 125 static void vm_pageout_mightbe_oom(struct vm_domain *vmd, int page_shortage,
 126     int starting_page_shortage);
 127
 128 SYSINIT(pagedaemon_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_FIRST, vm_pageout_init,
 129     NULL);
 130
 131 struct proc *pageproc;
 132
 133 static struct kproc_desc page_kp = {
 134         "pagedaemon",
 135         vm_pageout,
 136         &pageproc
 137 };
 138 SYSINIT(pagedaemon, SI_SUB_KTHREAD_PAGE, SI_ORDER_SECOND, kproc_start,
 139     &page_kp);
 140
 141 SDT_PROVIDER_DEFINE(vm);
 142 SDT_PROBE_DEFINE(vm, , , vm__lowmem_scan);
 143
 144 #if !defined(NO_SWAPPING)
 145 /* the kernel process "vm_daemon"*/
 146 static void vm_daemon(void);
 147 static struct   proc *vmproc;
 148
 149 static struct kproc_desc vm_kp = {
 150         "vmdaemon",
 151         vm_daemon,
 152         &vmproc
 153 };
 154 SYSINIT(vmdaemon, SI_SUB_KTHREAD_VM, SI_ORDER_FIRST, kproc_start, &vm_kp);
 155 #endif
 156
 157
 158 int vm_pageout_deficit;         /* Estimated number of pages deficit */
 159 int vm_pageout_wakeup_thresh;
 160 static int vm_pageout_oom_seq = 12;
 161 bool vm_pageout_wanted;         /* Event on which pageout daemon sleeps */
 162 bool vm_pages_needed;           /* Are threads waiting for free pages? */
 163
 164 #if !defined(NO_SWAPPING)
 165 static int vm_pageout_req_swapout;      /* XXX */
 166 static int vm_daemon_needed;
 167 static struct mtx vm_daemon_mtx;
 168 /* Allow for use by vm_pageout before vm_daemon is initialized. */
 169 MTX_SYSINIT(vm_daemon, &vm_daemon_mtx, "vm daemon", MTX_DEF);
 170 #endif
 171 static int vm_max_launder = 32;
 172 static int vm_pageout_update_period;
 173 static int defer_swap_pageouts;
 174 static int disable_swap_pageouts;
 175 static int lowmem_period = 10;
 176 static time_t lowmem_uptime;
 177
 178 #if defined(NO_SWAPPING)
 179 static int vm_swap_enabled = 0;
 180 static int vm_swap_idle_enabled = 0;
 181 #else
 182 static int vm_swap_enabled = 1;
 183 static int vm_swap_idle_enabled = 0;
 184 #endif
 185
 186 static int vm_panic_on_oom = 0;
 187
 188 SYSCTL_INT(_vm, OID_AUTO, panic_on_oom,
 189         CTLFLAG_RWTUN, &vm_panic_on_oom, 0,
 190         "panic on out of memory instead of killing the largest process");
 191
 192 SYSCTL_INT(_vm, OID_AUTO, pageout_wakeup_thresh,
 193         CTLFLAG_RW, &vm_pageout_wakeup_thresh, 0,
 194         "free page threshold for waking up the pageout daemon");
 195
 196 SYSCTL_INT(_vm, OID_AUTO, max_launder,
 197         CTLFLAG_RW, &vm_max_launder, 0, "Limit dirty flushes in pageout");
 198
 199 SYSCTL_INT(_vm, OID_AUTO, pageout_update_period,
 200         CTLFLAG_RW, &vm_pageout_update_period, 0,
 201         "Maximum active LRU update period");
 202
 203 SYSCTL_INT(_vm, OID_AUTO, lowmem_period, CTLFLAG_RW, &lowmem_period, 0,
 204         "Low memory callback period");
 205
 206 #if defined(NO_SWAPPING)
 207 SYSCTL_INT(_vm, VM_SWAPPING_ENABLED, swap_enabled,
 208         CTLFLAG_RD, &vm_swap_enabled, 0, "Enable entire process swapout");
 209 SYSCTL_INT(_vm, OID_AUTO, swap_idle_enabled,
 210         CTLFLAG_RD, &vm_swap_idle_enabled, 0, "Allow swapout on idle criteria");
 211 #else
 212 SYSCTL_INT(_vm, VM_SWAPPING_ENABLED, swap_enabled,
 213         CTLFLAG_RW, &vm_swap_enabled, 0, "Enable entire process swapout");
 214 SYSCTL_INT(_vm, OID_AUTO, swap_idle_enabled,
 215         CTLFLAG_RW, &vm_swap_idle_enabled, 0, "Allow swapout on idle criteria");
 216 #endif
 217
 218 SYSCTL_INT(_vm, OID_AUTO, defer_swapspace_pageouts,
 219         CTLFLAG_RW, &defer_swap_pageouts, 0, "Give preference to dirty pages in mem");
 220
 221 SYSCTL_INT(_vm, OID_AUTO, disable_swapspace_pageouts,
 222         CTLFLAG_RW, &disable_swap_pageouts, 0, "Disallow swapout of dirty pages");
 223
 224 static int pageout_lock_miss;
 225 SYSCTL_INT(_vm, OID_AUTO, pageout_lock_miss,
 226         CTLFLAG_RD, &pageout_lock_miss, 0, "vget() lock misses during pageout");
 227
 228 SYSCTL_INT(_vm, OID_AUTO, pageout_oom_seq,
 229         CTLFLAG_RW, &vm_pageout_oom_seq, 0,
 230         "back-to-back calls to oom detector to start OOM");
 231
 232 #define VM_PAGEOUT_PAGE_COUNT 16
 233 int vm_pageout_page_count = VM_PAGEOUT_PAGE_COUNT;
 234
 235 int vm_page_max_wired;          /* XXX max # of wired pages system-wide */
 236 SYSCTL_INT(_vm, OID_AUTO, max_wired,
 237         CTLFLAG_RW, &vm_page_max_wired, 0, "System-wide limit to wired page count");
 238
 239 static boolean_t vm_pageout_fallback_object_lock(vm_page_t, vm_page_t *);
 240 #if !defined(NO_SWAPPING)
 241 static void vm_pageout_map_deactivate_pages(vm_map_t, long);
 242 static void vm_pageout_object_deactivate_pages(pmap_t, vm_object_t, long);
 243 static void vm_req_vmdaemon(int req);
 244 #endif
 245 static boolean_t vm_pageout_page_lock(vm_page_t, vm_page_t *);
 246
 247 /*
 248  * Initialize a dummy page for marking the caller's place in the specified
 249  * paging queue.  In principle, this function only needs to set the flag
 250  * PG_MARKER.  Nonetheless, it write busies and initializes the hold count
 251  * to one as safety precautions.
 252  */
 253 static void
 254 vm_pageout_init_marker(vm_page_t marker, u_short queue)
 255 {
 256
 257         bzero(marker, sizeof(*marker));
 258         marker->flags = PG_MARKER;
 259         marker->busy_lock = VPB_SINGLE_EXCLUSIVER;
 260         marker->queue = queue;
 261         marker->hold_count = 1;
 262 }
 263
 264 /*
 265  * vm_pageout_fallback_object_lock:
 266  *
 267  * Lock vm object currently associated with `m'. VM_OBJECT_TRYWLOCK is
 268  * known to have failed and page queue must be either PQ_ACTIVE or
 269  * PQ_INACTIVE.  To avoid lock order violation, unlock the page queue
 270  * while locking the vm object.  Use marker page to detect page queue
 271  * changes and maintain notion of next page on page queue.  Return
 272  * TRUE if no changes were detected, FALSE otherwise.  vm object is
 273  * locked on return.
 274  *
 275  * This function depends on both the lock portion of struct vm_object
 276  * and normal struct vm_page being type stable.
 277  */
 278 static boolean_t
 279 vm_pageout_fallback_object_lock(vm_page_t m, vm_page_t *next)
 280 {
 281         struct vm_page marker;
 282         struct vm_pagequeue *pq;
 283         boolean_t unchanged;
 284         u_short queue;
 285         vm_object_t object;
 286
 287         queue = m->queue;
 288         vm_pageout_init_marker(&marker, queue);
 289         pq = vm_page_pagequeue(m);
 290         object = m->object;
 291
 292         TAILQ_INSERT_AFTER(&pq->pq_pl, m, &marker, plinks.q);
 293         vm_pagequeue_unlock(pq);
 294         vm_page_unlock(m);
 295         VM_OBJECT_WLOCK(object);
 296         vm_page_lock(m);
 297         vm_pagequeue_lock(pq);
 298
 299         /*
 300          * The page's object might have changed, and/or the page might
 301          * have moved from its original position in the queue.  If the
 302          * page's object has changed, then the caller should abandon
 303          * processing the page because the wrong object lock was
 304          * acquired.  Use the marker's plinks.q, not the page's, to
 305          * determine if the page has been moved.  The state of the
 306          * page's plinks.q can be indeterminate; whereas, the marker's
 307          * plinks.q must be valid.
 308          */
 309         *next = TAILQ_NEXT(&marker, plinks.q);
 310         unchanged = m->object == object &&
 311             m == TAILQ_PREV(&marker, pglist, plinks.q);
 312         KASSERT(!unchanged || m->queue == queue,
 313             ("page %p queue %d %d", m, queue, m->queue));
 314         TAILQ_REMOVE(&pq->pq_pl, &marker, plinks.q);
 315         return (unchanged);
 316 }
 317
 318 /*
 319  * Lock the page while holding the page queue lock.  Use marker page
 320  * to detect page queue changes and maintain notion of next page on
 321  * page queue.  Return TRUE if no changes were detected, FALSE
 322  * otherwise.  The page is locked on return. The page queue lock might
 323  * be dropped and reacquired.
 324  *
 325  * This function depends on normal struct vm_page being type stable.
 326  */
 327 static boolean_t
 328 vm_pageout_page_lock(vm_page_t m, vm_page_t *next)
 329 {
 330         struct vm_page marker;
 331         struct vm_pagequeue *pq;
 332         boolean_t unchanged;
 333         u_short queue;
 334
 335         vm_page_lock_assert(m, MA_NOTOWNED);
 336         if (vm_page_trylock(m))
 337                 return (TRUE);
 338
 339         queue = m->queue;
 340         vm_pageout_init_marker(&marker, queue);
 341         pq = vm_page_pagequeue(m);
 342
 343         TAILQ_INSERT_AFTER(&pq->pq_pl, m, &marker, plinks.q);
 344         vm_pagequeue_unlock(pq);
 345         vm_page_lock(m);
 346         vm_pagequeue_lock(pq);
 347
 348         /* Page queue might have changed. */
 349         *next = TAILQ_NEXT(&marker, plinks.q);
 350         unchanged = m == TAILQ_PREV(&marker, pglist, plinks.q);
 351         KASSERT(!unchanged || m->queue == queue,
 352             ("page %p queue %d %d", m, queue, m->queue));
 353         TAILQ_REMOVE(&pq->pq_pl, &marker, plinks.q);
 354         return (unchanged);
 355 }
 356
 357 /*
 358  * Scan for pages at adjacent offsets within the given page's object that are
 359  * eligible for laundering, form a cluster of these pages and the given page,
 360  * and launder that cluster.
 361  */
 362 static int
 363 vm_pageout_cluster(vm_page_t m)
 364 {
 365         vm_object_t object;
 366         vm_page_t mc[2 * vm_pageout_page_count], p, pb, ps;
 367         vm_pindex_t pindex;
 368         int ib, is, page_base, pageout_count;
 369
 370         vm_page_assert_locked(m);
 371         object = m->object;
 372         VM_OBJECT_ASSERT_WLOCKED(object);
 373         pindex = m->pindex;
 374
 375         /*
 376          * We can't clean the page if it is busy or held.
 377          */
 378         vm_page_assert_unbusied(m);
 379         KASSERT(m->hold_count == 0, ("page %p is held", m));
 380         vm_page_unlock(m);
 381
 382         mc[vm_pageout_page_count] = pb = ps = m;
 383         pageout_count = 1;
 384         page_base = vm_pageout_page_count;
 385         ib = 1;
 386         is = 1;
 387
 388         /*
 389          * We can cluster only if the page is not clean, busy, or held, and
 390          * the page is inactive.
 391          *
 392          * During heavy mmap/modification loads the pageout
 393          * daemon can really fragment the underlying file
 394          * due to flushing pages out of order and not trying to
 395          * align the clusters (which leaves sporadic out-of-order
 396          * holes).  To solve this problem we do the reverse scan
 397          * first and attempt to align our cluster, then do a
 398          * forward scan if room remains.
 399          */
 400 more:
 401         while (ib != 0 && pageout_count < vm_pageout_page_count) {
 402                 if (ib > pindex) {
 403                         ib = 0;
 404                         break;
 405                 }
 406                 if ((p = vm_page_prev(pb)) == NULL || vm_page_busied(p)) {
 407                         ib = 0;
 408                         break;
 409                 }
 410                 vm_page_test_dirty(p);
 411                 if (p->dirty == 0) {
 412                         ib = 0;
 413                         break;
 414                 }
 415                 vm_page_lock(p);
 416                 if (p->queue != PQ_INACTIVE ||
 417                     p->hold_count != 0) {       /* may be undergoing I/O */
 418                         vm_page_unlock(p);
 419                         ib = 0;
 420                         break;
 421                 }
 422                 vm_page_unlock(p);
 423                 mc[--page_base] = pb = p;
 424                 ++pageout_count;
 425                 ++ib;
 426
 427                 /*
 428                  * We are at an alignment boundary.  Stop here, and switch
 429                  * directions.  Do not clear ib.
 430                  */
 431                 if ((pindex - (ib - 1)) % vm_pageout_page_count == 0)
 432                         break;
 433         }
 434         while (pageout_count < vm_pageout_page_count &&
 435             pindex + is < object->size) {
 436                 if ((p = vm_page_next(ps)) == NULL || vm_page_busied(p))
 437                         break;
 438                 vm_page_test_dirty(p);
 439                 if (p->dirty == 0)
 440                         break;
 441                 vm_page_lock(p);
 442                 if (p->queue != PQ_INACTIVE ||
 443                     p->hold_count != 0) {       /* may be undergoing I/O */
 444                         vm_page_unlock(p);
 445                         break;
 446                 }
 447                 vm_page_unlock(p);
 448                 mc[page_base + pageout_count] = ps = p;
 449                 ++pageout_count;
 450                 ++is;
 451         }
 452
 453         /*
 454          * If we exhausted our forward scan, continue with the reverse scan
 455          * when possible, even past an alignment boundary.  This catches
 456          * boundary conditions.
 457          */
 458         if (ib != 0 && pageout_count < vm_pageout_page_count)
 459                 goto more;
 460
 461         return (vm_pageout_flush(&mc[page_base], pageout_count, 0, 0, NULL,
 462             NULL));
 463 }
 464
 465 /*
 466  * vm_pageout_flush() - launder the given pages
 467  *
 468  *      The given pages are laundered.  Note that we setup for the start of
 469  *      I/O ( i.e. busy the page ), mark it read-only, and bump the object
 470  *      reference count all in here rather then in the parent.  If we want
 471  *      the parent to do more sophisticated things we may have to change
 472  *      the ordering.
 473  *
 474  *      Returned runlen is the count of pages between mreq and first
 475  *      page after mreq with status VM_PAGER_AGAIN.
 476  *      *eio is set to TRUE if pager returned VM_PAGER_ERROR or VM_PAGER_FAIL
 477  *      for any page in runlen set.
 478  */
 479 int
 480 vm_pageout_flush(vm_page_t *mc, int count, int flags, int mreq, int *prunlen,
 481     boolean_t *eio)
 482 {
 483         vm_object_t object = mc[0]->object;
 484         int pageout_status[count];
 485         int numpagedout = 0;
 486         int i, runlen;
 487
 488         VM_OBJECT_ASSERT_WLOCKED(object);
 489
 490         /*
 491          * Initiate I/O.  Bump the vm_page_t->busy counter and
 492          * mark the pages read-only.
 493          *
 494          * We do not have to fixup the clean/dirty bits here... we can
 495          * allow the pager to do it after the I/O completes.
 496          *
 497          * NOTE! mc[i]->dirty may be partial or fragmented due to an
 498          * edge case with file fragments.
 499          */
 500         for (i = 0; i < count; i++) {
 501                 KASSERT(mc[i]->valid == VM_PAGE_BITS_ALL,
 502                     ("vm_pageout_flush: partially invalid page %p index %d/%d",
 503                         mc[i], i, count));
 504                 vm_page_sbusy(mc[i]);
 505                 pmap_remove_write(mc[i]);
 506         }
 507         vm_object_pip_add(object, count);
 508
 509         vm_pager_put_pages(object, mc, count, flags, pageout_status);
 510
 511         runlen = count - mreq;
 512         if (eio != NULL)
 513                 *eio = FALSE;
 514         for (i = 0; i < count; i++) {
 515                 vm_page_t mt = mc[i];
 516
 517                 KASSERT(pageout_status[i] == VM_PAGER_PEND ||
 518                     !pmap_page_is_write_mapped(mt),
 519                     ("vm_pageout_flush: page %p is not write protected", mt));
 520                 switch (pageout_status[i]) {
 521                 case VM_PAGER_OK:
 522                 case VM_PAGER_PEND:
 523                         numpagedout++;
 524                         break;
 525                 case VM_PAGER_BAD:
 526                         /*
 527                          * Page outside of range of object. Right now we
 528                          * essentially lose the changes by pretending it
 529                          * worked.
 530                          */
 531                         vm_page_undirty(mt);
 532                         break;
 533                 case VM_PAGER_ERROR:
 534                 case VM_PAGER_FAIL:
 535                         /*
 536                          * If page couldn't be paged out, then reactivate the
 537                          * page so it doesn't clog the inactive list.  (We
 538                          * will try paging out it again later).
 539                          */
 540                         vm_page_lock(mt);
 541                         vm_page_activate(mt);
 542                         vm_page_unlock(mt);
 543                         if (eio != NULL && i >= mreq && i - mreq < runlen)
 544                                 *eio = TRUE;
 545                         break;
 546                 case VM_PAGER_AGAIN:
 547                         if (i >= mreq && i - mreq < runlen)
 548                                 runlen = i - mreq;
 549                         break;
 550                 }
 551
 552                 /*
 553                  * If the operation is still going, leave the page busy to
 554                  * block all other accesses. Also, leave the paging in
 555                  * progress indicator set so that we don't attempt an object
 556                  * collapse.
 557                  */
 558                 if (pageout_status[i] != VM_PAGER_PEND) {
 559                         vm_object_pip_wakeup(object);
 560                         vm_page_sunbusy(mt);
 561                 }
 562         }
 563         if (prunlen != NULL)
 564                 *prunlen = runlen;
 565         return (numpagedout);
 566 }
 567
 568 #if !defined(NO_SWAPPING)
 569 /*
 570  *      vm_pageout_object_deactivate_pages
 571  *
 572  *      Deactivate enough pages to satisfy the inactive target
 573  *      requirements.
 574  *
 575  *      The object and map must be locked.
 576  */
 577 static void
 578 vm_pageout_object_deactivate_pages(pmap_t pmap, vm_object_t first_object,
 579     long desired)
 580 {
 581         vm_object_t backing_object, object;
 582         vm_page_t p;
 583         int act_delta, remove_mode;
 584
 585         VM_OBJECT_ASSERT_LOCKED(first_object);
 586         if ((first_object->flags & OBJ_FICTITIOUS) != 0)
 587                 return;
 588         for (object = first_object;; object = backing_object) {
 589                 if (pmap_resident_count(pmap) <= desired)
 590                         goto unlock_return;
 591                 VM_OBJECT_ASSERT_LOCKED(object);
 592                 if ((object->flags & OBJ_UNMANAGED) != 0 ||
 593                     object->paging_in_progress != 0)
 594                         goto unlock_return;
 595
 596                 remove_mode = 0;
 597                 if (object->shadow_count > 1)
 598                         remove_mode = 1;
 599                 /*
 600                  * Scan the object's entire memory queue.
 601                  */
 602                 TAILQ_FOREACH(p, &object->memq, listq) {
 603                         if (pmap_resident_count(pmap) <= desired)
 604                                 goto unlock_return;
 605                         if (vm_page_busied(p))
 606                                 continue;
 607                         PCPU_INC(cnt.v_pdpages);
 608                         vm_page_lock(p);
 609                         if (p->wire_count != 0 || p->hold_count != 0 ||
 610                             !pmap_page_exists_quick(pmap, p)) {
 611                                 vm_page_unlock(p);
 612                                 continue;
 613                         }
 614                         act_delta = pmap_ts_referenced(p);
 615                         if ((p->aflags & PGA_REFERENCED) != 0) {
 616                                 if (act_delta == 0)
 617                                         act_delta = 1;
 618                                 vm_page_aflag_clear(p, PGA_REFERENCED);
 619                         }
 620                         if (p->queue != PQ_ACTIVE && act_delta != 0) {
 621                                 vm_page_activate(p);
 622                                 p->act_count += act_delta;
 623                         } else if (p->queue == PQ_ACTIVE) {
 624                                 if (act_delta == 0) {
 625                                         p->act_count -= min(p->act_count,
 626                                             ACT_DECLINE);
 627                                         if (!remove_mode && p->act_count == 0) {
 628                                                 pmap_remove_all(p);
 629                                                 vm_page_deactivate(p);
 630                                         } else
 631                                                 vm_page_requeue(p);
 632                                 } else {
 633                                         vm_page_activate(p);
 634                                         if (p->act_count < ACT_MAX -
 635                                             ACT_ADVANCE)
 636                                                 p->act_count += ACT_ADVANCE;
 637                                         vm_page_requeue(p);
 638                                 }
 639                         } else if (p->queue == PQ_INACTIVE)
 640                                 pmap_remove_all(p);
 641                         vm_page_unlock(p);
 642                 }
 643                 if ((backing_object = object->backing_object) == NULL)
 644                         goto unlock_return;
 645                 VM_OBJECT_RLOCK(backing_object);
 646                 if (object != first_object)
 647                         VM_OBJECT_RUNLOCK(object);
 648         }
 649 unlock_return:
 650         if (object != first_object)
 651                 VM_OBJECT_RUNLOCK(object);
 652 }
 653
 654 /*
 655  * deactivate some number of pages in a map, try to do it fairly, but
 656  * that is really hard to do.
 657  */
 658 static void
 659 vm_pageout_map_deactivate_pages(map, desired)
 660         vm_map_t map;
 661         long desired;
 662 {
 663         vm_map_entry_t tmpe;
 664         vm_object_t obj, bigobj;
 665         int nothingwired;
 666
 667         if (!vm_map_trylock(map))
 668                 return;
 669
 670         bigobj = NULL;
 671         nothingwired = TRUE;
 672
 673         /*
 674          * first, search out the biggest object, and try to free pages from
 675          * that.
 676          */
 677         tmpe = map->header.next;
 678         while (tmpe != &map->header) {
 679                 if ((tmpe->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
 680                         obj = tmpe->object.vm_object;
 681                         if (obj != NULL && VM_OBJECT_TRYRLOCK(obj)) {
 682                                 if (obj->shadow_count <= 1 &&
 683                                     (bigobj == NULL ||
 684                                      bigobj->resident_page_count < obj->resident_page_count)) {
 685                                         if (bigobj != NULL)
 686                                                 VM_OBJECT_RUNLOCK(bigobj);
 687                                         bigobj = obj;
 688                                 } else
 689                                         VM_OBJECT_RUNLOCK(obj);
 690                         }
 691                 }
 692                 if (tmpe->wired_count > 0)
 693                         nothingwired = FALSE;
 694                 tmpe = tmpe->next;
 695         }
 696
 697         if (bigobj != NULL) {
 698                 vm_pageout_object_deactivate_pages(map->pmap, bigobj, desired);
 699                 VM_OBJECT_RUNLOCK(bigobj);
 700         }
 701         /*
 702          * Next, hunt around for other pages to deactivate.  We actually
 703          * do this search sort of wrong -- .text first is not the best idea.
 704          */
 705         tmpe = map->header.next;
 706         while (tmpe != &map->header) {
 707                 if (pmap_resident_count(vm_map_pmap(map)) <= desired)
 708                         break;
 709                 if ((tmpe->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
 710                         obj = tmpe->object.vm_object;
 711                         if (obj != NULL) {
 712                                 VM_OBJECT_RLOCK(obj);
 713                                 vm_pageout_object_deactivate_pages(map->pmap, obj, desired);
 714                                 VM_OBJECT_RUNLOCK(obj);
 715                         }
 716                 }
 717                 tmpe = tmpe->next;
 718         }
 719
 720         /*
 721          * Remove all mappings if a process is swapped out, this will free page
 722          * table pages.
 723          */
 724         if (desired == 0 && nothingwired) {
 725                 pmap_remove(vm_map_pmap(map), vm_map_min(map),
 726                     vm_map_max(map));
 727         }
 728
 729         vm_map_unlock(map);
 730 }
 731 #endif          /* !defined(NO_SWAPPING) */
 732
 733 /*
 734  * Attempt to acquire all of the necessary locks to launder a page and
 735  * then call through the clustering layer to PUTPAGES.  Wait a short
 736  * time for a vnode lock.
 737  *
 738  * Requires the page and object lock on entry, releases both before return.
 739  * Returns 0 on success and an errno otherwise.
 740  */
 741 static int
 742 vm_pageout_clean(vm_page_t m)
 743 {
 744         struct vnode *vp;
 745         struct mount *mp;
 746         vm_object_t object;
 747         vm_pindex_t pindex;
 748         int error, lockmode;
 749
 750         vm_page_assert_locked(m);
 751         object = m->object;
 752         VM_OBJECT_ASSERT_WLOCKED(object);
 753         error = 0;
 754         vp = NULL;
 755         mp = NULL;
 756
 757         /*
 758          * The object is already known NOT to be dead.   It
 759          * is possible for the vget() to block the whole
 760          * pageout daemon, but the new low-memory handling
 761          * code should prevent it.
 762          *
 763          * We can't wait forever for the vnode lock, we might
 764          * deadlock due to a vn_read() getting stuck in
 765          * vm_wait while holding this vnode.  We skip the
 766          * vnode if we can't get it in a reasonable amount
 767          * of time.
 768          */
 769         if (object->type == OBJT_VNODE) {
 770                 vm_page_unlock(m);
 771                 vp = object->handle;
 772                 if (vp->v_type == VREG &&
 773                     vn_start_write(vp, &mp, V_NOWAIT) != 0) {
 774                         mp = NULL;
 775                         error = EDEADLK;
 776                         goto unlock_all;
 777                 }
 778                 KASSERT(mp != NULL,
 779                     ("vp %p with NULL v_mount", vp));
 780                 vm_object_reference_locked(object);
 781                 pindex = m->pindex;
 782                 VM_OBJECT_WUNLOCK(object);
 783                 lockmode = MNT_SHARED_WRITES(vp->v_mount) ?
 784                     LK_SHARED : LK_EXCLUSIVE;
 785                 if (vget(vp, lockmode | LK_TIMELOCK, curthread)) {
 786                         vp = NULL;
 787                         error = EDEADLK;
 788                         goto unlock_mp;
 789                 }
 790                 VM_OBJECT_WLOCK(object);
 791                 vm_page_lock(m);
 792                 /*
 793                  * While the object and page were unlocked, the page
 794                  * may have been:
 795                  * (1) moved to a different queue,
 796                  * (2) reallocated to a different object,
 797                  * (3) reallocated to a different offset, or
 798                  * (4) cleaned.
 799                  */
 800                 if (m->queue != PQ_INACTIVE || m->object != object ||
 801                     m->pindex != pindex || m->dirty == 0) {
 802                         vm_page_unlock(m);
 803                         error = ENXIO;
 804                         goto unlock_all;
 805                 }
 806
 807                 /*
 808                  * The page may have been busied or held while the object
 809                  * and page locks were released.
 810                  */
 811                 if (vm_page_busied(m) || m->hold_count != 0) {
 812                         vm_page_unlock(m);
 813                         error = EBUSY;
 814                         goto unlock_all;
 815                 }
 816         }
 817
 818         /*
 819          * If a page is dirty, then it is either being washed
 820          * (but not yet cleaned) or it is still in the
 821          * laundry.  If it is still in the laundry, then we
 822          * start the cleaning operation.
 823          */
 824         if (vm_pageout_cluster(m) == 0)
 825                 error = EIO;
 826
 827 unlock_all:
 828         VM_OBJECT_WUNLOCK(object);
 829
 830 unlock_mp:
 831         vm_page_lock_assert(m, MA_NOTOWNED);
 832         if (mp != NULL) {
 833                 if (vp != NULL)
 834                         vput(vp);
 835                 vm_object_deallocate(object);
 836                 vn_finished_write(mp);
 837         }
 838
 839         return (error);
 840 }
 841
 842 /*
 843  *      vm_pageout_scan does the dirty work for the pageout daemon.
 844  *
 845  *      pass 0 - Update active LRU/deactivate pages
 846  *      pass 1 - Free inactive pages
 847  *      pass 2 - Launder dirty pages
 848  *
 849  * Returns true if pass was zero or enough pages were freed by the inactive
 850  * queue scan to meet the target.
 851  */
 852 static bool
 853 vm_pageout_scan(struct vm_domain *vmd, int pass)
 854 {
 855         vm_page_t m, next;
 856         struct vm_pagequeue *pq;
 857         vm_object_t object;
 858         long min_scan;
 859         int act_delta, addl_page_shortage, deficit, error, inactq_shortage;
 860         int maxlaunder, maxscan, page_shortage, scan_tick, scanned;
 861         int starting_page_shortage, vnodes_skipped;
 862         boolean_t pageout_ok, queue_locked;
 863
 864         /*
 865          * If we need to reclaim memory ask kernel caches to return
 866          * some.  We rate limit to avoid thrashing.
 867          */
 868         if (vmd == &vm_dom[0] && pass > 0 &&
 869             (time_uptime - lowmem_uptime) >= lowmem_period) {
 870                 /*
 871                  * Decrease registered cache sizes.
 872                  */
 873                 SDT_PROBE0(vm, , , vm__lowmem_scan);
 874                 EVENTHANDLER_INVOKE(vm_lowmem, 0);
 875                 /*
 876                  * We do this explicitly after the caches have been
 877                  * drained above.
 878                  */
 879                 uma_reclaim();
 880                 lowmem_uptime = time_uptime;
 881         }
 882
 883         /*
 884          * The addl_page_shortage is the number of temporarily
 885          * stuck pages in the inactive queue.  In other words, the
 886          * number of pages from the inactive count that should be
 887          * discounted in setting the target for the active queue scan.
 888          */
 889         addl_page_shortage = 0;
 890
 891         /*
 892          * Calculate the number of pages that we want to free.  This number
 893          * can be negative if many pages are freed between the wakeup call to
 894          * the page daemon and this calculation.
 895          */
 896         if (pass > 0) {
 897                 deficit = atomic_readandclear_int(&vm_pageout_deficit);
 898                 page_shortage = vm_paging_target() + deficit;
 899         } else
 900                 page_shortage = deficit = 0;
 901         starting_page_shortage = page_shortage;
 902
 903         /*
 904          * maxlaunder limits the number of dirty pages we flush per scan.
 905          * For most systems a smaller value (16 or 32) is more robust under
 906          * extreme memory and disk pressure because any unnecessary writes
 907          * to disk can result in extreme performance degredation.  However,
 908          * systems with excessive dirty pages (especially when MAP_NOSYNC is
 909          * used) will die horribly with limited laundering.  If the pageout
 910          * daemon cannot clean enough pages in the first pass, we let it go
 911          * all out in succeeding passes.
 912          */
 913         if ((maxlaunder = vm_max_launder) <= 1)
 914                 maxlaunder = 1;
 915         if (pass > 1)
 916                 maxlaunder = 10000;
 917
 918         vnodes_skipped = 0;
 919
 920         /*
 921          * Start scanning the inactive queue for pages that we can free.  The
 922          * scan will stop when we reach the target or we have scanned the
 923          * entire queue.  (Note that m->act_count is not used to make
 924          * decisions for the inactive queue, only for the active queue.)
 925          */
 926         pq = &vmd->vmd_pagequeues[PQ_INACTIVE];
 927         maxscan = pq->pq_cnt;
 928         vm_pagequeue_lock(pq);
 929         queue_locked = TRUE;
 930         for (m = TAILQ_FIRST(&pq->pq_pl);
 931              m != NULL && maxscan-- > 0 && page_shortage > 0;
 932              m = next) {
 933                 vm_pagequeue_assert_locked(pq);
 934                 KASSERT(queue_locked, ("unlocked inactive queue"));
 935                 KASSERT(m->queue == PQ_INACTIVE, ("Inactive queue %p", m));
 936
 937                 PCPU_INC(cnt.v_pdpages);
 938                 next = TAILQ_NEXT(m, plinks.q);
 939
 940                 /*
 941                  * skip marker pages
 942                  */
 943                 if (m->flags & PG_MARKER)
 944                         continue;
 945
 946                 KASSERT((m->flags & PG_FICTITIOUS) == 0,
 947                     ("Fictitious page %p cannot be in inactive queue", m));
 948                 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 949                     ("Unmanaged page %p cannot be in inactive queue", m));
 950
 951                 /*
 952                  * The page or object lock acquisitions fail if the
 953                  * page was removed from the queue or moved to a
 954                  * different position within the queue.  In either
 955                  * case, addl_page_shortage should not be incremented.
 956                  */
 957                 if (!vm_pageout_page_lock(m, &next))
 958                         goto unlock_page;
 959                 else if (m->hold_count != 0) {
 960                         /*
 961                          * Held pages are essentially stuck in the
 962                          * queue.  So, they ought to be discounted
 963                          * from the inactive count.  See the
 964                          * calculation of inactq_shortage before the
 965                          * loop over the active queue below.
 966                          */
 967                         addl_page_shortage++;
 968                         goto unlock_page;
 969                 }
 970                 object = m->object;
 971                 if (!VM_OBJECT_TRYWLOCK(object)) {
 972                         if (!vm_pageout_fallback_object_lock(m, &next))
 973                                 goto unlock_object;
 974                         else if (m->hold_count != 0) {
 975                                 addl_page_shortage++;
 976                                 goto unlock_object;
 977                         }
 978                 }
 979                 if (vm_page_busied(m)) {
 980                         /*
 981                          * Don't mess with busy pages.  Leave them at
 982                          * the front of the queue.  Most likely, they
 983                          * are being paged out and will leave the
 984                          * queue shortly after the scan finishes.  So,
 985                          * they ought to be discounted from the
 986                          * inactive count.
 987                          */
 988                         addl_page_shortage++;
 989 unlock_object:
 990                         VM_OBJECT_WUNLOCK(object);
 991 unlock_page:
 992                         vm_page_unlock(m);
 993                         continue;
 994                 }
 995                 KASSERT(m->hold_count == 0, ("Held page %p", m));
 996
 997                 /*
 998                  * We unlock the inactive page queue, invalidating the
 999                  * 'next' pointer.  Use our marker to remember our
1000                  * place.
1001                  */
1002                 TAILQ_INSERT_AFTER(&pq->pq_pl, m, &vmd->vmd_marker, plinks.q);
1003                 vm_pagequeue_unlock(pq);
1004                 queue_locked = FALSE;
1005
1006                 /*
1007                  * Invalid pages can be easily freed. They cannot be
1008                  * mapped, vm_page_free() asserts this.
1009                  */
1010                 if (m->valid == 0)
1011                         goto free_page;
1012
1013                 /*
1014                  * If the page has been referenced and the object is not dead,
1015                  * reactivate or requeue the page depending on whether the
1016                  * object is mapped.
1017                  */
1018                 if ((m->aflags & PGA_REFERENCED) != 0) {
1019                         vm_page_aflag_clear(m, PGA_REFERENCED);
1020                         act_delta = 1;
1021                 } else
1022                         act_delta = 0;
1023                 if (object->ref_count != 0) {
1024                         act_delta += pmap_ts_referenced(m);
1025                 } else {
1026                         KASSERT(!pmap_page_is_mapped(m),
1027                             ("vm_pageout_scan: page %p is mapped", m));
1028                 }
1029                 if (act_delta != 0) {
1030                         if (object->ref_count != 0) {
1031                                 vm_page_activate(m);
1032
1033                                 /*
1034                                  * Increase the activation count if the page
1035                                  * was referenced while in the inactive queue.
1036                                  * This makes it less likely that the page will
1037                                  * be returned prematurely to the inactive
1038                                  * queue.
1039                                  */
1040                                 m->act_count += act_delta + ACT_ADVANCE;
1041                                 goto drop_page;
1042                         } else if ((object->flags & OBJ_DEAD) == 0)
1043                                 goto requeue_page;
1044                 }
1045
1046                 /*
1047                  * If the page appears to be clean at the machine-independent
1048                  * layer, then remove all of its mappings from the pmap in
1049                  * anticipation of freeing it.  If, however, any of the page's
1050                  * mappings allow write access, then the page may still be
1051                  * modified until the last of those mappings are removed.
1052                  */
1053                 if (object->ref_count != 0) {
1054                         vm_page_test_dirty(m);
1055                         if (m->dirty == 0)
1056                                 pmap_remove_all(m);
1057                 }
1058
1059                 if (m->dirty == 0) {
1060                         /*
1061                          * Clean pages can be freed.
1062                          */
1063 free_page:
1064                         vm_page_free(m);
1065                         PCPU_INC(cnt.v_dfree);
1066                         --page_shortage;
1067                 } else if ((object->flags & OBJ_DEAD) != 0) {
1068                         /*
1069                          * Leave dirty pages from dead objects at the front of
1070                          * the queue.  They are being paged out and freed by
1071                          * the thread that destroyed the object.  They will
1072                          * leave the queue shortly after the scan finishes, so
1073                          * they should be discounted from the inactive count.
1074                          */
1075                         addl_page_shortage++;
1076                 } else if ((m->flags & PG_WINATCFLS) == 0 && pass < 2) {
1077                         /*
1078                          * Dirty pages need to be paged out, but flushing
1079                          * a page is extremely expensive versus freeing
1080                          * a clean page.  Rather then artificially limiting
1081                          * the number of pages we can flush, we instead give
1082                          * dirty pages extra priority on the inactive queue
1083                          * by forcing them to be cycled through the queue
1084                          * twice before being flushed, after which the
1085                          * (now clean) page will cycle through once more
1086                          * before being freed.  This significantly extends
1087                          * the thrash point for a heavily loaded machine.
1088                          */
1089                         m->flags |= PG_WINATCFLS;
1090 requeue_page:
1091                         vm_pagequeue_lock(pq);
1092                         queue_locked = TRUE;
1093                         vm_page_requeue_locked(m);
1094                 } else if (maxlaunder > 0) {
1095                         /*
1096                          * We always want to try to flush some dirty pages if
1097                          * we encounter them, to keep the system stable.
1098                          * Normally this number is small, but under extreme
1099                          * pressure where there are insufficient clean pages
1100                          * on the inactive queue, we may have to go all out.
1101                          */
1102
1103                         if (object->type != OBJT_SWAP &&
1104                             object->type != OBJT_DEFAULT)
1105                                 pageout_ok = TRUE;
1106                         else if (disable_swap_pageouts)
1107                                 pageout_ok = FALSE;
1108                         else if (defer_swap_pageouts)
1109                                 pageout_ok = vm_page_count_min();
1110                         else
1111                                 pageout_ok = TRUE;
1112                         if (!pageout_ok)
1113                                 goto requeue_page;
1114                         error = vm_pageout_clean(m);
1115                         /*
1116                          * Decrement page_shortage on success to account for
1117                          * the (future) cleaned page.  Otherwise we could wind
1118                          * up laundering or cleaning too many pages.
1119                          */
1120                         if (error == 0) {
1121                                 page_shortage--;
1122                                 maxlaunder--;
1123                         } else if (error == EDEADLK) {
1124                                 pageout_lock_miss++;
1125                                 vnodes_skipped++;
1126                         } else if (error == EBUSY) {
1127                                 addl_page_shortage++;
1128                         }
1129                         vm_page_lock_assert(m, MA_NOTOWNED);
1130                         goto relock_queue;
1131                 }
1132 drop_page:
1133                 vm_page_unlock(m);
1134                 VM_OBJECT_WUNLOCK(object);
1135 relock_queue:
1136                 if (!queue_locked) {
1137                         vm_pagequeue_lock(pq);
1138                         queue_locked = TRUE;
1139                 }
1140                 next = TAILQ_NEXT(&vmd->vmd_marker, plinks.q);
1141                 TAILQ_REMOVE(&pq->pq_pl, &vmd->vmd_marker, plinks.q);
1142         }
1143         vm_pagequeue_unlock(pq);
1144
1145 #if !defined(NO_SWAPPING)
1146         /*
1147          * Wakeup the swapout daemon if we didn't free the targeted number of
1148          * pages.
1149          */
1150         if (vm_swap_enabled && page_shortage > 0)
1151                 vm_req_vmdaemon(VM_SWAP_NORMAL);
1152 #endif
1153
1154         /*
1155          * Wakeup the sync daemon if we skipped a vnode in a writeable object
1156          * and we didn't free enough pages.
1157          */
1158         if (vnodes_skipped > 0 && page_shortage > vm_cnt.v_free_target -
1159             vm_cnt.v_free_min)
1160                 (void)speedup_syncer();
1161
1162         /*
1163          * If the inactive queue scan fails repeatedly to meet its
1164          * target, kill the largest process.
1165          */
1166         vm_pageout_mightbe_oom(vmd, page_shortage, starting_page_shortage);
1167
1168         /*
1169          * Compute the number of pages we want to try to move from the
1170          * active queue to the inactive queue.
1171          */
1172         inactq_shortage = vm_cnt.v_inactive_target - vm_cnt.v_inactive_count +
1173             vm_paging_target() + deficit + addl_page_shortage;
1174
1175         pq = &vmd->vmd_pagequeues[PQ_ACTIVE];
1176         vm_pagequeue_lock(pq);
1177         maxscan = pq->pq_cnt;
1178
1179         /*
1180          * If we're just idle polling attempt to visit every
1181          * active page within 'update_period' seconds.
1182          */
1183         scan_tick = ticks;
1184         if (vm_pageout_update_period != 0) {
1185                 min_scan = pq->pq_cnt;
1186                 min_scan *= scan_tick - vmd->vmd_last_active_scan;
1187                 min_scan /= hz * vm_pageout_update_period;
1188         } else
1189                 min_scan = 0;
1190         if (min_scan > 0 || (inactq_shortage > 0 && maxscan > 0))
1191                 vmd->vmd_last_active_scan = scan_tick;
1192
1193         /*
1194          * Scan the active queue for pages that can be deactivated.  Update
1195          * the per-page activity counter and use it to identify deactivation
1196          * candidates.  Held pages may be deactivated.
1197          */
1198         for (m = TAILQ_FIRST(&pq->pq_pl), scanned = 0; m != NULL && (scanned <
1199             min_scan || (inactq_shortage > 0 && scanned < maxscan)); m = next,
1200             scanned++) {
1201                 KASSERT(m->queue == PQ_ACTIVE,
1202                     ("vm_pageout_scan: page %p isn't active", m));
1203                 next = TAILQ_NEXT(m, plinks.q);
1204                 if ((m->flags & PG_MARKER) != 0)
1205                         continue;
1206                 KASSERT((m->flags & PG_FICTITIOUS) == 0,
1207                     ("Fictitious page %p cannot be in active queue", m));
1208                 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
1209                     ("Unmanaged page %p cannot be in active queue", m));
1210                 if (!vm_pageout_page_lock(m, &next)) {
1211                         vm_page_unlock(m);
1212                         continue;
1213                 }
1214
1215                 /*
1216                  * The count for page daemon pages is updated after checking
1217                  * the page for eligibility.
1218                  */
1219                 PCPU_INC(cnt.v_pdpages);
1220
1221                 /*
1222                  * Check to see "how much" the page has been used.
1223                  */
1224                 if ((m->aflags & PGA_REFERENCED) != 0) {
1225                         vm_page_aflag_clear(m, PGA_REFERENCED);
1226                         act_delta = 1;
1227                 } else
1228                         act_delta = 0;
1229
1230                 /*
1231                  * Perform an unsynchronized object ref count check.  While
1232                  * the page lock ensures that the page is not reallocated to
1233                  * another object, in particular, one with unmanaged mappings
1234                  * that cannot support pmap_ts_referenced(), two races are,
1235                  * nonetheless, possible:
1236                  * 1) The count was transitioning to zero, but we saw a non-
1237                  *    zero value.  pmap_ts_referenced() will return zero
1238                  *    because the page is not mapped.
1239                  * 2) The count was transitioning to one, but we saw zero.
1240                  *    This race delays the detection of a new reference.  At
1241                  *    worst, we will deactivate and reactivate the page.
1242                  */
1243                 if (m->object->ref_count != 0)
1244                         act_delta += pmap_ts_referenced(m);
1245
1246                 /*
1247                  * Advance or decay the act_count based on recent usage.
1248                  */
1249                 if (act_delta != 0) {
1250                         m->act_count += ACT_ADVANCE + act_delta;
1251                         if (m->act_count > ACT_MAX)
1252                                 m->act_count = ACT_MAX;
1253                 } else
1254                         m->act_count -= min(m->act_count, ACT_DECLINE);
1255
1256                 /*
1257                  * Move this page to the tail of the active or inactive
1258                  * queue depending on usage.
1259                  */
1260                 if (m->act_count == 0) {
1261                         /* Dequeue to avoid later lock recursion. */
1262                         vm_page_dequeue_locked(m);
1263                         vm_page_deactivate(m);
1264                         inactq_shortage--;
1265                 } else
1266                         vm_page_requeue_locked(m);
1267                 vm_page_unlock(m);
1268         }
1269         vm_pagequeue_unlock(pq);
1270 #if !defined(NO_SWAPPING)
1271         /*
1272          * Idle process swapout -- run once per second.
1273          */
1274         if (vm_swap_idle_enabled) {
1275                 static long lsec;
1276                 if (time_second != lsec) {
1277                         vm_req_vmdaemon(VM_SWAP_IDLE);
1278                         lsec = time_second;
1279                 }
1280         }
1281 #endif
1282         return (page_shortage <= 0);
1283 }
1284
1285 static int vm_pageout_oom_vote;
1286
1287 /*
1288  * The pagedaemon threads randlomly select one to perform the
1289  * OOM.  Trying to kill processes before all pagedaemons
1290  * failed to reach free target is premature.
1291  */
1292 static void
1293 vm_pageout_mightbe_oom(struct vm_domain *vmd, int page_shortage,
1294     int starting_page_shortage)
1295 {
1296         int old_vote;
1297
1298         if (starting_page_shortage <= 0 || starting_page_shortage !=
1299             page_shortage)
1300                 vmd->vmd_oom_seq = 0;
1301         else
1302                 vmd->vmd_oom_seq++;
1303         if (vmd->vmd_oom_seq < vm_pageout_oom_seq) {
1304                 if (vmd->vmd_oom) {
1305                         vmd->vmd_oom = FALSE;
1306                         atomic_subtract_int(&vm_pageout_oom_vote, 1);
1307                 }
1308                 return;
1309         }
1310
1311         /*
1312          * Do not follow the call sequence until OOM condition is
1313          * cleared.
1314          */
1315         vmd->vmd_oom_seq = 0;
1316
1317         if (vmd->vmd_oom)
1318                 return;
1319
1320         vmd->vmd_oom = TRUE;
1321         old_vote = atomic_fetchadd_int(&vm_pageout_oom_vote, 1);
1322         if (old_vote != vm_ndomains - 1)
1323                 return;
1324
1325         /*
1326          * The current pagedaemon thread is the last in the quorum to
1327          * start OOM.  Initiate the selection and signaling of the
1328          * victim.
1329          */
1330         vm_pageout_oom(VM_OOM_MEM);
1331
1332         /*
1333          * After one round of OOM terror, recall our vote.  On the
1334          * next pass, current pagedaemon would vote again if the low
1335          * memory condition is still there, due to vmd_oom being
1336          * false.
1337          */
1338         vmd->vmd_oom = FALSE;
1339         atomic_subtract_int(&vm_pageout_oom_vote, 1);
1340 }
1341
1342 /*
1343  * The OOM killer is the page daemon's action of last resort when
1344  * memory allocation requests have been stalled for a prolonged period
1345  * of time because it cannot reclaim memory.  This function computes
1346  * the approximate number of physical pages that could be reclaimed if
1347  * the specified address space is destroyed.
1348  *
1349  * Private, anonymous memory owned by the address space is the
1350  * principal resource that we expect to recover after an OOM kill.
1351  * Since the physical pages mapped by the address space's COW entries
1352  * are typically shared pages, they are unlikely to be released and so
1353  * they are not counted.
1354  *
1355  * To get to the point where the page daemon runs the OOM killer, its
1356  * efforts to write-back vnode-backed pages may have stalled.  This
1357  * could be caused by a memory allocation deadlock in the write path
1358  * that might be resolved by an OOM kill.  Therefore, physical pages
1359  * belonging to vnode-backed objects are counted, because they might
1360  * be freed without being written out first if the address space holds
1361  * the last reference to an unlinked vnode.
1362  *
1363  * Similarly, physical pages belonging to OBJT_PHYS objects are
1364  * counted because the address space might hold the last reference to
1365  * the object.
1366  */
1367 static long
1368 vm_pageout_oom_pagecount(struct vmspace *vmspace)
1369 {
1370         vm_map_t map;
1371         vm_map_entry_t entry;
1372         vm_object_t obj;
1373         long res;
1374
1375         map = &vmspace->vm_map;
1376         KASSERT(!map->system_map, ("system map"));
1377         sx_assert(&map->lock, SA_LOCKED);
1378         res = 0;
1379         for (entry = map->header.next; entry != &map->header;
1380             entry = entry->next) {
1381                 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0)
1382                         continue;
1383                 obj = entry->object.vm_object;
1384                 if (obj == NULL)
1385                         continue;
1386                 if ((entry->eflags & MAP_ENTRY_NEEDS_COPY) != 0 &&
1387                     obj->ref_count != 1)
1388                         continue;
1389                 switch (obj->type) {
1390                 case OBJT_DEFAULT:
1391                 case OBJT_SWAP:
1392                 case OBJT_PHYS:
1393                 case OBJT_VNODE:
1394                         res += obj->resident_page_count;
1395                         break;
1396                 }
1397         }
1398         return (res);
1399 }
1400
1401 void
1402 vm_pageout_oom(int shortage)
1403 {
1404         struct proc *p, *bigproc;
1405         vm_offset_t size, bigsize;
1406         struct thread *td;
1407         struct vmspace *vm;
1408
1409         /*
1410          * We keep the process bigproc locked once we find it to keep anyone
1411          * from messing with it; however, there is a possibility of
1412          * deadlock if process B is bigproc and one of it's child processes
1413          * attempts to propagate a signal to B while we are waiting for A's
1414          * lock while walking this list.  To avoid this, we don't block on
1415          * the process lock but just skip a process if it is already locked.
1416          */
1417         bigproc = NULL;
1418         bigsize = 0;
1419         sx_slock(&allproc_lock);
1420         FOREACH_PROC_IN_SYSTEM(p) {
1421                 int breakout;
1422
1423                 PROC_LOCK(p);
1424
1425                 /*
1426                  * If this is a system, protected or killed process, skip it.
1427                  */
1428                 if (p->p_state != PRS_NORMAL || (p->p_flag & (P_INEXEC |
1429                     P_PROTECTED | P_SYSTEM | P_WEXIT)) != 0 ||
1430                     p->p_pid == 1 || P_KILLED(p) ||
1431                     (p->p_pid < 48 && swap_pager_avail != 0)) {
1432                         PROC_UNLOCK(p);
1433                         continue;
1434                 }
1435                 /*
1436                  * If the process is in a non-running type state,
1437                  * don't touch it.  Check all the threads individually.
1438                  */
1439                 breakout = 0;
1440                 FOREACH_THREAD_IN_PROC(p, td) {
1441                         thread_lock(td);
1442                         if (!TD_ON_RUNQ(td) &&
1443                             !TD_IS_RUNNING(td) &&
1444                             !TD_IS_SLEEPING(td) &&
1445                             !TD_IS_SUSPENDED(td) &&
1446                             !TD_IS_SWAPPED(td)) {
1447                                 thread_unlock(td);
1448                                 breakout = 1;
1449                                 break;
1450                         }
1451                         thread_unlock(td);
1452                 }
1453                 if (breakout) {
1454                         PROC_UNLOCK(p);
1455                         continue;
1456                 }
1457                 /*
1458                  * get the process size
1459                  */
1460                 vm = vmspace_acquire_ref(p);
1461                 if (vm == NULL) {
1462                         PROC_UNLOCK(p);
1463                         continue;
1464                 }
1465                 _PHOLD_LITE(p);
1466                 PROC_UNLOCK(p);
1467                 sx_sunlock(&allproc_lock);
1468                 if (!vm_map_trylock_read(&vm->vm_map)) {
1469                         vmspace_free(vm);
1470                         sx_slock(&allproc_lock);
1471                         PRELE(p);
1472                         continue;
1473                 }
1474                 size = vmspace_swap_count(vm);
1475                 if (shortage == VM_OOM_MEM)
1476                         size += vm_pageout_oom_pagecount(vm);
1477                 vm_map_unlock_read(&vm->vm_map);
1478                 vmspace_free(vm);
1479                 sx_slock(&allproc_lock);
1480
1481                 /*
1482                  * If this process is bigger than the biggest one,
1483                  * remember it.
1484                  */
1485                 if (size > bigsize) {
1486                         if (bigproc != NULL)
1487                                 PRELE(bigproc);
1488                         bigproc = p;
1489                         bigsize = size;
1490                 } else {
1491                         PRELE(p);
1492                 }
1493         }
1494         sx_sunlock(&allproc_lock);
1495         if (bigproc != NULL) {
1496                 if (vm_panic_on_oom != 0)
1497                         panic("out of swap space");
1498                 PROC_LOCK(bigproc);
1499                 killproc(bigproc, "out of swap space");
1500                 sched_nice(bigproc, PRIO_MIN);
1501                 _PRELE(bigproc);
1502                 PROC_UNLOCK(bigproc);
1503                 wakeup(&vm_cnt.v_free_count);
1504         }
1505 }
1506
1507 static void
1508 vm_pageout_worker(void *arg)
1509 {
1510         struct vm_domain *domain;
1511         int domidx, pass;
1512         bool target_met;
1513
1514         domidx = (uintptr_t)arg;
1515         domain = &vm_dom[domidx];
1516         pass = 0;
1517         target_met = true;
1518
1519         /*
1520          * XXXKIB It could be useful to bind pageout daemon threads to
1521          * the cores belonging to the domain, from which vm_page_array
1522          * is allocated.
1523          */
1524
1525         KASSERT(domain->vmd_segs != 0, ("domain without segments"));
1526         domain->vmd_last_active_scan = ticks;
1527         vm_pageout_init_marker(&domain->vmd_marker, PQ_INACTIVE);
1528         vm_pageout_init_marker(&domain->vmd_inacthead, PQ_INACTIVE);
1529         TAILQ_INSERT_HEAD(&domain->vmd_pagequeues[PQ_INACTIVE].pq_pl,
1530             &domain->vmd_inacthead, plinks.q);
1531
1532         /*
1533          * The pageout daemon worker is never done, so loop forever.
1534          */
1535         while (TRUE) {
1536                 mtx_lock(&vm_page_queue_free_mtx);
1537
1538                 /*
1539                  * Generally, after a level >= 1 scan, if there are enough
1540                  * free pages to wakeup the waiters, then they are already
1541                  * awake.  A call to vm_page_free() during the scan awakened
1542                  * them.  However, in the following case, this wakeup serves
1543                  * to bound the amount of time that a thread might wait.
1544                  * Suppose a thread's call to vm_page_alloc() fails, but
1545                  * before that thread calls VM_WAIT, enough pages are freed by
1546                  * other threads to alleviate the free page shortage.  The
1547                  * thread will, nonetheless, wait until another page is freed
1548                  * or this wakeup is performed.
1549                  */
1550                 if (vm_pages_needed && !vm_page_count_min()) {
1551                         vm_pages_needed = false;
1552                         wakeup(&vm_cnt.v_free_count);
1553                 }
1554
1555                 /*
1556                  * Do not clear vm_pageout_wanted until we reach our free page
1557                  * target.  Otherwise, we may be awakened over and over again,
1558                  * wasting CPU time.
1559                  */
1560                 if (vm_pageout_wanted && target_met)
1561                         vm_pageout_wanted = false;
1562
1563                 /*
1564                  * Might the page daemon receive a wakeup call?
1565                  */
1566                 if (vm_pageout_wanted) {
1567                         /*
1568                          * No.  Either vm_pageout_wanted was set by another
1569                          * thread during the previous scan, which must have
1570                          * been a level 0 scan, or vm_pageout_wanted was
1571                          * already set and the scan failed to free enough
1572                          * pages.  If we haven't yet performed a level >= 2
1573                          * scan (unlimited dirty cleaning), then upgrade the
1574                          * level and scan again now.  Otherwise, sleep a bit
1575                          * and try again later.
1576                          */
1577                         mtx_unlock(&vm_page_queue_free_mtx);
1578                         if (pass > 1)
1579                                 pause("psleep", hz / 2);
1580                         pass++;
1581                 } else {
1582                         /*
1583                          * Yes.  Sleep until pages need to be reclaimed or
1584                          * have their reference stats updated.
1585                          */
1586                         if (mtx_sleep(&vm_pageout_wanted,
1587                             &vm_page_queue_free_mtx, PDROP | PVM, "psleep",
1588                             hz) == 0) {
1589                                 PCPU_INC(cnt.v_pdwakeups);
1590                                 pass = 1;
1591                         } else
1592                                 pass = 0;
1593                 }
1594
1595                 target_met = vm_pageout_scan(domain, pass);
1596         }
1597 }
1598
1599 /*
1600  *      vm_pageout_init initialises basic pageout daemon settings.
1601  */
1602 static void
1603 vm_pageout_init(void)
1604 {
1605         /*
1606          * Initialize some paging parameters.
1607          */
1608         vm_cnt.v_interrupt_free_min = 2;
1609         if (vm_cnt.v_page_count < 2000)
1610                 vm_pageout_page_count = 8;
1611
1612         /*
1613          * v_free_reserved needs to include enough for the largest
1614          * swap pager structures plus enough for any pv_entry structs
1615          * when paging.
1616          */
1617         if (vm_cnt.v_page_count > 1024)
1618                 vm_cnt.v_free_min = 4 + (vm_cnt.v_page_count - 1024) / 200;
1619         else
1620                 vm_cnt.v_free_min = 4;
1621         vm_cnt.v_pageout_free_min = (2*MAXBSIZE)/PAGE_SIZE +
1622             vm_cnt.v_interrupt_free_min;
1623         vm_cnt.v_free_reserved = vm_pageout_page_count +
1624             vm_cnt.v_pageout_free_min + (vm_cnt.v_page_count / 768);
1625         vm_cnt.v_free_severe = vm_cnt.v_free_min / 2;
1626         vm_cnt.v_free_target = 4 * vm_cnt.v_free_min + vm_cnt.v_free_reserved;
1627         vm_cnt.v_free_min += vm_cnt.v_free_reserved;
1628         vm_cnt.v_free_severe += vm_cnt.v_free_reserved;
1629         vm_cnt.v_inactive_target = (3 * vm_cnt.v_free_target) / 2;
1630         if (vm_cnt.v_inactive_target > vm_cnt.v_free_count / 3)
1631                 vm_cnt.v_inactive_target = vm_cnt.v_free_count / 3;
1632
1633         /*
1634          * Set the default wakeup threshold to be 10% above the minimum
1635          * page limit.  This keeps the steady state out of shortfall.
1636          */
1637         vm_pageout_wakeup_thresh = (vm_cnt.v_free_min / 10) * 11;
1638
1639         /*
1640          * Set interval in seconds for active scan.  We want to visit each
1641          * page at least once every ten minutes.  This is to prevent worst
1642          * case paging behaviors with stale active LRU.
1643          */
1644         if (vm_pageout_update_period == 0)
1645                 vm_pageout_update_period = 600;
1646
1647         /* XXX does not really belong here */
1648         if (vm_page_max_wired == 0)
1649                 vm_page_max_wired = vm_cnt.v_free_count / 3;
1650 }
1651
1652 /*
1653  *     vm_pageout is the high level pageout daemon.
1654  */
1655 static void
1656 vm_pageout(void)
1657 {
1658         int error;
1659 #ifdef VM_NUMA_ALLOC
1660         int i;
1661 #endif
1662
1663         swap_pager_swap_init();
1664 #ifdef VM_NUMA_ALLOC
1665         for (i = 1; i < vm_ndomains; i++) {
1666                 error = kthread_add(vm_pageout_worker, (void *)(uintptr_t)i,
1667                     curproc, NULL, 0, 0, "dom%d", i);
1668                 if (error != 0) {
1669                         panic("starting pageout for domain %d, error %d\n",
1670                             i, error);
1671                 }
1672         }
1673 #endif
1674         error = kthread_add(uma_reclaim_worker, NULL, curproc, NULL,
1675             0, 0, "uma");
1676         if (error != 0)
1677                 panic("starting uma_reclaim helper, error %d\n", error);
1678         vm_pageout_worker((void *)(uintptr_t)0);
1679 }
1680
1681 /*
1682  * Unless the free page queue lock is held by the caller, this function
1683  * should be regarded as advisory.  Specifically, the caller should
1684  * not msleep() on &vm_cnt.v_free_count following this function unless
1685  * the free page queue lock is held until the msleep() is performed.
1686  */
1687 void
1688 pagedaemon_wakeup(void)
1689 {
1690
1691         if (!vm_pageout_wanted && curthread->td_proc != pageproc) {
1692                 vm_pageout_wanted = true;
1693                 wakeup(&vm_pageout_wanted);
1694         }
1695 }
1696
1697 #if !defined(NO_SWAPPING)
1698 static void
1699 vm_req_vmdaemon(int req)
1700 {
1701         static int lastrun = 0;
1702
1703         mtx_lock(&vm_daemon_mtx);
1704         vm_pageout_req_swapout |= req;
1705         if ((ticks > (lastrun + hz)) || (ticks < lastrun)) {
1706                 wakeup(&vm_daemon_needed);
1707                 lastrun = ticks;
1708         }
1709         mtx_unlock(&vm_daemon_mtx);
1710 }
1711
1712 static void
1713 vm_daemon(void)
1714 {
1715         struct rlimit rsslim;
1716         struct proc *p;
1717         struct thread *td;
1718         struct vmspace *vm;
1719         int breakout, swapout_flags, tryagain, attempts;
1720 #ifdef RACCT
1721         uint64_t rsize, ravailable;
1722 #endif
1723
1724         while (TRUE) {
1725                 mtx_lock(&vm_daemon_mtx);
1726                 msleep(&vm_daemon_needed, &vm_daemon_mtx, PPAUSE, "psleep",
1727 #ifdef RACCT
1728                     racct_enable ? hz : 0
1729 #else
1730                     0
1731 #endif
1732                 );
1733                 swapout_flags = vm_pageout_req_swapout;
1734                 vm_pageout_req_swapout = 0;
1735                 mtx_unlock(&vm_daemon_mtx);
1736                 if (swapout_flags)
1737                         swapout_procs(swapout_flags);
1738
1739                 /*
1740                  * scan the processes for exceeding their rlimits or if
1741                  * process is swapped out -- deactivate pages
1742                  */
1743                 tryagain = 0;
1744                 attempts = 0;
1745 again:
1746                 attempts++;
1747                 sx_slock(&allproc_lock);
1748                 FOREACH_PROC_IN_SYSTEM(p) {
1749                         vm_pindex_t limit, size;
1750
1751                         /*
1752                          * if this is a system process or if we have already
1753                          * looked at this process, skip it.
1754                          */
1755                         PROC_LOCK(p);
1756                         if (p->p_state != PRS_NORMAL ||
1757                             p->p_flag & (P_INEXEC | P_SYSTEM | P_WEXIT)) {
1758                                 PROC_UNLOCK(p);
1759                                 continue;
1760                         }
1761                         /*
1762                          * if the process is in a non-running type state,
1763                          * don't touch it.
1764                          */
1765                         breakout = 0;
1766                         FOREACH_THREAD_IN_PROC(p, td) {
1767                                 thread_lock(td);
1768                                 if (!TD_ON_RUNQ(td) &&
1769                                     !TD_IS_RUNNING(td) &&
1770                                     !TD_IS_SLEEPING(td) &&
1771                                     !TD_IS_SUSPENDED(td)) {
1772                                         thread_unlock(td);
1773                                         breakout = 1;
1774                                         break;
1775                                 }
1776                                 thread_unlock(td);
1777                         }
1778                         if (breakout) {
1779                                 PROC_UNLOCK(p);
1780                                 continue;
1781                         }
1782                         /*
1783                          * get a limit
1784                          */
1785                         lim_rlimit_proc(p, RLIMIT_RSS, &rsslim);
1786                         limit = OFF_TO_IDX(
1787                             qmin(rsslim.rlim_cur, rsslim.rlim_max));
1788
1789                         /*
1790                          * let processes that are swapped out really be
1791                          * swapped out set the limit to nothing (will force a
1792                          * swap-out.)
1793                          */
1794                         if ((p->p_flag & P_INMEM) == 0)
1795                                 limit = 0;      /* XXX */
1796                         vm = vmspace_acquire_ref(p);
1797                         _PHOLD_LITE(p);
1798                         PROC_UNLOCK(p);
1799                         if (vm == NULL) {
1800                                 PRELE(p);
1801                                 continue;
1802                         }
1803                         sx_sunlock(&allproc_lock);
1804
1805                         size = vmspace_resident_count(vm);
1806                         if (size >= limit) {
1807                                 vm_pageout_map_deactivate_pages(
1808                                     &vm->vm_map, limit);
1809                         }
1810 #ifdef RACCT
1811                         if (racct_enable) {
1812                                 rsize = IDX_TO_OFF(size);
1813                                 PROC_LOCK(p);
1814                                 racct_set(p, RACCT_RSS, rsize);
1815                                 ravailable = racct_get_available(p, RACCT_RSS);
1816                                 PROC_UNLOCK(p);
1817                                 if (rsize > ravailable) {
1818                                         /*
1819                                          * Don't be overly aggressive; this
1820                                          * might be an innocent process,
1821                                          * and the limit could've been exceeded
1822                                          * by some memory hog.  Don't try
1823                                          * to deactivate more than 1/4th
1824                                          * of process' resident set size.
1825                                          */
1826                                         if (attempts <= 8) {
1827                                                 if (ravailable < rsize -
1828                                                     (rsize / 4)) {
1829                                                         ravailable = rsize -
1830                                                             (rsize / 4);
1831                                                 }
1832                                         }
1833                                         vm_pageout_map_deactivate_pages(
1834                                             &vm->vm_map,
1835                                             OFF_TO_IDX(ravailable));
1836                                         /* Update RSS usage after paging out. */
1837                                         size = vmspace_resident_count(vm);
1838                                         rsize = IDX_TO_OFF(size);
1839                                         PROC_LOCK(p);
1840                                         racct_set(p, RACCT_RSS, rsize);
1841                                         PROC_UNLOCK(p);
1842                                         if (rsize > ravailable)
1843                                                 tryagain = 1;
1844                                 }
1845                         }
1846 #endif
1847                         vmspace_free(vm);
1848                         sx_slock(&allproc_lock);
1849                         PRELE(p);
1850                 }
1851                 sx_sunlock(&allproc_lock);
1852                 if (tryagain != 0 && attempts <= 10)
1853                         goto again;
1854         }
1855 }
1856 #endif                  /* !defined(NO_SWAPPING) */