]> CyberLeo.Net >> Repos - FreeBSD/FreeBSD.git/blob - sys/vm/vm_object.c
Fix LOCAL_MODULES and improve the make output.
[FreeBSD/FreeBSD.git] / sys / vm / vm_object.c
1 /*-
2  * SPDX-License-Identifier: (BSD-3-Clause AND MIT-CMU)
3  *
4  * Copyright (c) 1991, 1993
5  *      The Regents of the University of California.  All rights reserved.
6  *
7  * This code is derived from software contributed to Berkeley by
8  * The Mach Operating System project at Carnegie-Mellon University.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  * 3. Neither the name of the University nor the names of its contributors
19  *    may be used to endorse or promote products derived from this software
20  *    without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  *
34  *      from: @(#)vm_object.c   8.5 (Berkeley) 3/22/94
35  *
36  *
37  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
38  * All rights reserved.
39  *
40  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
41  *
42  * Permission to use, copy, modify and distribute this software and
43  * its documentation is hereby granted, provided that both the copyright
44  * notice and this permission notice appear in all copies of the
45  * software, derivative works or modified versions, and any portions
46  * thereof, and that both notices appear in supporting documentation.
47  *
48  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
49  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
50  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
51  *
52  * Carnegie Mellon requests users of this software to return to
53  *
54  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
55  *  School of Computer Science
56  *  Carnegie Mellon University
57  *  Pittsburgh PA 15213-3890
58  *
59  * any improvements or extensions that they make and grant Carnegie the
60  * rights to redistribute these changes.
61  */
62
63 /*
64  *      Virtual memory object module.
65  */
66
67 #include <sys/cdefs.h>
68 __FBSDID("$FreeBSD$");
69
70 #include "opt_vm.h"
71
72 #include <sys/param.h>
73 #include <sys/systm.h>
74 #include <sys/cpuset.h>
75 #include <sys/lock.h>
76 #include <sys/mman.h>
77 #include <sys/mount.h>
78 #include <sys/kernel.h>
79 #include <sys/pctrie.h>
80 #include <sys/sysctl.h>
81 #include <sys/mutex.h>
82 #include <sys/proc.h>           /* for curproc, pageproc */
83 #include <sys/socket.h>
84 #include <sys/resourcevar.h>
85 #include <sys/rwlock.h>
86 #include <sys/user.h>
87 #include <sys/vnode.h>
88 #include <sys/vmmeter.h>
89 #include <sys/sx.h>
90
91 #include <vm/vm.h>
92 #include <vm/vm_param.h>
93 #include <vm/pmap.h>
94 #include <vm/vm_map.h>
95 #include <vm/vm_object.h>
96 #include <vm/vm_page.h>
97 #include <vm/vm_pageout.h>
98 #include <vm/vm_pager.h>
99 #include <vm/vm_phys.h>
100 #include <vm/vm_pagequeue.h>
101 #include <vm/swap_pager.h>
102 #include <vm/vm_kern.h>
103 #include <vm/vm_extern.h>
104 #include <vm/vm_radix.h>
105 #include <vm/vm_reserv.h>
106 #include <vm/uma.h>
107
108 static int old_msync;
109 SYSCTL_INT(_vm, OID_AUTO, old_msync, CTLFLAG_RW, &old_msync, 0,
110     "Use old (insecure) msync behavior");
111
112 static int      vm_object_page_collect_flush(vm_object_t object, vm_page_t p,
113                     int pagerflags, int flags, boolean_t *clearobjflags,
114                     boolean_t *eio);
115 static boolean_t vm_object_page_remove_write(vm_page_t p, int flags,
116                     boolean_t *clearobjflags);
117 static void     vm_object_qcollapse(vm_object_t object);
118 static void     vm_object_vndeallocate(vm_object_t object);
119
120 /*
121  *      Virtual memory objects maintain the actual data
122  *      associated with allocated virtual memory.  A given
123  *      page of memory exists within exactly one object.
124  *
125  *      An object is only deallocated when all "references"
126  *      are given up.  Only one "reference" to a given
127  *      region of an object should be writeable.
128  *
129  *      Associated with each object is a list of all resident
130  *      memory pages belonging to that object; this list is
131  *      maintained by the "vm_page" module, and locked by the object's
132  *      lock.
133  *
134  *      Each object also records a "pager" routine which is
135  *      used to retrieve (and store) pages to the proper backing
136  *      storage.  In addition, objects may be backed by other
137  *      objects from which they were virtual-copied.
138  *
139  *      The only items within the object structure which are
140  *      modified after time of creation are:
141  *              reference count         locked by object's lock
142  *              pager routine           locked by object's lock
143  *
144  */
145
146 struct object_q vm_object_list;
147 struct mtx vm_object_list_mtx;  /* lock for object list and count */
148
149 struct vm_object kernel_object_store;
150
151 static SYSCTL_NODE(_vm_stats, OID_AUTO, object, CTLFLAG_RD, 0,
152     "VM object stats");
153
154 static counter_u64_t object_collapses = EARLY_COUNTER;
155 SYSCTL_COUNTER_U64(_vm_stats_object, OID_AUTO, collapses, CTLFLAG_RD,
156     &object_collapses,
157     "VM object collapses");
158
159 static counter_u64_t object_bypasses = EARLY_COUNTER;
160 SYSCTL_COUNTER_U64(_vm_stats_object, OID_AUTO, bypasses, CTLFLAG_RD,
161     &object_bypasses,
162     "VM object bypasses");
163
164 static void
165 counter_startup(void)
166 {
167
168         object_collapses = counter_u64_alloc(M_WAITOK);
169         object_bypasses = counter_u64_alloc(M_WAITOK);
170 }
171 SYSINIT(object_counters, SI_SUB_CPU, SI_ORDER_ANY, counter_startup, NULL);
172
173 static uma_zone_t obj_zone;
174
175 static int vm_object_zinit(void *mem, int size, int flags);
176
177 #ifdef INVARIANTS
178 static void vm_object_zdtor(void *mem, int size, void *arg);
179
180 static void
181 vm_object_zdtor(void *mem, int size, void *arg)
182 {
183         vm_object_t object;
184
185         object = (vm_object_t)mem;
186         KASSERT(object->ref_count == 0,
187             ("object %p ref_count = %d", object, object->ref_count));
188         KASSERT(TAILQ_EMPTY(&object->memq),
189             ("object %p has resident pages in its memq", object));
190         KASSERT(vm_radix_is_empty(&object->rtree),
191             ("object %p has resident pages in its trie", object));
192 #if VM_NRESERVLEVEL > 0
193         KASSERT(LIST_EMPTY(&object->rvq),
194             ("object %p has reservations",
195             object));
196 #endif
197         KASSERT(object->paging_in_progress == 0,
198             ("object %p paging_in_progress = %d",
199             object, object->paging_in_progress));
200         KASSERT(object->resident_page_count == 0,
201             ("object %p resident_page_count = %d",
202             object, object->resident_page_count));
203         KASSERT(object->shadow_count == 0,
204             ("object %p shadow_count = %d",
205             object, object->shadow_count));
206         KASSERT(object->type == OBJT_DEAD,
207             ("object %p has non-dead type %d",
208             object, object->type));
209 }
210 #endif
211
212 static int
213 vm_object_zinit(void *mem, int size, int flags)
214 {
215         vm_object_t object;
216
217         object = (vm_object_t)mem;
218         rw_init_flags(&object->lock, "vm object", RW_DUPOK | RW_NEW);
219
220         /* These are true for any object that has been freed */
221         object->type = OBJT_DEAD;
222         object->ref_count = 0;
223         vm_radix_init(&object->rtree);
224         object->paging_in_progress = 0;
225         object->resident_page_count = 0;
226         object->shadow_count = 0;
227         object->flags = OBJ_DEAD;
228
229         mtx_lock(&vm_object_list_mtx);
230         TAILQ_INSERT_TAIL(&vm_object_list, object, object_list);
231         mtx_unlock(&vm_object_list_mtx);
232         return (0);
233 }
234
235 static void
236 _vm_object_allocate(objtype_t type, vm_pindex_t size, vm_object_t object)
237 {
238
239         TAILQ_INIT(&object->memq);
240         LIST_INIT(&object->shadow_head);
241
242         object->type = type;
243         if (type == OBJT_SWAP)
244                 pctrie_init(&object->un_pager.swp.swp_blks);
245
246         /*
247          * Ensure that swap_pager_swapoff() iteration over object_list
248          * sees up to date type and pctrie head if it observed
249          * non-dead object.
250          */
251         atomic_thread_fence_rel();
252
253         switch (type) {
254         case OBJT_DEAD:
255                 panic("_vm_object_allocate: can't create OBJT_DEAD");
256         case OBJT_DEFAULT:
257         case OBJT_SWAP:
258                 object->flags = OBJ_ONEMAPPING;
259                 break;
260         case OBJT_DEVICE:
261         case OBJT_SG:
262                 object->flags = OBJ_FICTITIOUS | OBJ_UNMANAGED;
263                 break;
264         case OBJT_MGTDEVICE:
265                 object->flags = OBJ_FICTITIOUS;
266                 break;
267         case OBJT_PHYS:
268                 object->flags = OBJ_UNMANAGED;
269                 break;
270         case OBJT_VNODE:
271                 object->flags = 0;
272                 break;
273         default:
274                 panic("_vm_object_allocate: type %d is undefined", type);
275         }
276         object->size = size;
277         object->domain.dr_policy = NULL;
278         object->generation = 1;
279         object->ref_count = 1;
280         object->memattr = VM_MEMATTR_DEFAULT;
281         object->cred = NULL;
282         object->charge = 0;
283         object->handle = NULL;
284         object->backing_object = NULL;
285         object->backing_object_offset = (vm_ooffset_t) 0;
286 #if VM_NRESERVLEVEL > 0
287         LIST_INIT(&object->rvq);
288 #endif
289         umtx_shm_object_init(object);
290 }
291
292 /*
293  *      vm_object_init:
294  *
295  *      Initialize the VM objects module.
296  */
297 void
298 vm_object_init(void)
299 {
300         TAILQ_INIT(&vm_object_list);
301         mtx_init(&vm_object_list_mtx, "vm object_list", NULL, MTX_DEF);
302         
303         rw_init(&kernel_object->lock, "kernel vm object");
304         _vm_object_allocate(OBJT_PHYS, atop(VM_MAX_KERNEL_ADDRESS -
305             VM_MIN_KERNEL_ADDRESS), kernel_object);
306 #if VM_NRESERVLEVEL > 0
307         kernel_object->flags |= OBJ_COLORED;
308         kernel_object->pg_color = (u_short)atop(VM_MIN_KERNEL_ADDRESS);
309 #endif
310
311         /*
312          * The lock portion of struct vm_object must be type stable due
313          * to vm_pageout_fallback_object_lock locking a vm object
314          * without holding any references to it.
315          */
316         obj_zone = uma_zcreate("VM OBJECT", sizeof (struct vm_object), NULL,
317 #ifdef INVARIANTS
318             vm_object_zdtor,
319 #else
320             NULL,
321 #endif
322             vm_object_zinit, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
323
324         vm_radix_zinit();
325 }
326
327 void
328 vm_object_clear_flag(vm_object_t object, u_short bits)
329 {
330
331         VM_OBJECT_ASSERT_WLOCKED(object);
332         object->flags &= ~bits;
333 }
334
335 /*
336  *      Sets the default memory attribute for the specified object.  Pages
337  *      that are allocated to this object are by default assigned this memory
338  *      attribute.
339  *
340  *      Presently, this function must be called before any pages are allocated
341  *      to the object.  In the future, this requirement may be relaxed for
342  *      "default" and "swap" objects.
343  */
344 int
345 vm_object_set_memattr(vm_object_t object, vm_memattr_t memattr)
346 {
347
348         VM_OBJECT_ASSERT_WLOCKED(object);
349         switch (object->type) {
350         case OBJT_DEFAULT:
351         case OBJT_DEVICE:
352         case OBJT_MGTDEVICE:
353         case OBJT_PHYS:
354         case OBJT_SG:
355         case OBJT_SWAP:
356         case OBJT_VNODE:
357                 if (!TAILQ_EMPTY(&object->memq))
358                         return (KERN_FAILURE);
359                 break;
360         case OBJT_DEAD:
361                 return (KERN_INVALID_ARGUMENT);
362         default:
363                 panic("vm_object_set_memattr: object %p is of undefined type",
364                     object);
365         }
366         object->memattr = memattr;
367         return (KERN_SUCCESS);
368 }
369
370 void
371 vm_object_pip_add(vm_object_t object, short i)
372 {
373
374         VM_OBJECT_ASSERT_WLOCKED(object);
375         object->paging_in_progress += i;
376 }
377
378 void
379 vm_object_pip_subtract(vm_object_t object, short i)
380 {
381
382         VM_OBJECT_ASSERT_WLOCKED(object);
383         object->paging_in_progress -= i;
384 }
385
386 void
387 vm_object_pip_wakeup(vm_object_t object)
388 {
389
390         VM_OBJECT_ASSERT_WLOCKED(object);
391         object->paging_in_progress--;
392         if ((object->flags & OBJ_PIPWNT) && object->paging_in_progress == 0) {
393                 vm_object_clear_flag(object, OBJ_PIPWNT);
394                 wakeup(object);
395         }
396 }
397
398 void
399 vm_object_pip_wakeupn(vm_object_t object, short i)
400 {
401
402         VM_OBJECT_ASSERT_WLOCKED(object);
403         if (i)
404                 object->paging_in_progress -= i;
405         if ((object->flags & OBJ_PIPWNT) && object->paging_in_progress == 0) {
406                 vm_object_clear_flag(object, OBJ_PIPWNT);
407                 wakeup(object);
408         }
409 }
410
411 void
412 vm_object_pip_wait(vm_object_t object, char *waitid)
413 {
414
415         VM_OBJECT_ASSERT_WLOCKED(object);
416         while (object->paging_in_progress) {
417                 object->flags |= OBJ_PIPWNT;
418                 VM_OBJECT_SLEEP(object, object, PVM, waitid, 0);
419         }
420 }
421
422 /*
423  *      vm_object_allocate:
424  *
425  *      Returns a new object with the given size.
426  */
427 vm_object_t
428 vm_object_allocate(objtype_t type, vm_pindex_t size)
429 {
430         vm_object_t object;
431
432         object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK);
433         _vm_object_allocate(type, size, object);
434         return (object);
435 }
436
437
438 /*
439  *      vm_object_reference:
440  *
441  *      Gets another reference to the given object.  Note: OBJ_DEAD
442  *      objects can be referenced during final cleaning.
443  */
444 void
445 vm_object_reference(vm_object_t object)
446 {
447         if (object == NULL)
448                 return;
449         VM_OBJECT_WLOCK(object);
450         vm_object_reference_locked(object);
451         VM_OBJECT_WUNLOCK(object);
452 }
453
454 /*
455  *      vm_object_reference_locked:
456  *
457  *      Gets another reference to the given object.
458  *
459  *      The object must be locked.
460  */
461 void
462 vm_object_reference_locked(vm_object_t object)
463 {
464         struct vnode *vp;
465
466         VM_OBJECT_ASSERT_WLOCKED(object);
467         object->ref_count++;
468         if (object->type == OBJT_VNODE) {
469                 vp = object->handle;
470                 vref(vp);
471         }
472 }
473
474 /*
475  * Handle deallocating an object of type OBJT_VNODE.
476  */
477 static void
478 vm_object_vndeallocate(vm_object_t object)
479 {
480         struct vnode *vp = (struct vnode *) object->handle;
481
482         VM_OBJECT_ASSERT_WLOCKED(object);
483         KASSERT(object->type == OBJT_VNODE,
484             ("vm_object_vndeallocate: not a vnode object"));
485         KASSERT(vp != NULL, ("vm_object_vndeallocate: missing vp"));
486 #ifdef INVARIANTS
487         if (object->ref_count == 0) {
488                 vn_printf(vp, "vm_object_vndeallocate ");
489                 panic("vm_object_vndeallocate: bad object reference count");
490         }
491 #endif
492
493         if (!umtx_shm_vnobj_persistent && object->ref_count == 1)
494                 umtx_shm_object_terminated(object);
495
496         object->ref_count--;
497
498         /* vrele may need the vnode lock. */
499         VM_OBJECT_WUNLOCK(object);
500         vrele(vp);
501 }
502
503 /*
504  *      vm_object_deallocate:
505  *
506  *      Release a reference to the specified object,
507  *      gained either through a vm_object_allocate
508  *      or a vm_object_reference call.  When all references
509  *      are gone, storage associated with this object
510  *      may be relinquished.
511  *
512  *      No object may be locked.
513  */
514 void
515 vm_object_deallocate(vm_object_t object)
516 {
517         vm_object_t temp;
518         struct vnode *vp;
519
520         while (object != NULL) {
521                 VM_OBJECT_WLOCK(object);
522                 if (object->type == OBJT_VNODE) {
523                         vm_object_vndeallocate(object);
524                         return;
525                 }
526
527                 KASSERT(object->ref_count != 0,
528                         ("vm_object_deallocate: object deallocated too many times: %d", object->type));
529
530                 /*
531                  * If the reference count goes to 0 we start calling
532                  * vm_object_terminate() on the object chain.
533                  * A ref count of 1 may be a special case depending on the
534                  * shadow count being 0 or 1.
535                  */
536                 object->ref_count--;
537                 if (object->ref_count > 1) {
538                         VM_OBJECT_WUNLOCK(object);
539                         return;
540                 } else if (object->ref_count == 1) {
541                         if (object->type == OBJT_SWAP &&
542                             (object->flags & OBJ_TMPFS) != 0) {
543                                 vp = object->un_pager.swp.swp_tmpfs;
544                                 vhold(vp);
545                                 VM_OBJECT_WUNLOCK(object);
546                                 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
547                                 VM_OBJECT_WLOCK(object);
548                                 if (object->type == OBJT_DEAD ||
549                                     object->ref_count != 1) {
550                                         VM_OBJECT_WUNLOCK(object);
551                                         VOP_UNLOCK(vp, 0);
552                                         vdrop(vp);
553                                         return;
554                                 }
555                                 if ((object->flags & OBJ_TMPFS) != 0)
556                                         VOP_UNSET_TEXT(vp);
557                                 VOP_UNLOCK(vp, 0);
558                                 vdrop(vp);
559                         }
560                         if (object->shadow_count == 0 &&
561                             object->handle == NULL &&
562                             (object->type == OBJT_DEFAULT ||
563                             (object->type == OBJT_SWAP &&
564                             (object->flags & OBJ_TMPFS_NODE) == 0))) {
565                                 vm_object_set_flag(object, OBJ_ONEMAPPING);
566                         } else if ((object->shadow_count == 1) &&
567                             (object->handle == NULL) &&
568                             (object->type == OBJT_DEFAULT ||
569                              object->type == OBJT_SWAP)) {
570                                 vm_object_t robject;
571
572                                 robject = LIST_FIRST(&object->shadow_head);
573                                 KASSERT(robject != NULL,
574                                     ("vm_object_deallocate: ref_count: %d, shadow_count: %d",
575                                          object->ref_count,
576                                          object->shadow_count));
577                                 KASSERT((robject->flags & OBJ_TMPFS_NODE) == 0,
578                                     ("shadowed tmpfs v_object %p", object));
579                                 if (!VM_OBJECT_TRYWLOCK(robject)) {
580                                         /*
581                                          * Avoid a potential deadlock.
582                                          */
583                                         object->ref_count++;
584                                         VM_OBJECT_WUNLOCK(object);
585                                         /*
586                                          * More likely than not the thread
587                                          * holding robject's lock has lower
588                                          * priority than the current thread.
589                                          * Let the lower priority thread run.
590                                          */
591                                         pause("vmo_de", 1);
592                                         continue;
593                                 }
594                                 /*
595                                  * Collapse object into its shadow unless its
596                                  * shadow is dead.  In that case, object will
597                                  * be deallocated by the thread that is
598                                  * deallocating its shadow.
599                                  */
600                                 if ((robject->flags & OBJ_DEAD) == 0 &&
601                                     (robject->handle == NULL) &&
602                                     (robject->type == OBJT_DEFAULT ||
603                                      robject->type == OBJT_SWAP)) {
604
605                                         robject->ref_count++;
606 retry:
607                                         if (robject->paging_in_progress) {
608                                                 VM_OBJECT_WUNLOCK(object);
609                                                 vm_object_pip_wait(robject,
610                                                     "objde1");
611                                                 temp = robject->backing_object;
612                                                 if (object == temp) {
613                                                         VM_OBJECT_WLOCK(object);
614                                                         goto retry;
615                                                 }
616                                         } else if (object->paging_in_progress) {
617                                                 VM_OBJECT_WUNLOCK(robject);
618                                                 object->flags |= OBJ_PIPWNT;
619                                                 VM_OBJECT_SLEEP(object, object,
620                                                     PDROP | PVM, "objde2", 0);
621                                                 VM_OBJECT_WLOCK(robject);
622                                                 temp = robject->backing_object;
623                                                 if (object == temp) {
624                                                         VM_OBJECT_WLOCK(object);
625                                                         goto retry;
626                                                 }
627                                         } else
628                                                 VM_OBJECT_WUNLOCK(object);
629
630                                         if (robject->ref_count == 1) {
631                                                 robject->ref_count--;
632                                                 object = robject;
633                                                 goto doterm;
634                                         }
635                                         object = robject;
636                                         vm_object_collapse(object);
637                                         VM_OBJECT_WUNLOCK(object);
638                                         continue;
639                                 }
640                                 VM_OBJECT_WUNLOCK(robject);
641                         }
642                         VM_OBJECT_WUNLOCK(object);
643                         return;
644                 }
645 doterm:
646                 umtx_shm_object_terminated(object);
647                 temp = object->backing_object;
648                 if (temp != NULL) {
649                         KASSERT((object->flags & OBJ_TMPFS_NODE) == 0,
650                             ("shadowed tmpfs v_object 2 %p", object));
651                         VM_OBJECT_WLOCK(temp);
652                         LIST_REMOVE(object, shadow_list);
653                         temp->shadow_count--;
654                         VM_OBJECT_WUNLOCK(temp);
655                         object->backing_object = NULL;
656                 }
657                 /*
658                  * Don't double-terminate, we could be in a termination
659                  * recursion due to the terminate having to sync data
660                  * to disk.
661                  */
662                 if ((object->flags & OBJ_DEAD) == 0)
663                         vm_object_terminate(object);
664                 else
665                         VM_OBJECT_WUNLOCK(object);
666                 object = temp;
667         }
668 }
669
670 /*
671  *      vm_object_destroy removes the object from the global object list
672  *      and frees the space for the object.
673  */
674 void
675 vm_object_destroy(vm_object_t object)
676 {
677
678         /*
679          * Release the allocation charge.
680          */
681         if (object->cred != NULL) {
682                 swap_release_by_cred(object->charge, object->cred);
683                 object->charge = 0;
684                 crfree(object->cred);
685                 object->cred = NULL;
686         }
687
688         /*
689          * Free the space for the object.
690          */
691         uma_zfree(obj_zone, object);
692 }
693
694 /*
695  *      vm_object_terminate_pages removes any remaining pageable pages
696  *      from the object and resets the object to an empty state.
697  */
698 static void
699 vm_object_terminate_pages(vm_object_t object)
700 {
701         vm_page_t p, p_next;
702         struct mtx *mtx;
703
704         VM_OBJECT_ASSERT_WLOCKED(object);
705
706         mtx = NULL;
707
708         /*
709          * Free any remaining pageable pages.  This also removes them from the
710          * paging queues.  However, don't free wired pages, just remove them
711          * from the object.  Rather than incrementally removing each page from
712          * the object, the page and object are reset to any empty state. 
713          */
714         TAILQ_FOREACH_SAFE(p, &object->memq, listq, p_next) {
715                 vm_page_assert_unbusied(p);
716                 if ((object->flags & OBJ_UNMANAGED) == 0)
717                         /*
718                          * vm_page_free_prep() only needs the page
719                          * lock for managed pages.
720                          */
721                         vm_page_change_lock(p, &mtx);
722                 p->object = NULL;
723                 if (vm_page_wired(p))
724                         continue;
725                 VM_CNT_INC(v_pfree);
726                 vm_page_free(p);
727         }
728         if (mtx != NULL)
729                 mtx_unlock(mtx);
730
731         /*
732          * If the object contained any pages, then reset it to an empty state.
733          * None of the object's fields, including "resident_page_count", were
734          * modified by the preceding loop.
735          */
736         if (object->resident_page_count != 0) {
737                 vm_radix_reclaim_allnodes(&object->rtree);
738                 TAILQ_INIT(&object->memq);
739                 object->resident_page_count = 0;
740                 if (object->type == OBJT_VNODE)
741                         vdrop(object->handle);
742         }
743 }
744
745 /*
746  *      vm_object_terminate actually destroys the specified object, freeing
747  *      up all previously used resources.
748  *
749  *      The object must be locked.
750  *      This routine may block.
751  */
752 void
753 vm_object_terminate(vm_object_t object)
754 {
755
756         VM_OBJECT_ASSERT_WLOCKED(object);
757
758         /*
759          * Make sure no one uses us.
760          */
761         vm_object_set_flag(object, OBJ_DEAD);
762
763         /*
764          * wait for the pageout daemon to be done with the object
765          */
766         vm_object_pip_wait(object, "objtrm");
767
768         KASSERT(!object->paging_in_progress,
769                 ("vm_object_terminate: pageout in progress"));
770
771         /*
772          * Clean and free the pages, as appropriate. All references to the
773          * object are gone, so we don't need to lock it.
774          */
775         if (object->type == OBJT_VNODE) {
776                 struct vnode *vp = (struct vnode *)object->handle;
777
778                 /*
779                  * Clean pages and flush buffers.
780                  */
781                 vm_object_page_clean(object, 0, 0, OBJPC_SYNC);
782                 VM_OBJECT_WUNLOCK(object);
783
784                 vinvalbuf(vp, V_SAVE, 0, 0);
785
786                 BO_LOCK(&vp->v_bufobj);
787                 vp->v_bufobj.bo_flag |= BO_DEAD;
788                 BO_UNLOCK(&vp->v_bufobj);
789
790                 VM_OBJECT_WLOCK(object);
791         }
792
793         KASSERT(object->ref_count == 0, 
794                 ("vm_object_terminate: object with references, ref_count=%d",
795                 object->ref_count));
796
797         if ((object->flags & OBJ_PG_DTOR) == 0)
798                 vm_object_terminate_pages(object);
799
800 #if VM_NRESERVLEVEL > 0
801         if (__predict_false(!LIST_EMPTY(&object->rvq)))
802                 vm_reserv_break_all(object);
803 #endif
804
805         KASSERT(object->cred == NULL || object->type == OBJT_DEFAULT ||
806             object->type == OBJT_SWAP,
807             ("%s: non-swap obj %p has cred", __func__, object));
808
809         /*
810          * Let the pager know object is dead.
811          */
812         vm_pager_deallocate(object);
813         VM_OBJECT_WUNLOCK(object);
814
815         vm_object_destroy(object);
816 }
817
818 /*
819  * Make the page read-only so that we can clear the object flags.  However, if
820  * this is a nosync mmap then the object is likely to stay dirty so do not
821  * mess with the page and do not clear the object flags.  Returns TRUE if the
822  * page should be flushed, and FALSE otherwise.
823  */
824 static boolean_t
825 vm_object_page_remove_write(vm_page_t p, int flags, boolean_t *clearobjflags)
826 {
827
828         /*
829          * If we have been asked to skip nosync pages and this is a
830          * nosync page, skip it.  Note that the object flags were not
831          * cleared in this case so we do not have to set them.
832          */
833         if ((flags & OBJPC_NOSYNC) != 0 && (p->oflags & VPO_NOSYNC) != 0) {
834                 *clearobjflags = FALSE;
835                 return (FALSE);
836         } else {
837                 pmap_remove_write(p);
838                 return (p->dirty != 0);
839         }
840 }
841
842 /*
843  *      vm_object_page_clean
844  *
845  *      Clean all dirty pages in the specified range of object.  Leaves page 
846  *      on whatever queue it is currently on.   If NOSYNC is set then do not
847  *      write out pages with VPO_NOSYNC set (originally comes from MAP_NOSYNC),
848  *      leaving the object dirty.
849  *
850  *      When stuffing pages asynchronously, allow clustering.  XXX we need a
851  *      synchronous clustering mode implementation.
852  *
853  *      Odd semantics: if start == end, we clean everything.
854  *
855  *      The object must be locked.
856  *
857  *      Returns FALSE if some page from the range was not written, as
858  *      reported by the pager, and TRUE otherwise.
859  */
860 boolean_t
861 vm_object_page_clean(vm_object_t object, vm_ooffset_t start, vm_ooffset_t end,
862     int flags)
863 {
864         vm_page_t np, p;
865         vm_pindex_t pi, tend, tstart;
866         int curgeneration, n, pagerflags;
867         boolean_t clearobjflags, eio, res;
868
869         VM_OBJECT_ASSERT_WLOCKED(object);
870
871         /*
872          * The OBJ_MIGHTBEDIRTY flag is only set for OBJT_VNODE
873          * objects.  The check below prevents the function from
874          * operating on non-vnode objects.
875          */
876         if ((object->flags & OBJ_MIGHTBEDIRTY) == 0 ||
877             object->resident_page_count == 0)
878                 return (TRUE);
879
880         pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) != 0 ?
881             VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK;
882         pagerflags |= (flags & OBJPC_INVAL) != 0 ? VM_PAGER_PUT_INVAL : 0;
883
884         tstart = OFF_TO_IDX(start);
885         tend = (end == 0) ? object->size : OFF_TO_IDX(end + PAGE_MASK);
886         clearobjflags = tstart == 0 && tend >= object->size;
887         res = TRUE;
888
889 rescan:
890         curgeneration = object->generation;
891
892         for (p = vm_page_find_least(object, tstart); p != NULL; p = np) {
893                 pi = p->pindex;
894                 if (pi >= tend)
895                         break;
896                 np = TAILQ_NEXT(p, listq);
897                 if (p->valid == 0)
898                         continue;
899                 if (vm_page_sleep_if_busy(p, "vpcwai")) {
900                         if (object->generation != curgeneration) {
901                                 if ((flags & OBJPC_SYNC) != 0)
902                                         goto rescan;
903                                 else
904                                         clearobjflags = FALSE;
905                         }
906                         np = vm_page_find_least(object, pi);
907                         continue;
908                 }
909                 if (!vm_object_page_remove_write(p, flags, &clearobjflags))
910                         continue;
911
912                 n = vm_object_page_collect_flush(object, p, pagerflags,
913                     flags, &clearobjflags, &eio);
914                 if (eio) {
915                         res = FALSE;
916                         clearobjflags = FALSE;
917                 }
918                 if (object->generation != curgeneration) {
919                         if ((flags & OBJPC_SYNC) != 0)
920                                 goto rescan;
921                         else
922                                 clearobjflags = FALSE;
923                 }
924
925                 /*
926                  * If the VOP_PUTPAGES() did a truncated write, so
927                  * that even the first page of the run is not fully
928                  * written, vm_pageout_flush() returns 0 as the run
929                  * length.  Since the condition that caused truncated
930                  * write may be permanent, e.g. exhausted free space,
931                  * accepting n == 0 would cause an infinite loop.
932                  *
933                  * Forwarding the iterator leaves the unwritten page
934                  * behind, but there is not much we can do there if
935                  * filesystem refuses to write it.
936                  */
937                 if (n == 0) {
938                         n = 1;
939                         clearobjflags = FALSE;
940                 }
941                 np = vm_page_find_least(object, pi + n);
942         }
943 #if 0
944         VOP_FSYNC(vp, (pagerflags & VM_PAGER_PUT_SYNC) ? MNT_WAIT : 0);
945 #endif
946
947         if (clearobjflags)
948                 vm_object_clear_flag(object, OBJ_MIGHTBEDIRTY);
949         return (res);
950 }
951
952 static int
953 vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int pagerflags,
954     int flags, boolean_t *clearobjflags, boolean_t *eio)
955 {
956         vm_page_t ma[vm_pageout_page_count], p_first, tp;
957         int count, i, mreq, runlen;
958
959         vm_page_lock_assert(p, MA_NOTOWNED);
960         VM_OBJECT_ASSERT_WLOCKED(object);
961
962         count = 1;
963         mreq = 0;
964
965         for (tp = p; count < vm_pageout_page_count; count++) {
966                 tp = vm_page_next(tp);
967                 if (tp == NULL || vm_page_busied(tp))
968                         break;
969                 if (!vm_object_page_remove_write(tp, flags, clearobjflags))
970                         break;
971         }
972
973         for (p_first = p; count < vm_pageout_page_count; count++) {
974                 tp = vm_page_prev(p_first);
975                 if (tp == NULL || vm_page_busied(tp))
976                         break;
977                 if (!vm_object_page_remove_write(tp, flags, clearobjflags))
978                         break;
979                 p_first = tp;
980                 mreq++;
981         }
982
983         for (tp = p_first, i = 0; i < count; tp = TAILQ_NEXT(tp, listq), i++)
984                 ma[i] = tp;
985
986         vm_pageout_flush(ma, count, pagerflags, mreq, &runlen, eio);
987         return (runlen);
988 }
989
990 /*
991  * Note that there is absolutely no sense in writing out
992  * anonymous objects, so we track down the vnode object
993  * to write out.
994  * We invalidate (remove) all pages from the address space
995  * for semantic correctness.
996  *
997  * If the backing object is a device object with unmanaged pages, then any
998  * mappings to the specified range of pages must be removed before this
999  * function is called.
1000  *
1001  * Note: certain anonymous maps, such as MAP_NOSYNC maps,
1002  * may start out with a NULL object.
1003  */
1004 boolean_t
1005 vm_object_sync(vm_object_t object, vm_ooffset_t offset, vm_size_t size,
1006     boolean_t syncio, boolean_t invalidate)
1007 {
1008         vm_object_t backing_object;
1009         struct vnode *vp;
1010         struct mount *mp;
1011         int error, flags, fsync_after;
1012         boolean_t res;
1013
1014         if (object == NULL)
1015                 return (TRUE);
1016         res = TRUE;
1017         error = 0;
1018         VM_OBJECT_WLOCK(object);
1019         while ((backing_object = object->backing_object) != NULL) {
1020                 VM_OBJECT_WLOCK(backing_object);
1021                 offset += object->backing_object_offset;
1022                 VM_OBJECT_WUNLOCK(object);
1023                 object = backing_object;
1024                 if (object->size < OFF_TO_IDX(offset + size))
1025                         size = IDX_TO_OFF(object->size) - offset;
1026         }
1027         /*
1028          * Flush pages if writing is allowed, invalidate them
1029          * if invalidation requested.  Pages undergoing I/O
1030          * will be ignored by vm_object_page_remove().
1031          *
1032          * We cannot lock the vnode and then wait for paging
1033          * to complete without deadlocking against vm_fault.
1034          * Instead we simply call vm_object_page_remove() and
1035          * allow it to block internally on a page-by-page
1036          * basis when it encounters pages undergoing async
1037          * I/O.
1038          */
1039         if (object->type == OBJT_VNODE &&
1040             (object->flags & OBJ_MIGHTBEDIRTY) != 0 &&
1041             ((vp = object->handle)->v_vflag & VV_NOSYNC) == 0) {
1042                 VM_OBJECT_WUNLOCK(object);
1043                 (void) vn_start_write(vp, &mp, V_WAIT);
1044                 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
1045                 if (syncio && !invalidate && offset == 0 &&
1046                     atop(size) == object->size) {
1047                         /*
1048                          * If syncing the whole mapping of the file,
1049                          * it is faster to schedule all the writes in
1050                          * async mode, also allowing the clustering,
1051                          * and then wait for i/o to complete.
1052                          */
1053                         flags = 0;
1054                         fsync_after = TRUE;
1055                 } else {
1056                         flags = (syncio || invalidate) ? OBJPC_SYNC : 0;
1057                         flags |= invalidate ? (OBJPC_SYNC | OBJPC_INVAL) : 0;
1058                         fsync_after = FALSE;
1059                 }
1060                 VM_OBJECT_WLOCK(object);
1061                 res = vm_object_page_clean(object, offset, offset + size,
1062                     flags);
1063                 VM_OBJECT_WUNLOCK(object);
1064                 if (fsync_after)
1065                         error = VOP_FSYNC(vp, MNT_WAIT, curthread);
1066                 VOP_UNLOCK(vp, 0);
1067                 vn_finished_write(mp);
1068                 if (error != 0)
1069                         res = FALSE;
1070                 VM_OBJECT_WLOCK(object);
1071         }
1072         if ((object->type == OBJT_VNODE ||
1073              object->type == OBJT_DEVICE) && invalidate) {
1074                 if (object->type == OBJT_DEVICE)
1075                         /*
1076                          * The option OBJPR_NOTMAPPED must be passed here
1077                          * because vm_object_page_remove() cannot remove
1078                          * unmanaged mappings.
1079                          */
1080                         flags = OBJPR_NOTMAPPED;
1081                 else if (old_msync)
1082                         flags = 0;
1083                 else
1084                         flags = OBJPR_CLEANONLY;
1085                 vm_object_page_remove(object, OFF_TO_IDX(offset),
1086                     OFF_TO_IDX(offset + size + PAGE_MASK), flags);
1087         }
1088         VM_OBJECT_WUNLOCK(object);
1089         return (res);
1090 }
1091
1092 /*
1093  * Determine whether the given advice can be applied to the object.  Advice is
1094  * not applied to unmanaged pages since they never belong to page queues, and
1095  * since MADV_FREE is destructive, it can apply only to anonymous pages that
1096  * have been mapped at most once.
1097  */
1098 static bool
1099 vm_object_advice_applies(vm_object_t object, int advice)
1100 {
1101
1102         if ((object->flags & OBJ_UNMANAGED) != 0)
1103                 return (false);
1104         if (advice != MADV_FREE)
1105                 return (true);
1106         return ((object->type == OBJT_DEFAULT || object->type == OBJT_SWAP) &&
1107             (object->flags & OBJ_ONEMAPPING) != 0);
1108 }
1109
1110 static void
1111 vm_object_madvise_freespace(vm_object_t object, int advice, vm_pindex_t pindex,
1112     vm_size_t size)
1113 {
1114
1115         if (advice == MADV_FREE && object->type == OBJT_SWAP)
1116                 swap_pager_freespace(object, pindex, size);
1117 }
1118
1119 /*
1120  *      vm_object_madvise:
1121  *
1122  *      Implements the madvise function at the object/page level.
1123  *
1124  *      MADV_WILLNEED   (any object)
1125  *
1126  *          Activate the specified pages if they are resident.
1127  *
1128  *      MADV_DONTNEED   (any object)
1129  *
1130  *          Deactivate the specified pages if they are resident.
1131  *
1132  *      MADV_FREE       (OBJT_DEFAULT/OBJT_SWAP objects,
1133  *                       OBJ_ONEMAPPING only)
1134  *
1135  *          Deactivate and clean the specified pages if they are
1136  *          resident.  This permits the process to reuse the pages
1137  *          without faulting or the kernel to reclaim the pages
1138  *          without I/O.
1139  */
1140 void
1141 vm_object_madvise(vm_object_t object, vm_pindex_t pindex, vm_pindex_t end,
1142     int advice)
1143 {
1144         vm_pindex_t tpindex;
1145         vm_object_t backing_object, tobject;
1146         vm_page_t m, tm;
1147
1148         if (object == NULL)
1149                 return;
1150
1151 relookup:
1152         VM_OBJECT_WLOCK(object);
1153         if (!vm_object_advice_applies(object, advice)) {
1154                 VM_OBJECT_WUNLOCK(object);
1155                 return;
1156         }
1157         for (m = vm_page_find_least(object, pindex); pindex < end; pindex++) {
1158                 tobject = object;
1159
1160                 /*
1161                  * If the next page isn't resident in the top-level object, we
1162                  * need to search the shadow chain.  When applying MADV_FREE, we
1163                  * take care to release any swap space used to store
1164                  * non-resident pages.
1165                  */
1166                 if (m == NULL || pindex < m->pindex) {
1167                         /*
1168                          * Optimize a common case: if the top-level object has
1169                          * no backing object, we can skip over the non-resident
1170                          * range in constant time.
1171                          */
1172                         if (object->backing_object == NULL) {
1173                                 tpindex = (m != NULL && m->pindex < end) ?
1174                                     m->pindex : end;
1175                                 vm_object_madvise_freespace(object, advice,
1176                                     pindex, tpindex - pindex);
1177                                 if ((pindex = tpindex) == end)
1178                                         break;
1179                                 goto next_page;
1180                         }
1181
1182                         tpindex = pindex;
1183                         do {
1184                                 vm_object_madvise_freespace(tobject, advice,
1185                                     tpindex, 1);
1186                                 /*
1187                                  * Prepare to search the next object in the
1188                                  * chain.
1189                                  */
1190                                 backing_object = tobject->backing_object;
1191                                 if (backing_object == NULL)
1192                                         goto next_pindex;
1193                                 VM_OBJECT_WLOCK(backing_object);
1194                                 tpindex +=
1195                                     OFF_TO_IDX(tobject->backing_object_offset);
1196                                 if (tobject != object)
1197                                         VM_OBJECT_WUNLOCK(tobject);
1198                                 tobject = backing_object;
1199                                 if (!vm_object_advice_applies(tobject, advice))
1200                                         goto next_pindex;
1201                         } while ((tm = vm_page_lookup(tobject, tpindex)) ==
1202                             NULL);
1203                 } else {
1204 next_page:
1205                         tm = m;
1206                         m = TAILQ_NEXT(m, listq);
1207                 }
1208
1209                 /*
1210                  * If the page is not in a normal state, skip it.
1211                  */
1212                 if (tm->valid != VM_PAGE_BITS_ALL)
1213                         goto next_pindex;
1214                 vm_page_lock(tm);
1215                 if (vm_page_wired(tm)) {
1216                         vm_page_unlock(tm);
1217                         goto next_pindex;
1218                 }
1219                 KASSERT((tm->flags & PG_FICTITIOUS) == 0,
1220                     ("vm_object_madvise: page %p is fictitious", tm));
1221                 KASSERT((tm->oflags & VPO_UNMANAGED) == 0,
1222                     ("vm_object_madvise: page %p is not managed", tm));
1223                 if (vm_page_busied(tm)) {
1224                         if (object != tobject)
1225                                 VM_OBJECT_WUNLOCK(tobject);
1226                         VM_OBJECT_WUNLOCK(object);
1227                         if (advice == MADV_WILLNEED) {
1228                                 /*
1229                                  * Reference the page before unlocking and
1230                                  * sleeping so that the page daemon is less
1231                                  * likely to reclaim it.
1232                                  */
1233                                 vm_page_aflag_set(tm, PGA_REFERENCED);
1234                         }
1235                         vm_page_busy_sleep(tm, "madvpo", false);
1236                         goto relookup;
1237                 }
1238                 vm_page_advise(tm, advice);
1239                 vm_page_unlock(tm);
1240                 vm_object_madvise_freespace(tobject, advice, tm->pindex, 1);
1241 next_pindex:
1242                 if (tobject != object)
1243                         VM_OBJECT_WUNLOCK(tobject);
1244         }
1245         VM_OBJECT_WUNLOCK(object);
1246 }
1247
1248 /*
1249  *      vm_object_shadow:
1250  *
1251  *      Create a new object which is backed by the
1252  *      specified existing object range.  The source
1253  *      object reference is deallocated.
1254  *
1255  *      The new object and offset into that object
1256  *      are returned in the source parameters.
1257  */
1258 void
1259 vm_object_shadow(
1260         vm_object_t *object,    /* IN/OUT */
1261         vm_ooffset_t *offset,   /* IN/OUT */
1262         vm_size_t length)
1263 {
1264         vm_object_t source;
1265         vm_object_t result;
1266
1267         source = *object;
1268
1269         /*
1270          * Don't create the new object if the old object isn't shared.
1271          */
1272         if (source != NULL) {
1273                 VM_OBJECT_WLOCK(source);
1274                 if (source->ref_count == 1 &&
1275                     source->handle == NULL &&
1276                     (source->type == OBJT_DEFAULT ||
1277                      source->type == OBJT_SWAP)) {
1278                         VM_OBJECT_WUNLOCK(source);
1279                         return;
1280                 }
1281                 VM_OBJECT_WUNLOCK(source);
1282         }
1283
1284         /*
1285          * Allocate a new object with the given length.
1286          */
1287         result = vm_object_allocate(OBJT_DEFAULT, atop(length));
1288
1289         /*
1290          * The new object shadows the source object, adding a reference to it.
1291          * Our caller changes his reference to point to the new object,
1292          * removing a reference to the source object.  Net result: no change
1293          * of reference count.
1294          *
1295          * Try to optimize the result object's page color when shadowing
1296          * in order to maintain page coloring consistency in the combined 
1297          * shadowed object.
1298          */
1299         result->backing_object = source;
1300         /*
1301          * Store the offset into the source object, and fix up the offset into
1302          * the new object.
1303          */
1304         result->backing_object_offset = *offset;
1305         if (source != NULL) {
1306                 VM_OBJECT_WLOCK(source);
1307                 result->domain = source->domain;
1308                 LIST_INSERT_HEAD(&source->shadow_head, result, shadow_list);
1309                 source->shadow_count++;
1310 #if VM_NRESERVLEVEL > 0
1311                 result->flags |= source->flags & OBJ_COLORED;
1312                 result->pg_color = (source->pg_color + OFF_TO_IDX(*offset)) &
1313                     ((1 << (VM_NFREEORDER - 1)) - 1);
1314 #endif
1315                 VM_OBJECT_WUNLOCK(source);
1316         }
1317
1318
1319         /*
1320          * Return the new things
1321          */
1322         *offset = 0;
1323         *object = result;
1324 }
1325
1326 /*
1327  *      vm_object_split:
1328  *
1329  * Split the pages in a map entry into a new object.  This affords
1330  * easier removal of unused pages, and keeps object inheritance from
1331  * being a negative impact on memory usage.
1332  */
1333 void
1334 vm_object_split(vm_map_entry_t entry)
1335 {
1336         vm_page_t m, m_next;
1337         vm_object_t orig_object, new_object, source;
1338         vm_pindex_t idx, offidxstart;
1339         vm_size_t size;
1340
1341         orig_object = entry->object.vm_object;
1342         if (orig_object->type != OBJT_DEFAULT && orig_object->type != OBJT_SWAP)
1343                 return;
1344         if (orig_object->ref_count <= 1)
1345                 return;
1346         VM_OBJECT_WUNLOCK(orig_object);
1347
1348         offidxstart = OFF_TO_IDX(entry->offset);
1349         size = atop(entry->end - entry->start);
1350
1351         /*
1352          * If swap_pager_copy() is later called, it will convert new_object
1353          * into a swap object.
1354          */
1355         new_object = vm_object_allocate(OBJT_DEFAULT, size);
1356
1357         /*
1358          * At this point, the new object is still private, so the order in
1359          * which the original and new objects are locked does not matter.
1360          */
1361         VM_OBJECT_WLOCK(new_object);
1362         VM_OBJECT_WLOCK(orig_object);
1363         new_object->domain = orig_object->domain;
1364         source = orig_object->backing_object;
1365         if (source != NULL) {
1366                 VM_OBJECT_WLOCK(source);
1367                 if ((source->flags & OBJ_DEAD) != 0) {
1368                         VM_OBJECT_WUNLOCK(source);
1369                         VM_OBJECT_WUNLOCK(orig_object);
1370                         VM_OBJECT_WUNLOCK(new_object);
1371                         vm_object_deallocate(new_object);
1372                         VM_OBJECT_WLOCK(orig_object);
1373                         return;
1374                 }
1375                 LIST_INSERT_HEAD(&source->shadow_head,
1376                                   new_object, shadow_list);
1377                 source->shadow_count++;
1378                 vm_object_reference_locked(source);     /* for new_object */
1379                 vm_object_clear_flag(source, OBJ_ONEMAPPING);
1380                 VM_OBJECT_WUNLOCK(source);
1381                 new_object->backing_object_offset = 
1382                         orig_object->backing_object_offset + entry->offset;
1383                 new_object->backing_object = source;
1384         }
1385         if (orig_object->cred != NULL) {
1386                 new_object->cred = orig_object->cred;
1387                 crhold(orig_object->cred);
1388                 new_object->charge = ptoa(size);
1389                 KASSERT(orig_object->charge >= ptoa(size),
1390                     ("orig_object->charge < 0"));
1391                 orig_object->charge -= ptoa(size);
1392         }
1393 retry:
1394         m = vm_page_find_least(orig_object, offidxstart);
1395         for (; m != NULL && (idx = m->pindex - offidxstart) < size;
1396             m = m_next) {
1397                 m_next = TAILQ_NEXT(m, listq);
1398
1399                 /*
1400                  * We must wait for pending I/O to complete before we can
1401                  * rename the page.
1402                  *
1403                  * We do not have to VM_PROT_NONE the page as mappings should
1404                  * not be changed by this operation.
1405                  */
1406                 if (vm_page_busied(m)) {
1407                         VM_OBJECT_WUNLOCK(new_object);
1408                         vm_page_lock(m);
1409                         VM_OBJECT_WUNLOCK(orig_object);
1410                         vm_page_busy_sleep(m, "spltwt", false);
1411                         VM_OBJECT_WLOCK(orig_object);
1412                         VM_OBJECT_WLOCK(new_object);
1413                         goto retry;
1414                 }
1415
1416                 /* vm_page_rename() will dirty the page. */
1417                 if (vm_page_rename(m, new_object, idx)) {
1418                         VM_OBJECT_WUNLOCK(new_object);
1419                         VM_OBJECT_WUNLOCK(orig_object);
1420                         vm_radix_wait();
1421                         VM_OBJECT_WLOCK(orig_object);
1422                         VM_OBJECT_WLOCK(new_object);
1423                         goto retry;
1424                 }
1425 #if VM_NRESERVLEVEL > 0
1426                 /*
1427                  * If some of the reservation's allocated pages remain with
1428                  * the original object, then transferring the reservation to
1429                  * the new object is neither particularly beneficial nor
1430                  * particularly harmful as compared to leaving the reservation
1431                  * with the original object.  If, however, all of the
1432                  * reservation's allocated pages are transferred to the new
1433                  * object, then transferring the reservation is typically
1434                  * beneficial.  Determining which of these two cases applies
1435                  * would be more costly than unconditionally renaming the
1436                  * reservation.
1437                  */
1438                 vm_reserv_rename(m, new_object, orig_object, offidxstart);
1439 #endif
1440                 if (orig_object->type == OBJT_SWAP)
1441                         vm_page_xbusy(m);
1442         }
1443         if (orig_object->type == OBJT_SWAP) {
1444                 /*
1445                  * swap_pager_copy() can sleep, in which case the orig_object's
1446                  * and new_object's locks are released and reacquired. 
1447                  */
1448                 swap_pager_copy(orig_object, new_object, offidxstart, 0);
1449                 TAILQ_FOREACH(m, &new_object->memq, listq)
1450                         vm_page_xunbusy(m);
1451         }
1452         VM_OBJECT_WUNLOCK(orig_object);
1453         VM_OBJECT_WUNLOCK(new_object);
1454         entry->object.vm_object = new_object;
1455         entry->offset = 0LL;
1456         vm_object_deallocate(orig_object);
1457         VM_OBJECT_WLOCK(new_object);
1458 }
1459
1460 #define OBSC_COLLAPSE_NOWAIT    0x0002
1461 #define OBSC_COLLAPSE_WAIT      0x0004
1462
1463 static vm_page_t
1464 vm_object_collapse_scan_wait(vm_object_t object, vm_page_t p, vm_page_t next,
1465     int op)
1466 {
1467         vm_object_t backing_object;
1468
1469         VM_OBJECT_ASSERT_WLOCKED(object);
1470         backing_object = object->backing_object;
1471         VM_OBJECT_ASSERT_WLOCKED(backing_object);
1472
1473         KASSERT(p == NULL || vm_page_busied(p), ("unbusy page %p", p));
1474         KASSERT(p == NULL || p->object == object || p->object == backing_object,
1475             ("invalid ownership %p %p %p", p, object, backing_object));
1476         if ((op & OBSC_COLLAPSE_NOWAIT) != 0)
1477                 return (next);
1478         if (p != NULL)
1479                 vm_page_lock(p);
1480         VM_OBJECT_WUNLOCK(object);
1481         VM_OBJECT_WUNLOCK(backing_object);
1482         /* The page is only NULL when rename fails. */
1483         if (p == NULL)
1484                 vm_radix_wait();
1485         else
1486                 vm_page_busy_sleep(p, "vmocol", false);
1487         VM_OBJECT_WLOCK(object);
1488         VM_OBJECT_WLOCK(backing_object);
1489         return (TAILQ_FIRST(&backing_object->memq));
1490 }
1491
1492 static bool
1493 vm_object_scan_all_shadowed(vm_object_t object)
1494 {
1495         vm_object_t backing_object;
1496         vm_page_t p, pp;
1497         vm_pindex_t backing_offset_index, new_pindex, pi, ps;
1498
1499         VM_OBJECT_ASSERT_WLOCKED(object);
1500         VM_OBJECT_ASSERT_WLOCKED(object->backing_object);
1501
1502         backing_object = object->backing_object;
1503
1504         if (backing_object->type != OBJT_DEFAULT &&
1505             backing_object->type != OBJT_SWAP)
1506                 return (false);
1507
1508         pi = backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
1509         p = vm_page_find_least(backing_object, pi);
1510         ps = swap_pager_find_least(backing_object, pi);
1511
1512         /*
1513          * Only check pages inside the parent object's range and
1514          * inside the parent object's mapping of the backing object.
1515          */
1516         for (;; pi++) {
1517                 if (p != NULL && p->pindex < pi)
1518                         p = TAILQ_NEXT(p, listq);
1519                 if (ps < pi)
1520                         ps = swap_pager_find_least(backing_object, pi);
1521                 if (p == NULL && ps >= backing_object->size)
1522                         break;
1523                 else if (p == NULL)
1524                         pi = ps;
1525                 else
1526                         pi = MIN(p->pindex, ps);
1527
1528                 new_pindex = pi - backing_offset_index;
1529                 if (new_pindex >= object->size)
1530                         break;
1531
1532                 /*
1533                  * See if the parent has the page or if the parent's object
1534                  * pager has the page.  If the parent has the page but the page
1535                  * is not valid, the parent's object pager must have the page.
1536                  *
1537                  * If this fails, the parent does not completely shadow the
1538                  * object and we might as well give up now.
1539                  */
1540                 pp = vm_page_lookup(object, new_pindex);
1541                 if ((pp == NULL || pp->valid == 0) &&
1542                     !vm_pager_has_page(object, new_pindex, NULL, NULL))
1543                         return (false);
1544         }
1545         return (true);
1546 }
1547
1548 static bool
1549 vm_object_collapse_scan(vm_object_t object, int op)
1550 {
1551         vm_object_t backing_object;
1552         vm_page_t next, p, pp;
1553         vm_pindex_t backing_offset_index, new_pindex;
1554
1555         VM_OBJECT_ASSERT_WLOCKED(object);
1556         VM_OBJECT_ASSERT_WLOCKED(object->backing_object);
1557
1558         backing_object = object->backing_object;
1559         backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
1560
1561         /*
1562          * Initial conditions
1563          */
1564         if ((op & OBSC_COLLAPSE_WAIT) != 0)
1565                 vm_object_set_flag(backing_object, OBJ_DEAD);
1566
1567         /*
1568          * Our scan
1569          */
1570         for (p = TAILQ_FIRST(&backing_object->memq); p != NULL; p = next) {
1571                 next = TAILQ_NEXT(p, listq);
1572                 new_pindex = p->pindex - backing_offset_index;
1573
1574                 /*
1575                  * Check for busy page
1576                  */
1577                 if (vm_page_busied(p)) {
1578                         next = vm_object_collapse_scan_wait(object, p, next, op);
1579                         continue;
1580                 }
1581
1582                 KASSERT(p->object == backing_object,
1583                     ("vm_object_collapse_scan: object mismatch"));
1584
1585                 if (p->pindex < backing_offset_index ||
1586                     new_pindex >= object->size) {
1587                         if (backing_object->type == OBJT_SWAP)
1588                                 swap_pager_freespace(backing_object, p->pindex,
1589                                     1);
1590
1591                         /*
1592                          * Page is out of the parent object's range, we can
1593                          * simply destroy it.
1594                          */
1595                         vm_page_lock(p);
1596                         KASSERT(!pmap_page_is_mapped(p),
1597                             ("freeing mapped page %p", p));
1598                         if (vm_page_remove(p))
1599                                 vm_page_free(p);
1600                         vm_page_unlock(p);
1601                         continue;
1602                 }
1603
1604                 pp = vm_page_lookup(object, new_pindex);
1605                 if (pp != NULL && vm_page_busied(pp)) {
1606                         /*
1607                          * The page in the parent is busy and possibly not
1608                          * (yet) valid.  Until its state is finalized by the
1609                          * busy bit owner, we can't tell whether it shadows the
1610                          * original page.  Therefore, we must either skip it
1611                          * and the original (backing_object) page or wait for
1612                          * its state to be finalized.
1613                          *
1614                          * This is due to a race with vm_fault() where we must
1615                          * unbusy the original (backing_obj) page before we can
1616                          * (re)lock the parent.  Hence we can get here.
1617                          */
1618                         next = vm_object_collapse_scan_wait(object, pp, next,
1619                             op);
1620                         continue;
1621                 }
1622
1623                 KASSERT(pp == NULL || pp->valid != 0,
1624                     ("unbusy invalid page %p", pp));
1625
1626                 if (pp != NULL || vm_pager_has_page(object, new_pindex, NULL,
1627                         NULL)) {
1628                         /*
1629                          * The page already exists in the parent OR swap exists
1630                          * for this location in the parent.  Leave the parent's
1631                          * page alone.  Destroy the original page from the
1632                          * backing object.
1633                          */
1634                         if (backing_object->type == OBJT_SWAP)
1635                                 swap_pager_freespace(backing_object, p->pindex,
1636                                     1);
1637                         vm_page_lock(p);
1638                         KASSERT(!pmap_page_is_mapped(p),
1639                             ("freeing mapped page %p", p));
1640                         if (vm_page_remove(p))
1641                                 vm_page_free(p);
1642                         vm_page_unlock(p);
1643                         continue;
1644                 }
1645
1646                 /*
1647                  * Page does not exist in parent, rename the page from the
1648                  * backing object to the main object.
1649                  *
1650                  * If the page was mapped to a process, it can remain mapped
1651                  * through the rename.  vm_page_rename() will dirty the page.
1652                  */
1653                 if (vm_page_rename(p, object, new_pindex)) {
1654                         next = vm_object_collapse_scan_wait(object, NULL, next,
1655                             op);
1656                         continue;
1657                 }
1658
1659                 /* Use the old pindex to free the right page. */
1660                 if (backing_object->type == OBJT_SWAP)
1661                         swap_pager_freespace(backing_object,
1662                             new_pindex + backing_offset_index, 1);
1663
1664 #if VM_NRESERVLEVEL > 0
1665                 /*
1666                  * Rename the reservation.
1667                  */
1668                 vm_reserv_rename(p, object, backing_object,
1669                     backing_offset_index);
1670 #endif
1671         }
1672         return (true);
1673 }
1674
1675
1676 /*
1677  * this version of collapse allows the operation to occur earlier and
1678  * when paging_in_progress is true for an object...  This is not a complete
1679  * operation, but should plug 99.9% of the rest of the leaks.
1680  */
1681 static void
1682 vm_object_qcollapse(vm_object_t object)
1683 {
1684         vm_object_t backing_object = object->backing_object;
1685
1686         VM_OBJECT_ASSERT_WLOCKED(object);
1687         VM_OBJECT_ASSERT_WLOCKED(backing_object);
1688
1689         if (backing_object->ref_count != 1)
1690                 return;
1691
1692         vm_object_collapse_scan(object, OBSC_COLLAPSE_NOWAIT);
1693 }
1694
1695 /*
1696  *      vm_object_collapse:
1697  *
1698  *      Collapse an object with the object backing it.
1699  *      Pages in the backing object are moved into the
1700  *      parent, and the backing object is deallocated.
1701  */
1702 void
1703 vm_object_collapse(vm_object_t object)
1704 {
1705         vm_object_t backing_object, new_backing_object;
1706
1707         VM_OBJECT_ASSERT_WLOCKED(object);
1708
1709         while (TRUE) {
1710                 /*
1711                  * Verify that the conditions are right for collapse:
1712                  *
1713                  * The object exists and the backing object exists.
1714                  */
1715                 if ((backing_object = object->backing_object) == NULL)
1716                         break;
1717
1718                 /*
1719                  * we check the backing object first, because it is most likely
1720                  * not collapsable.
1721                  */
1722                 VM_OBJECT_WLOCK(backing_object);
1723                 if (backing_object->handle != NULL ||
1724                     (backing_object->type != OBJT_DEFAULT &&
1725                     backing_object->type != OBJT_SWAP) ||
1726                     (backing_object->flags & (OBJ_DEAD | OBJ_NOSPLIT)) != 0 ||
1727                     object->handle != NULL ||
1728                     (object->type != OBJT_DEFAULT &&
1729                      object->type != OBJT_SWAP) ||
1730                     (object->flags & OBJ_DEAD)) {
1731                         VM_OBJECT_WUNLOCK(backing_object);
1732                         break;
1733                 }
1734
1735                 if (object->paging_in_progress != 0 ||
1736                     backing_object->paging_in_progress != 0) {
1737                         vm_object_qcollapse(object);
1738                         VM_OBJECT_WUNLOCK(backing_object);
1739                         break;
1740                 }
1741
1742                 /*
1743                  * We know that we can either collapse the backing object (if
1744                  * the parent is the only reference to it) or (perhaps) have
1745                  * the parent bypass the object if the parent happens to shadow
1746                  * all the resident pages in the entire backing object.
1747                  *
1748                  * This is ignoring pager-backed pages such as swap pages.
1749                  * vm_object_collapse_scan fails the shadowing test in this
1750                  * case.
1751                  */
1752                 if (backing_object->ref_count == 1) {
1753                         vm_object_pip_add(object, 1);
1754                         vm_object_pip_add(backing_object, 1);
1755
1756                         /*
1757                          * If there is exactly one reference to the backing
1758                          * object, we can collapse it into the parent.
1759                          */
1760                         vm_object_collapse_scan(object, OBSC_COLLAPSE_WAIT);
1761
1762 #if VM_NRESERVLEVEL > 0
1763                         /*
1764                          * Break any reservations from backing_object.
1765                          */
1766                         if (__predict_false(!LIST_EMPTY(&backing_object->rvq)))
1767                                 vm_reserv_break_all(backing_object);
1768 #endif
1769
1770                         /*
1771                          * Move the pager from backing_object to object.
1772                          */
1773                         if (backing_object->type == OBJT_SWAP) {
1774                                 /*
1775                                  * swap_pager_copy() can sleep, in which case
1776                                  * the backing_object's and object's locks are
1777                                  * released and reacquired.
1778                                  * Since swap_pager_copy() is being asked to
1779                                  * destroy the source, it will change the
1780                                  * backing_object's type to OBJT_DEFAULT.
1781                                  */
1782                                 swap_pager_copy(
1783                                     backing_object,
1784                                     object,
1785                                     OFF_TO_IDX(object->backing_object_offset), TRUE);
1786                         }
1787                         /*
1788                          * Object now shadows whatever backing_object did.
1789                          * Note that the reference to 
1790                          * backing_object->backing_object moves from within 
1791                          * backing_object to within object.
1792                          */
1793                         LIST_REMOVE(object, shadow_list);
1794                         backing_object->shadow_count--;
1795                         if (backing_object->backing_object) {
1796                                 VM_OBJECT_WLOCK(backing_object->backing_object);
1797                                 LIST_REMOVE(backing_object, shadow_list);
1798                                 LIST_INSERT_HEAD(
1799                                     &backing_object->backing_object->shadow_head,
1800                                     object, shadow_list);
1801                                 /*
1802                                  * The shadow_count has not changed.
1803                                  */
1804                                 VM_OBJECT_WUNLOCK(backing_object->backing_object);
1805                         }
1806                         object->backing_object = backing_object->backing_object;
1807                         object->backing_object_offset +=
1808                             backing_object->backing_object_offset;
1809
1810                         /*
1811                          * Discard backing_object.
1812                          *
1813                          * Since the backing object has no pages, no pager left,
1814                          * and no object references within it, all that is
1815                          * necessary is to dispose of it.
1816                          */
1817                         KASSERT(backing_object->ref_count == 1, (
1818 "backing_object %p was somehow re-referenced during collapse!",
1819                             backing_object));
1820                         vm_object_pip_wakeup(backing_object);
1821                         backing_object->type = OBJT_DEAD;
1822                         backing_object->ref_count = 0;
1823                         VM_OBJECT_WUNLOCK(backing_object);
1824                         vm_object_destroy(backing_object);
1825
1826                         vm_object_pip_wakeup(object);
1827                         counter_u64_add(object_collapses, 1);
1828                 } else {
1829                         /*
1830                          * If we do not entirely shadow the backing object,
1831                          * there is nothing we can do so we give up.
1832                          */
1833                         if (object->resident_page_count != object->size &&
1834                             !vm_object_scan_all_shadowed(object)) {
1835                                 VM_OBJECT_WUNLOCK(backing_object);
1836                                 break;
1837                         }
1838
1839                         /*
1840                          * Make the parent shadow the next object in the
1841                          * chain.  Deallocating backing_object will not remove
1842                          * it, since its reference count is at least 2.
1843                          */
1844                         LIST_REMOVE(object, shadow_list);
1845                         backing_object->shadow_count--;
1846
1847                         new_backing_object = backing_object->backing_object;
1848                         if ((object->backing_object = new_backing_object) != NULL) {
1849                                 VM_OBJECT_WLOCK(new_backing_object);
1850                                 LIST_INSERT_HEAD(
1851                                     &new_backing_object->shadow_head,
1852                                     object,
1853                                     shadow_list
1854                                 );
1855                                 new_backing_object->shadow_count++;
1856                                 vm_object_reference_locked(new_backing_object);
1857                                 VM_OBJECT_WUNLOCK(new_backing_object);
1858                                 object->backing_object_offset +=
1859                                         backing_object->backing_object_offset;
1860                         }
1861
1862                         /*
1863                          * Drop the reference count on backing_object. Since
1864                          * its ref_count was at least 2, it will not vanish.
1865                          */
1866                         backing_object->ref_count--;
1867                         VM_OBJECT_WUNLOCK(backing_object);
1868                         counter_u64_add(object_bypasses, 1);
1869                 }
1870
1871                 /*
1872                  * Try again with this object's new backing object.
1873                  */
1874         }
1875 }
1876
1877 /*
1878  *      vm_object_page_remove:
1879  *
1880  *      For the given object, either frees or invalidates each of the
1881  *      specified pages.  In general, a page is freed.  However, if a page is
1882  *      wired for any reason other than the existence of a managed, wired
1883  *      mapping, then it may be invalidated but not removed from the object.
1884  *      Pages are specified by the given range ["start", "end") and the option
1885  *      OBJPR_CLEANONLY.  As a special case, if "end" is zero, then the range
1886  *      extends from "start" to the end of the object.  If the option
1887  *      OBJPR_CLEANONLY is specified, then only the non-dirty pages within the
1888  *      specified range are affected.  If the option OBJPR_NOTMAPPED is
1889  *      specified, then the pages within the specified range must have no
1890  *      mappings.  Otherwise, if this option is not specified, any mappings to
1891  *      the specified pages are removed before the pages are freed or
1892  *      invalidated.
1893  *
1894  *      In general, this operation should only be performed on objects that
1895  *      contain managed pages.  There are, however, two exceptions.  First, it
1896  *      is performed on the kernel and kmem objects by vm_map_entry_delete().
1897  *      Second, it is used by msync(..., MS_INVALIDATE) to invalidate device-
1898  *      backed pages.  In both of these cases, the option OBJPR_CLEANONLY must
1899  *      not be specified and the option OBJPR_NOTMAPPED must be specified.
1900  *
1901  *      The object must be locked.
1902  */
1903 void
1904 vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end,
1905     int options)
1906 {
1907         vm_page_t p, next;
1908         struct mtx *mtx;
1909
1910         VM_OBJECT_ASSERT_WLOCKED(object);
1911         KASSERT((object->flags & OBJ_UNMANAGED) == 0 ||
1912             (options & (OBJPR_CLEANONLY | OBJPR_NOTMAPPED)) == OBJPR_NOTMAPPED,
1913             ("vm_object_page_remove: illegal options for object %p", object));
1914         if (object->resident_page_count == 0)
1915                 return;
1916         vm_object_pip_add(object, 1);
1917 again:
1918         p = vm_page_find_least(object, start);
1919         mtx = NULL;
1920
1921         /*
1922          * Here, the variable "p" is either (1) the page with the least pindex
1923          * greater than or equal to the parameter "start" or (2) NULL. 
1924          */
1925         for (; p != NULL && (p->pindex < end || end == 0); p = next) {
1926                 next = TAILQ_NEXT(p, listq);
1927
1928                 /*
1929                  * If the page is wired for any reason besides the existence
1930                  * of managed, wired mappings, then it cannot be freed.  For
1931                  * example, fictitious pages, which represent device memory,
1932                  * are inherently wired and cannot be freed.  They can,
1933                  * however, be invalidated if the option OBJPR_CLEANONLY is
1934                  * not specified.
1935                  */
1936                 vm_page_change_lock(p, &mtx);
1937                 if (vm_page_xbusied(p)) {
1938                         VM_OBJECT_WUNLOCK(object);
1939                         vm_page_busy_sleep(p, "vmopax", true);
1940                         VM_OBJECT_WLOCK(object);
1941                         goto again;
1942                 }
1943                 if (vm_page_wired(p)) {
1944                         if ((options & OBJPR_NOTMAPPED) == 0 &&
1945                             object->ref_count != 0)
1946                                 pmap_remove_all(p);
1947                         if ((options & OBJPR_CLEANONLY) == 0) {
1948                                 p->valid = 0;
1949                                 vm_page_undirty(p);
1950                         }
1951                         continue;
1952                 }
1953                 if (vm_page_busied(p)) {
1954                         VM_OBJECT_WUNLOCK(object);
1955                         vm_page_busy_sleep(p, "vmopar", false);
1956                         VM_OBJECT_WLOCK(object);
1957                         goto again;
1958                 }
1959                 KASSERT((p->flags & PG_FICTITIOUS) == 0,
1960                     ("vm_object_page_remove: page %p is fictitious", p));
1961                 if ((options & OBJPR_CLEANONLY) != 0 && p->valid != 0) {
1962                         if ((options & OBJPR_NOTMAPPED) == 0 &&
1963                             object->ref_count != 0)
1964                                 pmap_remove_write(p);
1965                         if (p->dirty != 0)
1966                                 continue;
1967                 }
1968                 if ((options & OBJPR_NOTMAPPED) == 0 && object->ref_count != 0)
1969                         pmap_remove_all(p);
1970                 vm_page_free(p);
1971         }
1972         if (mtx != NULL)
1973                 mtx_unlock(mtx);
1974         vm_object_pip_wakeup(object);
1975 }
1976
1977 /*
1978  *      vm_object_page_noreuse:
1979  *
1980  *      For the given object, attempt to move the specified pages to
1981  *      the head of the inactive queue.  This bypasses regular LRU
1982  *      operation and allows the pages to be reused quickly under memory
1983  *      pressure.  If a page is wired for any reason, then it will not
1984  *      be queued.  Pages are specified by the range ["start", "end").
1985  *      As a special case, if "end" is zero, then the range extends from
1986  *      "start" to the end of the object.
1987  *
1988  *      This operation should only be performed on objects that
1989  *      contain non-fictitious, managed pages.
1990  *
1991  *      The object must be locked.
1992  */
1993 void
1994 vm_object_page_noreuse(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
1995 {
1996         struct mtx *mtx;
1997         vm_page_t p, next;
1998
1999         VM_OBJECT_ASSERT_LOCKED(object);
2000         KASSERT((object->flags & (OBJ_FICTITIOUS | OBJ_UNMANAGED)) == 0,
2001             ("vm_object_page_noreuse: illegal object %p", object));
2002         if (object->resident_page_count == 0)
2003                 return;
2004         p = vm_page_find_least(object, start);
2005
2006         /*
2007          * Here, the variable "p" is either (1) the page with the least pindex
2008          * greater than or equal to the parameter "start" or (2) NULL. 
2009          */
2010         mtx = NULL;
2011         for (; p != NULL && (p->pindex < end || end == 0); p = next) {
2012                 next = TAILQ_NEXT(p, listq);
2013                 vm_page_change_lock(p, &mtx);
2014                 vm_page_deactivate_noreuse(p);
2015         }
2016         if (mtx != NULL)
2017                 mtx_unlock(mtx);
2018 }
2019
2020 /*
2021  *      Populate the specified range of the object with valid pages.  Returns
2022  *      TRUE if the range is successfully populated and FALSE otherwise.
2023  *
2024  *      Note: This function should be optimized to pass a larger array of
2025  *      pages to vm_pager_get_pages() before it is applied to a non-
2026  *      OBJT_DEVICE object.
2027  *
2028  *      The object must be locked.
2029  */
2030 boolean_t
2031 vm_object_populate(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
2032 {
2033         vm_page_t m;
2034         vm_pindex_t pindex;
2035         int rv;
2036
2037         VM_OBJECT_ASSERT_WLOCKED(object);
2038         for (pindex = start; pindex < end; pindex++) {
2039                 m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL);
2040                 if (m->valid != VM_PAGE_BITS_ALL) {
2041                         rv = vm_pager_get_pages(object, &m, 1, NULL, NULL);
2042                         if (rv != VM_PAGER_OK) {
2043                                 vm_page_lock(m);
2044                                 vm_page_free(m);
2045                                 vm_page_unlock(m);
2046                                 break;
2047                         }
2048                 }
2049                 /*
2050                  * Keep "m" busy because a subsequent iteration may unlock
2051                  * the object.
2052                  */
2053         }
2054         if (pindex > start) {
2055                 m = vm_page_lookup(object, start);
2056                 while (m != NULL && m->pindex < pindex) {
2057                         vm_page_xunbusy(m);
2058                         m = TAILQ_NEXT(m, listq);
2059                 }
2060         }
2061         return (pindex == end);
2062 }
2063
2064 /*
2065  *      Routine:        vm_object_coalesce
2066  *      Function:       Coalesces two objects backing up adjoining
2067  *                      regions of memory into a single object.
2068  *
2069  *      returns TRUE if objects were combined.
2070  *
2071  *      NOTE:   Only works at the moment if the second object is NULL -
2072  *              if it's not, which object do we lock first?
2073  *
2074  *      Parameters:
2075  *              prev_object     First object to coalesce
2076  *              prev_offset     Offset into prev_object
2077  *              prev_size       Size of reference to prev_object
2078  *              next_size       Size of reference to the second object
2079  *              reserved        Indicator that extension region has
2080  *                              swap accounted for
2081  *
2082  *      Conditions:
2083  *      The object must *not* be locked.
2084  */
2085 boolean_t
2086 vm_object_coalesce(vm_object_t prev_object, vm_ooffset_t prev_offset,
2087     vm_size_t prev_size, vm_size_t next_size, boolean_t reserved)
2088 {
2089         vm_pindex_t next_pindex;
2090
2091         if (prev_object == NULL)
2092                 return (TRUE);
2093         VM_OBJECT_WLOCK(prev_object);
2094         if ((prev_object->type != OBJT_DEFAULT &&
2095             prev_object->type != OBJT_SWAP) ||
2096             (prev_object->flags & OBJ_TMPFS_NODE) != 0) {
2097                 VM_OBJECT_WUNLOCK(prev_object);
2098                 return (FALSE);
2099         }
2100
2101         /*
2102          * Try to collapse the object first
2103          */
2104         vm_object_collapse(prev_object);
2105
2106         /*
2107          * Can't coalesce if: . more than one reference . paged out . shadows
2108          * another object . has a copy elsewhere (any of which mean that the
2109          * pages not mapped to prev_entry may be in use anyway)
2110          */
2111         if (prev_object->backing_object != NULL) {
2112                 VM_OBJECT_WUNLOCK(prev_object);
2113                 return (FALSE);
2114         }
2115
2116         prev_size >>= PAGE_SHIFT;
2117         next_size >>= PAGE_SHIFT;
2118         next_pindex = OFF_TO_IDX(prev_offset) + prev_size;
2119
2120         if (prev_object->ref_count > 1 &&
2121             prev_object->size != next_pindex &&
2122             (prev_object->flags & OBJ_ONEMAPPING) == 0) {
2123                 VM_OBJECT_WUNLOCK(prev_object);
2124                 return (FALSE);
2125         }
2126
2127         /*
2128          * Account for the charge.
2129          */
2130         if (prev_object->cred != NULL) {
2131
2132                 /*
2133                  * If prev_object was charged, then this mapping,
2134                  * although not charged now, may become writable
2135                  * later. Non-NULL cred in the object would prevent
2136                  * swap reservation during enabling of the write
2137                  * access, so reserve swap now. Failed reservation
2138                  * cause allocation of the separate object for the map
2139                  * entry, and swap reservation for this entry is
2140                  * managed in appropriate time.
2141                  */
2142                 if (!reserved && !swap_reserve_by_cred(ptoa(next_size),
2143                     prev_object->cred)) {
2144                         VM_OBJECT_WUNLOCK(prev_object);
2145                         return (FALSE);
2146                 }
2147                 prev_object->charge += ptoa(next_size);
2148         }
2149
2150         /*
2151          * Remove any pages that may still be in the object from a previous
2152          * deallocation.
2153          */
2154         if (next_pindex < prev_object->size) {
2155                 vm_object_page_remove(prev_object, next_pindex, next_pindex +
2156                     next_size, 0);
2157                 if (prev_object->type == OBJT_SWAP)
2158                         swap_pager_freespace(prev_object,
2159                                              next_pindex, next_size);
2160 #if 0
2161                 if (prev_object->cred != NULL) {
2162                         KASSERT(prev_object->charge >=
2163                             ptoa(prev_object->size - next_pindex),
2164                             ("object %p overcharged 1 %jx %jx", prev_object,
2165                                 (uintmax_t)next_pindex, (uintmax_t)next_size));
2166                         prev_object->charge -= ptoa(prev_object->size -
2167                             next_pindex);
2168                 }
2169 #endif
2170         }
2171
2172         /*
2173          * Extend the object if necessary.
2174          */
2175         if (next_pindex + next_size > prev_object->size)
2176                 prev_object->size = next_pindex + next_size;
2177
2178         VM_OBJECT_WUNLOCK(prev_object);
2179         return (TRUE);
2180 }
2181
2182 void
2183 vm_object_set_writeable_dirty(vm_object_t object)
2184 {
2185
2186         VM_OBJECT_ASSERT_WLOCKED(object);
2187         if (object->type != OBJT_VNODE) {
2188                 if ((object->flags & OBJ_TMPFS_NODE) != 0) {
2189                         KASSERT(object->type == OBJT_SWAP, ("non-swap tmpfs"));
2190                         vm_object_set_flag(object, OBJ_TMPFS_DIRTY);
2191                 }
2192                 return;
2193         }
2194         object->generation++;
2195         if ((object->flags & OBJ_MIGHTBEDIRTY) != 0)
2196                 return;
2197         vm_object_set_flag(object, OBJ_MIGHTBEDIRTY);
2198 }
2199
2200 /*
2201  *      vm_object_unwire:
2202  *
2203  *      For each page offset within the specified range of the given object,
2204  *      find the highest-level page in the shadow chain and unwire it.  A page
2205  *      must exist at every page offset, and the highest-level page must be
2206  *      wired.
2207  */
2208 void
2209 vm_object_unwire(vm_object_t object, vm_ooffset_t offset, vm_size_t length,
2210     uint8_t queue)
2211 {
2212         vm_object_t tobject, t1object;
2213         vm_page_t m, tm;
2214         vm_pindex_t end_pindex, pindex, tpindex;
2215         int depth, locked_depth;
2216
2217         KASSERT((offset & PAGE_MASK) == 0,
2218             ("vm_object_unwire: offset is not page aligned"));
2219         KASSERT((length & PAGE_MASK) == 0,
2220             ("vm_object_unwire: length is not a multiple of PAGE_SIZE"));
2221         /* The wired count of a fictitious page never changes. */
2222         if ((object->flags & OBJ_FICTITIOUS) != 0)
2223                 return;
2224         pindex = OFF_TO_IDX(offset);
2225         end_pindex = pindex + atop(length);
2226 again:
2227         locked_depth = 1;
2228         VM_OBJECT_RLOCK(object);
2229         m = vm_page_find_least(object, pindex);
2230         while (pindex < end_pindex) {
2231                 if (m == NULL || pindex < m->pindex) {
2232                         /*
2233                          * The first object in the shadow chain doesn't
2234                          * contain a page at the current index.  Therefore,
2235                          * the page must exist in a backing object.
2236                          */
2237                         tobject = object;
2238                         tpindex = pindex;
2239                         depth = 0;
2240                         do {
2241                                 tpindex +=
2242                                     OFF_TO_IDX(tobject->backing_object_offset);
2243                                 tobject = tobject->backing_object;
2244                                 KASSERT(tobject != NULL,
2245                                     ("vm_object_unwire: missing page"));
2246                                 if ((tobject->flags & OBJ_FICTITIOUS) != 0)
2247                                         goto next_page;
2248                                 depth++;
2249                                 if (depth == locked_depth) {
2250                                         locked_depth++;
2251                                         VM_OBJECT_RLOCK(tobject);
2252                                 }
2253                         } while ((tm = vm_page_lookup(tobject, tpindex)) ==
2254                             NULL);
2255                 } else {
2256                         tm = m;
2257                         m = TAILQ_NEXT(m, listq);
2258                 }
2259                 vm_page_lock(tm);
2260                 if (vm_page_xbusied(tm)) {
2261                         for (tobject = object; locked_depth >= 1;
2262                             locked_depth--) {
2263                                 t1object = tobject->backing_object;
2264                                 VM_OBJECT_RUNLOCK(tobject);
2265                                 tobject = t1object;
2266                         }
2267                         vm_page_busy_sleep(tm, "unwbo", true);
2268                         goto again;
2269                 }
2270                 vm_page_unwire(tm, queue);
2271                 vm_page_unlock(tm);
2272 next_page:
2273                 pindex++;
2274         }
2275         /* Release the accumulated object locks. */
2276         for (tobject = object; locked_depth >= 1; locked_depth--) {
2277                 t1object = tobject->backing_object;
2278                 VM_OBJECT_RUNLOCK(tobject);
2279                 tobject = t1object;
2280         }
2281 }
2282
2283 /*
2284  * Return the vnode for the given object, or NULL if none exists.
2285  * For tmpfs objects, the function may return NULL if there is
2286  * no vnode allocated at the time of the call.
2287  */
2288 struct vnode *
2289 vm_object_vnode(vm_object_t object)
2290 {
2291         struct vnode *vp;
2292
2293         VM_OBJECT_ASSERT_LOCKED(object);
2294         if (object->type == OBJT_VNODE) {
2295                 vp = object->handle;
2296                 KASSERT(vp != NULL, ("%s: OBJT_VNODE has no vnode", __func__));
2297         } else if (object->type == OBJT_SWAP &&
2298             (object->flags & OBJ_TMPFS) != 0) {
2299                 vp = object->un_pager.swp.swp_tmpfs;
2300                 KASSERT(vp != NULL, ("%s: OBJT_TMPFS has no vnode", __func__));
2301         } else {
2302                 vp = NULL;
2303         }
2304         return (vp);
2305 }
2306
2307 /*
2308  * Return the kvme type of the given object.
2309  * If vpp is not NULL, set it to the object's vm_object_vnode() or NULL.
2310  */
2311 int
2312 vm_object_kvme_type(vm_object_t object, struct vnode **vpp)
2313 {
2314
2315         VM_OBJECT_ASSERT_LOCKED(object);
2316         if (vpp != NULL)
2317                 *vpp = vm_object_vnode(object);
2318         switch (object->type) {
2319         case OBJT_DEFAULT:
2320                 return (KVME_TYPE_DEFAULT);
2321         case OBJT_VNODE:
2322                 return (KVME_TYPE_VNODE);
2323         case OBJT_SWAP:
2324                 if ((object->flags & OBJ_TMPFS_NODE) != 0)
2325                         return (KVME_TYPE_VNODE);
2326                 return (KVME_TYPE_SWAP);
2327         case OBJT_DEVICE:
2328                 return (KVME_TYPE_DEVICE);
2329         case OBJT_PHYS:
2330                 return (KVME_TYPE_PHYS);
2331         case OBJT_DEAD:
2332                 return (KVME_TYPE_DEAD);
2333         case OBJT_SG:
2334                 return (KVME_TYPE_SG);
2335         case OBJT_MGTDEVICE:
2336                 return (KVME_TYPE_MGTDEVICE);
2337         default:
2338                 return (KVME_TYPE_UNKNOWN);
2339         }
2340 }
2341
2342 static int
2343 sysctl_vm_object_list(SYSCTL_HANDLER_ARGS)
2344 {
2345         struct kinfo_vmobject *kvo;
2346         char *fullpath, *freepath;
2347         struct vnode *vp;
2348         struct vattr va;
2349         vm_object_t obj;
2350         vm_page_t m;
2351         int count, error;
2352
2353         if (req->oldptr == NULL) {
2354                 /*
2355                  * If an old buffer has not been provided, generate an
2356                  * estimate of the space needed for a subsequent call.
2357                  */
2358                 mtx_lock(&vm_object_list_mtx);
2359                 count = 0;
2360                 TAILQ_FOREACH(obj, &vm_object_list, object_list) {
2361                         if (obj->type == OBJT_DEAD)
2362                                 continue;
2363                         count++;
2364                 }
2365                 mtx_unlock(&vm_object_list_mtx);
2366                 return (SYSCTL_OUT(req, NULL, sizeof(struct kinfo_vmobject) *
2367                     count * 11 / 10));
2368         }
2369
2370         kvo = malloc(sizeof(*kvo), M_TEMP, M_WAITOK);
2371         error = 0;
2372
2373         /*
2374          * VM objects are type stable and are never removed from the
2375          * list once added.  This allows us to safely read obj->object_list
2376          * after reacquiring the VM object lock.
2377          */
2378         mtx_lock(&vm_object_list_mtx);
2379         TAILQ_FOREACH(obj, &vm_object_list, object_list) {
2380                 if (obj->type == OBJT_DEAD)
2381                         continue;
2382                 VM_OBJECT_RLOCK(obj);
2383                 if (obj->type == OBJT_DEAD) {
2384                         VM_OBJECT_RUNLOCK(obj);
2385                         continue;
2386                 }
2387                 mtx_unlock(&vm_object_list_mtx);
2388                 kvo->kvo_size = ptoa(obj->size);
2389                 kvo->kvo_resident = obj->resident_page_count;
2390                 kvo->kvo_ref_count = obj->ref_count;
2391                 kvo->kvo_shadow_count = obj->shadow_count;
2392                 kvo->kvo_memattr = obj->memattr;
2393                 kvo->kvo_active = 0;
2394                 kvo->kvo_inactive = 0;
2395                 TAILQ_FOREACH(m, &obj->memq, listq) {
2396                         /*
2397                          * A page may belong to the object but be
2398                          * dequeued and set to PQ_NONE while the
2399                          * object lock is not held.  This makes the
2400                          * reads of m->queue below racy, and we do not
2401                          * count pages set to PQ_NONE.  However, this
2402                          * sysctl is only meant to give an
2403                          * approximation of the system anyway.
2404                          */
2405                         if (m->queue == PQ_ACTIVE)
2406                                 kvo->kvo_active++;
2407                         else if (m->queue == PQ_INACTIVE)
2408                                 kvo->kvo_inactive++;
2409                 }
2410
2411                 kvo->kvo_vn_fileid = 0;
2412                 kvo->kvo_vn_fsid = 0;
2413                 kvo->kvo_vn_fsid_freebsd11 = 0;
2414                 freepath = NULL;
2415                 fullpath = "";
2416                 kvo->kvo_type = vm_object_kvme_type(obj, &vp);
2417                 if (vp != NULL)
2418                         vref(vp);
2419                 VM_OBJECT_RUNLOCK(obj);
2420                 if (vp != NULL) {
2421                         vn_fullpath(curthread, vp, &fullpath, &freepath);
2422                         vn_lock(vp, LK_SHARED | LK_RETRY);
2423                         if (VOP_GETATTR(vp, &va, curthread->td_ucred) == 0) {
2424                                 kvo->kvo_vn_fileid = va.va_fileid;
2425                                 kvo->kvo_vn_fsid = va.va_fsid;
2426                                 kvo->kvo_vn_fsid_freebsd11 = va.va_fsid;
2427                                                                 /* truncate */
2428                         }
2429                         vput(vp);
2430                 }
2431
2432                 strlcpy(kvo->kvo_path, fullpath, sizeof(kvo->kvo_path));
2433                 if (freepath != NULL)
2434                         free(freepath, M_TEMP);
2435
2436                 /* Pack record size down */
2437                 kvo->kvo_structsize = offsetof(struct kinfo_vmobject, kvo_path)
2438                     + strlen(kvo->kvo_path) + 1;
2439                 kvo->kvo_structsize = roundup(kvo->kvo_structsize,
2440                     sizeof(uint64_t));
2441                 error = SYSCTL_OUT(req, kvo, kvo->kvo_structsize);
2442                 mtx_lock(&vm_object_list_mtx);
2443                 if (error)
2444                         break;
2445         }
2446         mtx_unlock(&vm_object_list_mtx);
2447         free(kvo, M_TEMP);
2448         return (error);
2449 }
2450 SYSCTL_PROC(_vm, OID_AUTO, objects, CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_SKIP |
2451     CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_object_list, "S,kinfo_vmobject",
2452     "List of VM objects");
2453
2454 #include "opt_ddb.h"
2455 #ifdef DDB
2456 #include <sys/kernel.h>
2457
2458 #include <sys/cons.h>
2459
2460 #include <ddb/ddb.h>
2461
2462 static int
2463 _vm_object_in_map(vm_map_t map, vm_object_t object, vm_map_entry_t entry)
2464 {
2465         vm_map_t tmpm;
2466         vm_map_entry_t tmpe;
2467         vm_object_t obj;
2468         int entcount;
2469
2470         if (map == 0)
2471                 return 0;
2472
2473         if (entry == 0) {
2474                 tmpe = map->header.next;
2475                 entcount = map->nentries;
2476                 while (entcount-- && (tmpe != &map->header)) {
2477                         if (_vm_object_in_map(map, object, tmpe)) {
2478                                 return 1;
2479                         }
2480                         tmpe = tmpe->next;
2481                 }
2482         } else if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) {
2483                 tmpm = entry->object.sub_map;
2484                 tmpe = tmpm->header.next;
2485                 entcount = tmpm->nentries;
2486                 while (entcount-- && tmpe != &tmpm->header) {
2487                         if (_vm_object_in_map(tmpm, object, tmpe)) {
2488                                 return 1;
2489                         }
2490                         tmpe = tmpe->next;
2491                 }
2492         } else if ((obj = entry->object.vm_object) != NULL) {
2493                 for (; obj; obj = obj->backing_object)
2494                         if (obj == object) {
2495                                 return 1;
2496                         }
2497         }
2498         return 0;
2499 }
2500
2501 static int
2502 vm_object_in_map(vm_object_t object)
2503 {
2504         struct proc *p;
2505
2506         /* sx_slock(&allproc_lock); */
2507         FOREACH_PROC_IN_SYSTEM(p) {
2508                 if (!p->p_vmspace /* || (p->p_flag & (P_SYSTEM|P_WEXIT)) */)
2509                         continue;
2510                 if (_vm_object_in_map(&p->p_vmspace->vm_map, object, 0)) {
2511                         /* sx_sunlock(&allproc_lock); */
2512                         return 1;
2513                 }
2514         }
2515         /* sx_sunlock(&allproc_lock); */
2516         if (_vm_object_in_map(kernel_map, object, 0))
2517                 return 1;
2518         return 0;
2519 }
2520
2521 DB_SHOW_COMMAND(vmochk, vm_object_check)
2522 {
2523         vm_object_t object;
2524
2525         /*
2526          * make sure that internal objs are in a map somewhere
2527          * and none have zero ref counts.
2528          */
2529         TAILQ_FOREACH(object, &vm_object_list, object_list) {
2530                 if (object->handle == NULL &&
2531                     (object->type == OBJT_DEFAULT || object->type == OBJT_SWAP)) {
2532                         if (object->ref_count == 0) {
2533                                 db_printf("vmochk: internal obj has zero ref count: %ld\n",
2534                                         (long)object->size);
2535                         }
2536                         if (!vm_object_in_map(object)) {
2537                                 db_printf(
2538                         "vmochk: internal obj is not in a map: "
2539                         "ref: %d, size: %lu: 0x%lx, backing_object: %p\n",
2540                                     object->ref_count, (u_long)object->size, 
2541                                     (u_long)object->size,
2542                                     (void *)object->backing_object);
2543                         }
2544                 }
2545         }
2546 }
2547
2548 /*
2549  *      vm_object_print:        [ debug ]
2550  */
2551 DB_SHOW_COMMAND(object, vm_object_print_static)
2552 {
2553         /* XXX convert args. */
2554         vm_object_t object = (vm_object_t)addr;
2555         boolean_t full = have_addr;
2556
2557         vm_page_t p;
2558
2559         /* XXX count is an (unused) arg.  Avoid shadowing it. */
2560 #define count   was_count
2561
2562         int count;
2563
2564         if (object == NULL)
2565                 return;
2566
2567         db_iprintf(
2568             "Object %p: type=%d, size=0x%jx, res=%d, ref=%d, flags=0x%x ruid %d charge %jx\n",
2569             object, (int)object->type, (uintmax_t)object->size,
2570             object->resident_page_count, object->ref_count, object->flags,
2571             object->cred ? object->cred->cr_ruid : -1, (uintmax_t)object->charge);
2572         db_iprintf(" sref=%d, backing_object(%d)=(%p)+0x%jx\n",
2573             object->shadow_count, 
2574             object->backing_object ? object->backing_object->ref_count : 0,
2575             object->backing_object, (uintmax_t)object->backing_object_offset);
2576
2577         if (!full)
2578                 return;
2579
2580         db_indent += 2;
2581         count = 0;
2582         TAILQ_FOREACH(p, &object->memq, listq) {
2583                 if (count == 0)
2584                         db_iprintf("memory:=");
2585                 else if (count == 6) {
2586                         db_printf("\n");
2587                         db_iprintf(" ...");
2588                         count = 0;
2589                 } else
2590                         db_printf(",");
2591                 count++;
2592
2593                 db_printf("(off=0x%jx,page=0x%jx)",
2594                     (uintmax_t)p->pindex, (uintmax_t)VM_PAGE_TO_PHYS(p));
2595         }
2596         if (count != 0)
2597                 db_printf("\n");
2598         db_indent -= 2;
2599 }
2600
2601 /* XXX. */
2602 #undef count
2603
2604 /* XXX need this non-static entry for calling from vm_map_print. */
2605 void
2606 vm_object_print(
2607         /* db_expr_t */ long addr,
2608         boolean_t have_addr,
2609         /* db_expr_t */ long count,
2610         char *modif)
2611 {
2612         vm_object_print_static(addr, have_addr, count, modif);
2613 }
2614
2615 DB_SHOW_COMMAND(vmopag, vm_object_print_pages)
2616 {
2617         vm_object_t object;
2618         vm_pindex_t fidx;
2619         vm_paddr_t pa;
2620         vm_page_t m, prev_m;
2621         int rcount, nl, c;
2622
2623         nl = 0;
2624         TAILQ_FOREACH(object, &vm_object_list, object_list) {
2625                 db_printf("new object: %p\n", (void *)object);
2626                 if (nl > 18) {
2627                         c = cngetc();
2628                         if (c != ' ')
2629                                 return;
2630                         nl = 0;
2631                 }
2632                 nl++;
2633                 rcount = 0;
2634                 fidx = 0;
2635                 pa = -1;
2636                 TAILQ_FOREACH(m, &object->memq, listq) {
2637                         if (m->pindex > 128)
2638                                 break;
2639                         if ((prev_m = TAILQ_PREV(m, pglist, listq)) != NULL &&
2640                             prev_m->pindex + 1 != m->pindex) {
2641                                 if (rcount) {
2642                                         db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
2643                                                 (long)fidx, rcount, (long)pa);
2644                                         if (nl > 18) {
2645                                                 c = cngetc();
2646                                                 if (c != ' ')
2647                                                         return;
2648                                                 nl = 0;
2649                                         }
2650                                         nl++;
2651                                         rcount = 0;
2652                                 }
2653                         }                               
2654                         if (rcount &&
2655                                 (VM_PAGE_TO_PHYS(m) == pa + rcount * PAGE_SIZE)) {
2656                                 ++rcount;
2657                                 continue;
2658                         }
2659                         if (rcount) {
2660                                 db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
2661                                         (long)fidx, rcount, (long)pa);
2662                                 if (nl > 18) {
2663                                         c = cngetc();
2664                                         if (c != ' ')
2665                                                 return;
2666                                         nl = 0;
2667                                 }
2668                                 nl++;
2669                         }
2670                         fidx = m->pindex;
2671                         pa = VM_PAGE_TO_PHYS(m);
2672                         rcount = 1;
2673                 }
2674                 if (rcount) {
2675                         db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
2676                                 (long)fidx, rcount, (long)pa);
2677                         if (nl > 18) {
2678                                 c = cngetc();
2679                                 if (c != ' ')
2680                                         return;
2681                                 nl = 0;
2682                         }
2683                         nl++;
2684                 }
2685         }
2686 }
2687 #endif /* DDB */