sys/vm/vm_object.c

   1 /*
   2  * Copyright (c) 1991, 1993
   3  *      The Regents of the University of California.  All rights reserved.
   4  *
   5  * This code is derived from software contributed to Berkeley by
   6  * The Mach Operating System project at Carnegie-Mellon University.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. All advertising materials mentioning features or use of this software
  17  *    must display the following acknowledgement:
  18  *      This product includes software developed by the University of
  19  *      California, Berkeley and its contributors.
  20  * 4. Neither the name of the University nor the names of its contributors
  21  *    may be used to endorse or promote products derived from this software
  22  *    without specific prior written permission.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  25  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  27  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  28  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  29  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  30  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  31  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  32  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  33  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  34  * SUCH DAMAGE.
  35  *
  36  *      from: @(#)vm_object.c   8.5 (Berkeley) 3/22/94
  37  *
  38  *
  39  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
  40  * All rights reserved.
  41  *
  42  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
  43  *
  44  * Permission to use, copy, modify and distribute this software and
  45  * its documentation is hereby granted, provided that both the copyright
  46  * notice and this permission notice appear in all copies of the
  47  * software, derivative works or modified versions, and any portions
  48  * thereof, and that both notices appear in supporting documentation.
  49  *
  50  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  51  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
  52  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  53  *
  54  * Carnegie Mellon requests users of this software to return to
  55  *
  56  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  57  *  School of Computer Science
  58  *  Carnegie Mellon University
  59  *  Pittsburgh PA 15213-3890
  60  *
  61  * any improvements or extensions that they make and grant Carnegie the
  62  * rights to redistribute these changes.
  63  *
  64  * $Id: vm_object.c,v 1.135 1998/11/05 14:28:26 dg Exp $
  65  */
  66
  67 /*
  68  *      Virtual memory object module.
  69  */
  70
  71 #include <sys/param.h>
  72 #include <sys/systm.h>
  73 #include <sys/proc.h>           /* for curproc, pageproc */
  74 #include <sys/vnode.h>
  75 #include <sys/vmmeter.h>
  76 #include <sys/mman.h>
  77 #include <sys/mount.h>
  78
  79 #include <vm/vm.h>
  80 #include <vm/vm_param.h>
  81 #include <vm/vm_prot.h>
  82 #include <vm/pmap.h>
  83 #include <vm/vm_map.h>
  84 #include <vm/vm_object.h>
  85 #include <vm/vm_page.h>
  86 #include <vm/vm_pageout.h>
  87 #include <vm/vm_pager.h>
  88 #include <vm/swap_pager.h>
  89 #include <vm/vm_kern.h>
  90 #include <vm/vm_extern.h>
  91 #include <vm/vm_zone.h>
  92
  93 static void     vm_object_qcollapse __P((vm_object_t object));
  94
  95 /*
  96  *      Virtual memory objects maintain the actual data
  97  *      associated with allocated virtual memory.  A given
  98  *      page of memory exists within exactly one object.
  99  *
 100  *      An object is only deallocated when all "references"
 101  *      are given up.  Only one "reference" to a given
 102  *      region of an object should be writeable.
 103  *
 104  *      Associated with each object is a list of all resident
 105  *      memory pages belonging to that object; this list is
 106  *      maintained by the "vm_page" module, and locked by the object's
 107  *      lock.
 108  *
 109  *      Each object also records a "pager" routine which is
 110  *      used to retrieve (and store) pages to the proper backing
 111  *      storage.  In addition, objects may be backed by other
 112  *      objects from which they were virtual-copied.
 113  *
 114  *      The only items within the object structure which are
 115  *      modified after time of creation are:
 116  *              reference count         locked by object's lock
 117  *              pager routine           locked by object's lock
 118  *
 119  */
 120
 121 struct object_q vm_object_list;
 122 #ifndef NULL_SIMPLELOCKS
 123 static struct simplelock vm_object_list_lock;
 124 #endif
 125 static long vm_object_count;            /* count of all objects */
 126 vm_object_t kernel_object;
 127 vm_object_t kmem_object;
 128 static struct vm_object kernel_object_store;
 129 static struct vm_object kmem_object_store;
 130 extern int vm_pageout_page_count;
 131
 132 static long object_collapses;
 133 static long object_bypasses;
 134 static int next_index;
 135 static vm_zone_t obj_zone;
 136 static struct vm_zone obj_zone_store;
 137 #define VM_OBJECTS_INIT 256
 138 static struct vm_object vm_objects_init[VM_OBJECTS_INIT];
 139 static int objidnumber;
 140
 141 void
 142 _vm_object_allocate(type, size, object)
 143         objtype_t type;
 144         vm_size_t size;
 145         register vm_object_t object;
 146 {
 147         int incr;
 148         TAILQ_INIT(&object->memq);
 149         TAILQ_INIT(&object->shadow_head);
 150
 151         object->type = type;
 152         object->size = size;
 153         object->ref_count = 1;
 154         object->flags = 0;
 155         object->id = ++objidnumber;
 156         if ((object->type == OBJT_DEFAULT) || (object->type == OBJT_SWAP))
 157                 vm_object_set_flag(object, OBJ_ONEMAPPING);
 158         object->behavior = OBJ_NORMAL;
 159         object->paging_in_progress = 0;
 160         object->resident_page_count = 0;
 161         object->cache_count = 0;
 162         object->wire_count = 0;
 163         object->shadow_count = 0;
 164         object->pg_color = next_index;
 165         if ( size > (PQ_L2_SIZE / 3 + PQ_PRIME1))
 166                 incr = PQ_L2_SIZE / 3 + PQ_PRIME1;
 167         else
 168                 incr = size;
 169         next_index = (next_index + incr) & PQ_L2_MASK;
 170         object->handle = NULL;
 171         object->paging_offset = (vm_ooffset_t) 0;
 172         object->backing_object = NULL;
 173         object->backing_object_offset = (vm_ooffset_t) 0;
 174         object->page_hint = NULL;
 175
 176         object->last_read = 0;
 177         object->generation++;
 178
 179         TAILQ_INSERT_TAIL(&vm_object_list, object, object_list);
 180         vm_object_count++;
 181 }
 182
 183 /*
 184  *      vm_object_init:
 185  *
 186  *      Initialize the VM objects module.
 187  */
 188 void
 189 vm_object_init()
 190 {
 191         TAILQ_INIT(&vm_object_list);
 192         simple_lock_init(&vm_object_list_lock);
 193         vm_object_count = 0;
 194
 195         kernel_object = &kernel_object_store;
 196         _vm_object_allocate(OBJT_DEFAULT, OFF_TO_IDX(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS),
 197             kernel_object);
 198
 199         kmem_object = &kmem_object_store;
 200         _vm_object_allocate(OBJT_DEFAULT, OFF_TO_IDX(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS),
 201             kmem_object);
 202
 203         obj_zone = &obj_zone_store;
 204         zbootinit(obj_zone, "VM OBJECT", sizeof (struct vm_object),
 205                 vm_objects_init, VM_OBJECTS_INIT);
 206 }
 207
 208 void
 209 vm_object_init2() {
 210         zinitna(obj_zone, NULL, NULL, 0, 0, 0, 1);
 211 }
 212
 213 /*
 214  *      vm_object_allocate:
 215  *
 216  *      Returns a new object with the given size.
 217  */
 218
 219 vm_object_t
 220 vm_object_allocate(type, size)
 221         objtype_t type;
 222         vm_size_t size;
 223 {
 224         register vm_object_t result;
 225         result = (vm_object_t) zalloc(obj_zone);
 226
 227         _vm_object_allocate(type, size, result);
 228
 229         return (result);
 230 }
 231
 232
 233 /*
 234  *      vm_object_reference:
 235  *
 236  *      Gets another reference to the given object.
 237  */
 238 void
 239 vm_object_reference(object)
 240         register vm_object_t object;
 241 {
 242         if (object == NULL)
 243                 return;
 244
 245 #if defined(DIAGNOSTIC)
 246         if (object->flags & OBJ_DEAD)
 247                 panic("vm_object_reference: attempting to reference dead obj");
 248 #endif
 249
 250         object->ref_count++;
 251         if (object->type == OBJT_VNODE) {
 252                 while (vget((struct vnode *) object->handle, LK_RETRY|LK_NOOBJ, curproc)) {
 253 #if !defined(MAX_PERF)
 254                         printf("vm_object_reference: delay in getting object\n");
 255 #endif
 256                 }
 257         }
 258 }
 259
 260 void
 261 vm_object_vndeallocate(object)
 262         vm_object_t object;
 263 {
 264         struct vnode *vp = (struct vnode *) object->handle;
 265 #if defined(DIAGNOSTIC)
 266         if (object->type != OBJT_VNODE)
 267                 panic("vm_object_vndeallocate: not a vnode object");
 268         if (vp == NULL)
 269                 panic("vm_object_vndeallocate: missing vp");
 270         if (object->ref_count == 0) {
 271                 vprint("vm_object_vndeallocate", vp);
 272                 panic("vm_object_vndeallocate: bad object reference count");
 273         }
 274 #endif
 275
 276         object->ref_count--;
 277         if (object->ref_count == 0) {
 278                 vp->v_flag &= ~VTEXT;
 279                 vm_object_clear_flag(object, OBJ_OPT);
 280         }
 281         vrele(vp);
 282 }
 283
 284 /*
 285  *      vm_object_deallocate:
 286  *
 287  *      Release a reference to the specified object,
 288  *      gained either through a vm_object_allocate
 289  *      or a vm_object_reference call.  When all references
 290  *      are gone, storage associated with this object
 291  *      may be relinquished.
 292  *
 293  *      No object may be locked.
 294  */
 295 void
 296 vm_object_deallocate(object)
 297         vm_object_t object;
 298 {
 299         vm_object_t temp;
 300
 301         while (object != NULL) {
 302
 303                 if (object->type == OBJT_VNODE) {
 304                         vm_object_vndeallocate(object);
 305                         return;
 306                 }
 307
 308                 if (object->ref_count == 0) {
 309                         panic("vm_object_deallocate: object deallocated too many times: %d", object->type);
 310                 } else if (object->ref_count > 2) {
 311                         object->ref_count--;
 312                         return;
 313                 }
 314
 315                 /*
 316                  * Here on ref_count of one or two, which are special cases for
 317                  * objects.
 318                  */
 319                 if ((object->ref_count == 2) && (object->shadow_count == 0)) {
 320                         vm_object_set_flag(object, OBJ_ONEMAPPING);
 321                         object->ref_count--;
 322                         return;
 323                 } else if ((object->ref_count == 2) && (object->shadow_count == 1)) {
 324                         object->ref_count--;
 325                         if ((object->handle == NULL) &&
 326                             (object->type == OBJT_DEFAULT ||
 327                              object->type == OBJT_SWAP)) {
 328                                 vm_object_t robject;
 329
 330                                 robject = TAILQ_FIRST(&object->shadow_head);
 331 #if defined(DIAGNOSTIC)
 332                                 if (robject == NULL)
 333                                         panic("vm_object_deallocate: ref_count: %d,"
 334                                                   " shadow_count: %d",
 335                                                   object->ref_count, object->shadow_count);
 336 #endif
 337                                 if ((robject->handle == NULL) &&
 338                                     (robject->type == OBJT_DEFAULT ||
 339                                      robject->type == OBJT_SWAP)) {
 340
 341                                         robject->ref_count++;
 342
 343                         retry:
 344                                         if (robject->paging_in_progress ||
 345                                                         object->paging_in_progress) {
 346                                                 vm_object_pip_sleep(robject, "objde1");
 347                                                 if (robject->paging_in_progress &&
 348                                                         robject->type == OBJT_SWAP) {
 349                                                         swap_pager_sync();
 350                                                         goto retry;
 351                                                 }
 352
 353                                                 vm_object_pip_sleep(object, "objde2");
 354                                                 if (object->paging_in_progress &&
 355                                                         object->type == OBJT_SWAP) {
 356                                                         swap_pager_sync();
 357                                                 }
 358                                                 goto retry;
 359                                         }
 360
 361                                         if( robject->ref_count == 1) {
 362                                                 robject->ref_count--;
 363                                                 object = robject;
 364                                                 goto doterm;
 365                                         }
 366
 367                                         object = robject;
 368                                         vm_object_collapse(object);
 369                                         continue;
 370                                 }
 371                         }
 372
 373                         return;
 374
 375                 } else {
 376                         object->ref_count--;
 377                         if (object->ref_count != 0)
 378                                 return;
 379                 }
 380
 381 doterm:
 382
 383                 temp = object->backing_object;
 384                 if (temp) {
 385                         TAILQ_REMOVE(&temp->shadow_head, object, shadow_list);
 386                         temp->shadow_count--;
 387                         if (temp->ref_count == 0)
 388                                 vm_object_clear_flag(temp, OBJ_OPT);
 389                         temp->generation++;
 390                         object->backing_object = NULL;
 391                 }
 392                 vm_object_terminate(object);
 393                 /* unlocks and deallocates object */
 394                 object = temp;
 395         }
 396 }
 397
 398 /*
 399  *      vm_object_terminate actually destroys the specified object, freeing
 400  *      up all previously used resources.
 401  *
 402  *      The object must be locked.
 403  */
 404 void
 405 vm_object_terminate(object)
 406         register vm_object_t object;
 407 {
 408         register vm_page_t p;
 409         int s;
 410
 411         /*
 412          * Make sure no one uses us.
 413          */
 414         vm_object_set_flag(object, OBJ_DEAD);
 415
 416         /*
 417          * wait for the pageout daemon to be done with the object
 418          */
 419         vm_object_pip_wait(object, "objtrm");
 420
 421 #if defined(DIAGNOSTIC)
 422         if (object->paging_in_progress != 0)
 423                 panic("vm_object_terminate: pageout in progress");
 424 #endif
 425
 426         /*
 427          * Clean and free the pages, as appropriate. All references to the
 428          * object are gone, so we don't need to lock it.
 429          */
 430         if (object->type == OBJT_VNODE) {
 431                 struct vnode *vp;
 432
 433                 /*
 434                  * Freeze optimized copies.
 435                  */
 436                 vm_freeze_copyopts(object, 0, object->size);
 437
 438                 /*
 439                  * Clean pages and flush buffers.
 440                  */
 441                 vm_object_page_clean(object, 0, 0, OBJPC_SYNC);
 442
 443                 vp = (struct vnode *) object->handle;
 444                 vinvalbuf(vp, V_SAVE, NOCRED, NULL, 0, 0);
 445         }
 446
 447         if (object->ref_count != 0)
 448                 panic("vm_object_terminate: object with references, ref_count=%d", object->ref_count);
 449
 450         /*
 451          * Now free any remaining pages. For internal objects, this also
 452          * removes them from paging queues. Don't free wired pages, just
 453          * remove them from the object.
 454          */
 455         s = splvm();
 456         while ((p = TAILQ_FIRST(&object->memq)) != NULL) {
 457 #if !defined(MAX_PERF)
 458                 if (p->busy || (p->flags & PG_BUSY))
 459                         printf("vm_object_terminate: freeing busy page\n");
 460 #endif
 461                 if (p->wire_count == 0) {
 462                         vm_page_busy(p);
 463                         vm_page_free(p);
 464                         cnt.v_pfree++;
 465                 } else {
 466                         vm_page_busy(p);
 467                         vm_page_remove(p);
 468                 }
 469         }
 470         splx(s);
 471
 472         /*
 473          * Let the pager know object is dead.
 474          */
 475         vm_pager_deallocate(object);
 476
 477         /*
 478          * Remove the object from the global object list.
 479          */
 480         simple_lock(&vm_object_list_lock);
 481         TAILQ_REMOVE(&vm_object_list, object, object_list);
 482         simple_unlock(&vm_object_list_lock);
 483
 484         wakeup(object);
 485
 486         /*
 487          * Free the space for the object.
 488          */
 489         zfree(obj_zone, object);
 490 }
 491
 492 /*
 493  *      vm_object_page_clean
 494  *
 495  *      Clean all dirty pages in the specified range of object.
 496  *      Leaves page on whatever queue it is currently on.
 497  *
 498  *      Odd semantics: if start == end, we clean everything.
 499  *
 500  *      The object must be locked.
 501  */
 502
 503 void
 504 vm_object_page_clean(object, start, end, flags)
 505         vm_object_t object;
 506         vm_pindex_t start;
 507         vm_pindex_t end;
 508         int flags;
 509 {
 510         register vm_page_t p, np, tp;
 511         register vm_offset_t tstart, tend;
 512         vm_pindex_t pi;
 513         int s;
 514         struct vnode *vp;
 515         int runlen;
 516         int maxf;
 517         int chkb;
 518         int maxb;
 519         int i;
 520         int pagerflags;
 521         vm_page_t maf[vm_pageout_page_count];
 522         vm_page_t mab[vm_pageout_page_count];
 523         vm_page_t ma[vm_pageout_page_count];
 524         int curgeneration;
 525
 526         if (object->type != OBJT_VNODE ||
 527                 (object->flags & OBJ_MIGHTBEDIRTY) == 0)
 528                 return;
 529
 530         pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) ? VM_PAGER_PUT_SYNC : 0;
 531         pagerflags |= (flags & OBJPC_INVAL) ? VM_PAGER_PUT_INVAL : 0;
 532
 533         vp = object->handle;
 534
 535         vm_object_set_flag(object, OBJ_CLEANING);
 536
 537         tstart = start;
 538         if (end == 0) {
 539                 tend = object->size;
 540         } else {
 541                 tend = end;
 542         }
 543
 544         for(p = TAILQ_FIRST(&object->memq); p; p = TAILQ_NEXT(p, listq)) {
 545                 vm_page_flag_set(p, PG_CLEANCHK);
 546                 vm_page_protect(p, VM_PROT_READ);
 547         }
 548
 549         if ((tstart == 0) && (tend == object->size)) {
 550                 vm_object_clear_flag(object, OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY);
 551         }
 552
 553 rescan:
 554         curgeneration = object->generation;
 555
 556         for(p = TAILQ_FIRST(&object->memq); p; p = np) {
 557                 np = TAILQ_NEXT(p, listq);
 558
 559                 pi = p->pindex;
 560                 if (((p->flags & PG_CLEANCHK) == 0) ||
 561                         (pi < tstart) || (pi >= tend) ||
 562                         (p->valid == 0) ||
 563                         ((p->queue - p->pc) == PQ_CACHE)) {
 564                         vm_page_flag_clear(p, PG_CLEANCHK);
 565                         continue;
 566                 }
 567
 568                 vm_page_test_dirty(p);
 569                 if ((p->dirty & p->valid) == 0) {
 570                         vm_page_flag_clear(p, PG_CLEANCHK);
 571                         continue;
 572                 }
 573
 574                 s = splvm();
 575                 while ((p->flags & PG_BUSY) || p->busy) {
 576                         vm_page_flag_set(p, PG_WANTED | PG_REFERENCED);
 577                         tsleep(p, PVM, "vpcwai", 0);
 578                         if (object->generation != curgeneration) {
 579                                 splx(s);
 580                                 goto rescan;
 581                         }
 582                 }
 583
 584                 maxf = 0;
 585                 for(i=1;i<vm_pageout_page_count;i++) {
 586                         if (tp = vm_page_lookup(object, pi + i)) {
 587                                 if ((tp->flags & PG_BUSY) ||
 588                                         (tp->flags & PG_CLEANCHK) == 0 ||
 589                                         (tp->busy != 0))
 590                                         break;
 591                                 if((tp->queue - tp->pc) == PQ_CACHE) {
 592                                         vm_page_flag_clear(tp, PG_CLEANCHK);
 593                                         break;
 594                                 }
 595                                 vm_page_test_dirty(tp);
 596                                 if ((tp->dirty & tp->valid) == 0) {
 597                                         vm_page_flag_clear(tp, PG_CLEANCHK);
 598                                         break;
 599                                 }
 600                                 maf[ i - 1 ] = tp;
 601                                 maxf++;
 602                                 continue;
 603                         }
 604                         break;
 605                 }
 606
 607                 maxb = 0;
 608                 chkb = vm_pageout_page_count -  maxf;
 609                 if (chkb) {
 610                         for(i = 1; i < chkb;i++) {
 611                                 if (tp = vm_page_lookup(object, pi - i)) {
 612                                         if ((tp->flags & PG_BUSY) ||
 613                                                 (tp->flags & PG_CLEANCHK) == 0 ||
 614                                                 (tp->busy != 0))
 615                                                 break;
 616                                         if((tp->queue - tp->pc) == PQ_CACHE) {
 617                                                 vm_page_flag_clear(tp, PG_CLEANCHK);
 618                                                 break;
 619                                         }
 620                                         vm_page_test_dirty(tp);
 621                                         if ((tp->dirty & tp->valid) == 0) {
 622                                                 vm_page_flag_clear(tp, PG_CLEANCHK);
 623                                                 break;
 624                                         }
 625                                         mab[ i - 1 ] = tp;
 626                                         maxb++;
 627                                         continue;
 628                                 }
 629                                 break;
 630                         }
 631                 }
 632
 633                 for(i=0;i<maxb;i++) {
 634                         int index = (maxb - i) - 1;
 635                         ma[index] = mab[i];
 636                         vm_page_flag_clear(ma[index], PG_CLEANCHK);
 637                 }
 638                 vm_page_flag_clear(p, PG_CLEANCHK);
 639                 ma[maxb] = p;
 640                 for(i=0;i<maxf;i++) {
 641                         int index = (maxb + i) + 1;
 642                         ma[index] = maf[i];
 643                         vm_page_flag_clear(ma[index], PG_CLEANCHK);
 644                 }
 645                 runlen = maxb + maxf + 1;
 646
 647                 splx(s);
 648                 vm_pageout_flush(ma, runlen, pagerflags);
 649                 for (i = 0; i<runlen; i++) {
 650                         if (ma[i]->valid & ma[i]->dirty) {
 651                                 vm_page_protect(ma[i], VM_PROT_READ);
 652                                 vm_page_flag_set(ma[i], PG_CLEANCHK);
 653                         }
 654                 }
 655                 if (object->generation != curgeneration)
 656                         goto rescan;
 657         }
 658
 659         VOP_FSYNC(vp, NULL, (pagerflags & VM_PAGER_PUT_SYNC)?MNT_WAIT:0, curproc);
 660
 661         vm_object_clear_flag(object, OBJ_CLEANING);
 662         return;
 663 }
 664
 665 #ifdef not_used
 666 /* XXX I cannot tell if this should be an exported symbol */
 667 /*
 668  *      vm_object_deactivate_pages
 669  *
 670  *      Deactivate all pages in the specified object.  (Keep its pages
 671  *      in memory even though it is no longer referenced.)
 672  *
 673  *      The object must be locked.
 674  */
 675 static void
 676 vm_object_deactivate_pages(object)
 677         register vm_object_t object;
 678 {
 679         register vm_page_t p, next;
 680
 681         for (p = TAILQ_FIRST(&object->memq); p != NULL; p = next) {
 682                 next = TAILQ_NEXT(p, listq);
 683                 vm_page_deactivate(p);
 684         }
 685 }
 686 #endif
 687
 688 /*
 689  *      vm_object_pmap_copy:
 690  *
 691  *      Makes all physical pages in the specified
 692  *      object range copy-on-write.  No writeable
 693  *      references to these pages should remain.
 694  *
 695  *      The object must *not* be locked.
 696  */
 697 void
 698 vm_object_pmap_copy(object, start, end)
 699         register vm_object_t object;
 700         register vm_pindex_t start;
 701         register vm_pindex_t end;
 702 {
 703         register vm_page_t p;
 704
 705         if (object == NULL || (object->flags & OBJ_WRITEABLE) == 0)
 706                 return;
 707
 708         for (p = TAILQ_FIRST(&object->memq);
 709                 p != NULL;
 710                 p = TAILQ_NEXT(p, listq)) {
 711                 vm_page_protect(p, VM_PROT_READ);
 712         }
 713
 714         vm_object_clear_flag(object, OBJ_WRITEABLE);
 715 }
 716
 717 /*
 718  * Same as vm_object_pmap_copy_1, except range checking really
 719  * works, and is meant for small sections of an object.
 720  */
 721 void
 722 vm_object_pmap_copy_1(object, start, end)
 723         register vm_object_t object;
 724         register vm_pindex_t start;
 725         register vm_pindex_t end;
 726 {
 727         vm_pindex_t idx;
 728         register vm_page_t p;
 729
 730         if (object == NULL || (object->flags & OBJ_WRITEABLE) == 0)
 731                 return;
 732
 733         for (idx = start; idx < end; idx++) {
 734                 p = vm_page_lookup(object, idx);
 735                 if (p == NULL)
 736                         continue;
 737                 vm_page_protect(p, VM_PROT_READ);
 738         }
 739 }
 740
 741 /*
 742  *      vm_object_pmap_remove:
 743  *
 744  *      Removes all physical pages in the specified
 745  *      object range from all physical maps.
 746  *
 747  *      The object must *not* be locked.
 748  */
 749 void
 750 vm_object_pmap_remove(object, start, end)
 751         register vm_object_t object;
 752         register vm_pindex_t start;
 753         register vm_pindex_t end;
 754 {
 755         register vm_page_t p;
 756         if (object == NULL)
 757                 return;
 758         for (p = TAILQ_FIRST(&object->memq);
 759                 p != NULL;
 760                 p = TAILQ_NEXT(p, listq)) {
 761                 if (p->pindex >= start && p->pindex < end)
 762                         vm_page_protect(p, VM_PROT_NONE);
 763         }
 764         if ((start == 0) && (object->size == end))
 765                 vm_object_clear_flag(object, OBJ_WRITEABLE);
 766 }
 767
 768 /*
 769  *      vm_object_madvise:
 770  *
 771  *      Implements the madvise function at the object/page level.
 772  */
 773 void
 774 vm_object_madvise(object, pindex, count, advise)
 775         vm_object_t object;
 776         vm_pindex_t pindex;
 777         int count;
 778         int advise;
 779 {
 780         vm_pindex_t end, tpindex;
 781         vm_object_t tobject;
 782         vm_page_t m;
 783
 784         if (object == NULL)
 785                 return;
 786
 787         end = pindex + count;
 788
 789         for (; pindex < end; pindex += 1) {
 790
 791 relookup:
 792                 tobject = object;
 793                 tpindex = pindex;
 794 shadowlookup:
 795                 m = vm_page_lookup(tobject, tpindex);
 796                 if (m == NULL) {
 797                         if (tobject->type != OBJT_DEFAULT) {
 798                                 continue;
 799                         }
 800
 801                         tobject = tobject->backing_object;
 802                         if ((tobject == NULL) || (tobject->ref_count != 1)) {
 803                                 continue;
 804                         }
 805                         tpindex += OFF_TO_IDX(tobject->backing_object_offset);
 806                         goto shadowlookup;
 807                 }
 808
 809                 /*
 810                  * If the page is busy or not in a normal active state,
 811                  * we skip it.  Things can break if we mess with pages
 812                  * in any of the below states.
 813                  */
 814                 if (m->hold_count || m->wire_count ||
 815                         m->valid != VM_PAGE_BITS_ALL) {
 816                         continue;
 817                 }
 818
 819                 if (vm_page_sleep(m, "madvpo", &m->busy))
 820                         goto relookup;
 821
 822                 if (advise == MADV_WILLNEED) {
 823                         vm_page_activate(m);
 824                 } else if (advise == MADV_DONTNEED) {
 825                         vm_page_deactivate(m);
 826                 } else if (advise == MADV_FREE) {
 827                         pmap_clear_modify(VM_PAGE_TO_PHYS(m));
 828                         m->dirty = 0;
 829                         /*
 830                          * Force a demand zero if attempt to read from swap.
 831                          * We currently don't handle vnode files correctly,
 832                          * and will reread stale contents unnecessarily.
 833                          */
 834                         if (object->type == OBJT_SWAP)
 835                                 swap_pager_dmzspace(tobject, m->pindex, 1);
 836                 }
 837         }
 838 }
 839
 840 /*
 841  *      vm_object_shadow:
 842  *
 843  *      Create a new object which is backed by the
 844  *      specified existing object range.  The source
 845  *      object reference is deallocated.
 846  *
 847  *      The new object and offset into that object
 848  *      are returned in the source parameters.
 849  */
 850
 851 void
 852 vm_object_shadow(object, offset, length)
 853         vm_object_t *object;    /* IN/OUT */
 854         vm_ooffset_t *offset;   /* IN/OUT */
 855         vm_size_t length;
 856 {
 857         register vm_object_t source;
 858         register vm_object_t result;
 859
 860         source = *object;
 861
 862         /*
 863          * Allocate a new object with the given length
 864          */
 865
 866         if ((result = vm_object_allocate(OBJT_DEFAULT, length)) == NULL)
 867                 panic("vm_object_shadow: no object for shadowing");
 868
 869         /*
 870          * The new object shadows the source object, adding a reference to it.
 871          * Our caller changes his reference to point to the new object,
 872          * removing a reference to the source object.  Net result: no change
 873          * of reference count.
 874          */
 875         result->backing_object = source;
 876         if (source) {
 877                 TAILQ_INSERT_TAIL(&source->shadow_head, result, shadow_list);
 878                 vm_object_clear_flag(source, OBJ_ONEMAPPING);
 879                 source->shadow_count++;
 880                 source->generation++;
 881         }
 882
 883         /*
 884          * Store the offset into the source object, and fix up the offset into
 885          * the new object.
 886          */
 887
 888         result->backing_object_offset = *offset;
 889
 890         /*
 891          * Return the new things
 892          */
 893
 894         *offset = 0;
 895         *object = result;
 896 }
 897
 898
 899 /*
 900  * this version of collapse allows the operation to occur earlier and
 901  * when paging_in_progress is true for an object...  This is not a complete
 902  * operation, but should plug 99.9% of the rest of the leaks.
 903  */
 904 static void
 905 vm_object_qcollapse(object)
 906         register vm_object_t object;
 907 {
 908         register vm_object_t backing_object;
 909         register vm_pindex_t backing_offset_index, paging_offset_index;
 910         vm_pindex_t backing_object_paging_offset_index;
 911         vm_pindex_t new_pindex;
 912         register vm_page_t p, pp;
 913         register vm_size_t size;
 914
 915         backing_object = object->backing_object;
 916         if (backing_object->ref_count != 1)
 917                 return;
 918
 919         backing_object->ref_count += 2;
 920
 921         backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
 922         backing_object_paging_offset_index = OFF_TO_IDX(backing_object->paging_offset);
 923         paging_offset_index = OFF_TO_IDX(object->paging_offset);
 924         size = object->size;
 925         p = TAILQ_FIRST(&backing_object->memq);
 926         while (p) {
 927                 vm_page_t next;
 928
 929                 next = TAILQ_NEXT(p, listq);
 930                 if ((p->flags & (PG_BUSY | PG_FICTITIOUS)) ||
 931                     !p->valid || p->hold_count || p->wire_count || p->busy) {
 932                         p = next;
 933                         continue;
 934                 }
 935                 vm_page_busy(p);
 936
 937                 new_pindex = p->pindex - backing_offset_index;
 938                 if (p->pindex < backing_offset_index ||
 939                     new_pindex >= size) {
 940                         if (backing_object->type == OBJT_SWAP)
 941                                 swap_pager_freespace(backing_object,
 942                                     backing_object_paging_offset_index+p->pindex,
 943                                     1);
 944                         vm_page_protect(p, VM_PROT_NONE);
 945                         vm_page_free(p);
 946                 } else {
 947                         pp = vm_page_lookup(object, new_pindex);
 948                         if (pp != NULL ||
 949                                 (object->type == OBJT_SWAP && vm_pager_has_page(object,
 950                                     paging_offset_index + new_pindex, NULL, NULL))) {
 951                                 if (backing_object->type == OBJT_SWAP)
 952                                         swap_pager_freespace(backing_object,
 953                                             backing_object_paging_offset_index + p->pindex, 1);
 954                                 vm_page_protect(p, VM_PROT_NONE);
 955                                 vm_page_free(p);
 956                         } else {
 957                                 if (backing_object->type == OBJT_SWAP)
 958                                         swap_pager_freespace(backing_object,
 959                                             backing_object_paging_offset_index + p->pindex, 1);
 960
 961                                 if ((p->queue - p->pc) == PQ_CACHE)
 962                                         vm_page_deactivate(p);
 963                                 else
 964                                         vm_page_protect(p, VM_PROT_NONE);
 965
 966                                 vm_page_rename(p, object, new_pindex);
 967                                 p->dirty = VM_PAGE_BITS_ALL;
 968                         }
 969                 }
 970                 p = next;
 971         }
 972         backing_object->ref_count -= 2;
 973 }
 974
 975 /*
 976  *      vm_object_collapse:
 977  *
 978  *      Collapse an object with the object backing it.
 979  *      Pages in the backing object are moved into the
 980  *      parent, and the backing object is deallocated.
 981  */
 982 void
 983 vm_object_collapse(object)
 984         vm_object_t object;
 985
 986 {
 987         vm_object_t backing_object;
 988         vm_ooffset_t backing_offset;
 989         vm_size_t size;
 990         vm_pindex_t new_pindex, backing_offset_index;
 991         vm_page_t p, pp;
 992
 993         while (TRUE) {
 994                 /*
 995                  * Verify that the conditions are right for collapse:
 996                  *
 997                  * The object exists and no pages in it are currently being paged
 998                  * out.
 999                  */
1000                 if (object == NULL)
1001                         return;
1002
1003                 /*
1004                  * Make sure there is a backing object.
1005                  */
1006                 if ((backing_object = object->backing_object) == NULL)
1007                         return;
1008
1009                 /*
1010                  * we check the backing object first, because it is most likely
1011                  * not collapsable.
1012                  */
1013                 if (backing_object->handle != NULL ||
1014                     (backing_object->type != OBJT_DEFAULT &&
1015                      backing_object->type != OBJT_SWAP) ||
1016                     (backing_object->flags & OBJ_DEAD) ||
1017                     object->handle != NULL ||
1018                     (object->type != OBJT_DEFAULT &&
1019                      object->type != OBJT_SWAP) ||
1020                     (object->flags & OBJ_DEAD)) {
1021                         return;
1022                 }
1023
1024                 if (object->paging_in_progress != 0 ||
1025                     backing_object->paging_in_progress != 0) {
1026                         vm_object_qcollapse(object);
1027                         return;
1028                 }
1029
1030                 /*
1031                  * We know that we can either collapse the backing object (if
1032                  * the parent is the only reference to it) or (perhaps) remove
1033                  * the parent's reference to it.
1034                  */
1035
1036                 backing_offset = object->backing_object_offset;
1037                 backing_offset_index = OFF_TO_IDX(backing_offset);
1038                 size = object->size;
1039
1040                 /*
1041                  * If there is exactly one reference to the backing object, we
1042                  * can collapse it into the parent.
1043                  */
1044
1045                 if (backing_object->ref_count == 1) {
1046
1047                         vm_object_set_flag(backing_object, OBJ_DEAD);
1048                         /*
1049                          * We can collapse the backing object.
1050                          *
1051                          * Move all in-memory pages from backing_object to the
1052                          * parent.  Pages that have been paged out will be
1053                          * overwritten by any of the parent's pages that
1054                          * shadow them.
1055                          */
1056
1057                         while ((p = TAILQ_FIRST(&backing_object->memq)) != 0) {
1058
1059                                 new_pindex = p->pindex - backing_offset_index;
1060                                 vm_page_busy(p);
1061
1062                                 /*
1063                                  * If the parent has a page here, or if this
1064                                  * page falls outside the parent, dispose of
1065                                  * it.
1066                                  *
1067                                  * Otherwise, move it as planned.
1068                                  */
1069
1070                                 if (p->pindex < backing_offset_index ||
1071                                     new_pindex >= size) {
1072                                         vm_page_protect(p, VM_PROT_NONE);
1073                                         vm_page_free(p);
1074                                 } else {
1075                                         pp = vm_page_lookup(object, new_pindex);
1076                                         if (pp != NULL || (object->type == OBJT_SWAP && vm_pager_has_page(object,
1077                                             OFF_TO_IDX(object->paging_offset) + new_pindex, NULL, NULL))) {
1078                                                 vm_page_protect(p, VM_PROT_NONE);
1079                                                 vm_page_free(p);
1080                                         } else {
1081                                                 if ((p->queue - p->pc) == PQ_CACHE)
1082                                                         vm_page_deactivate(p);
1083                                                 else
1084                                                         vm_page_protect(p, VM_PROT_NONE);
1085                                                 vm_page_rename(p, object, new_pindex);
1086                                                 p->dirty = VM_PAGE_BITS_ALL;
1087                                         }
1088                                 }
1089                         }
1090
1091                         /*
1092                          * Move the pager from backing_object to object.
1093                          */
1094
1095                         if (backing_object->type == OBJT_SWAP) {
1096                                 vm_object_pip_add(backing_object, 1);
1097                                 if (object->type == OBJT_SWAP) {
1098                                         vm_object_pip_add(object, 1);
1099                                         /*
1100                                          * copy shadow object pages into ours
1101                                          * and destroy unneeded pages in
1102                                          * shadow object.
1103                                          */
1104                                         swap_pager_copy(
1105                                             backing_object,
1106                                             OFF_TO_IDX(backing_object->paging_offset),
1107                                             object,
1108                                             OFF_TO_IDX(object->paging_offset),
1109                                             OFF_TO_IDX(object->backing_object_offset), TRUE);
1110                                         vm_object_pip_wakeup(object);
1111                                 } else {
1112                                         vm_object_pip_add(object, 1);
1113                                         /*
1114                                          * move the shadow backing_object's pager data to
1115                                          * "object" and convert "object" type to OBJT_SWAP.
1116                                          */
1117                                         object->type = OBJT_SWAP;
1118                                         object->un_pager.swp.swp_nblocks =
1119                                             backing_object->un_pager.swp.swp_nblocks;
1120                                         object->un_pager.swp.swp_allocsize =
1121                                             backing_object->un_pager.swp.swp_allocsize;
1122                                         object->un_pager.swp.swp_blocks =
1123                                             backing_object->un_pager.swp.swp_blocks;
1124                                         object->un_pager.swp.swp_poip =         /* XXX */
1125                                             backing_object->un_pager.swp.swp_poip;
1126                                         object->paging_offset = backing_object->paging_offset + backing_offset;
1127                                         TAILQ_INSERT_TAIL(&swap_pager_un_object_list, object, pager_object_list);
1128
1129                                         /*
1130                                          * Convert backing object from OBJT_SWAP to
1131                                          * OBJT_DEFAULT. XXX - only the TAILQ_REMOVE is
1132                                          * actually necessary.
1133                                          */
1134                                         backing_object->type = OBJT_DEFAULT;
1135                                         TAILQ_REMOVE(&swap_pager_un_object_list, backing_object, pager_object_list);
1136                                         /*
1137                                          * free unnecessary blocks
1138                                          */
1139                                         swap_pager_freespace(object, 0,
1140                                                 OFF_TO_IDX(object->paging_offset));
1141                                         vm_object_pip_wakeup(object);
1142                                 }
1143
1144                                 vm_object_pip_wakeup(backing_object);
1145                         }
1146                         /*
1147                          * Object now shadows whatever backing_object did.
1148                          * Note that the reference to backing_object->backing_object
1149                          * moves from within backing_object to within object.
1150                          */
1151
1152                         TAILQ_REMOVE(&object->backing_object->shadow_head, object,
1153                             shadow_list);
1154                         object->backing_object->shadow_count--;
1155                         object->backing_object->generation++;
1156                         if (backing_object->backing_object) {
1157                                 TAILQ_REMOVE(&backing_object->backing_object->shadow_head,
1158                                     backing_object, shadow_list);
1159                                 backing_object->backing_object->shadow_count--;
1160                                 backing_object->backing_object->generation++;
1161                         }
1162                         object->backing_object = backing_object->backing_object;
1163                         if (object->backing_object) {
1164                                 TAILQ_INSERT_TAIL(&object->backing_object->shadow_head,
1165                                     object, shadow_list);
1166                                 object->backing_object->shadow_count++;
1167                                 object->backing_object->generation++;
1168                         }
1169
1170                         object->backing_object_offset += backing_object->backing_object_offset;
1171                         /*
1172                          * Discard backing_object.
1173                          *
1174                          * Since the backing object has no pages, no pager left,
1175                          * and no object references within it, all that is
1176                          * necessary is to dispose of it.
1177                          */
1178
1179                         TAILQ_REMOVE(&vm_object_list, backing_object,
1180                             object_list);
1181                         vm_object_count--;
1182
1183                         zfree(obj_zone, backing_object);
1184
1185                         object_collapses++;
1186                 } else {
1187                         vm_object_t new_backing_object;
1188                         /*
1189                          * If all of the pages in the backing object are
1190                          * shadowed by the parent object, the parent object no
1191                          * longer has to shadow the backing object; it can
1192                          * shadow the next one in the chain.
1193                          *
1194                          * The backing object must not be paged out - we'd have
1195                          * to check all of the paged-out pages, as well.
1196                          */
1197
1198                         if (backing_object->type != OBJT_DEFAULT) {
1199                                 return;
1200                         }
1201                         /*
1202                          * Should have a check for a 'small' number of pages
1203                          * here.
1204                          */
1205
1206                         for (p = TAILQ_FIRST(&backing_object->memq); p;
1207                                         p = TAILQ_NEXT(p, listq)) {
1208
1209                                 new_pindex = p->pindex - backing_offset_index;
1210                                 vm_page_busy(p);
1211
1212                                 /*
1213                                  * If the parent has a page here, or if this
1214                                  * page falls outside the parent, keep going.
1215                                  *
1216                                  * Otherwise, the backing_object must be left in
1217                                  * the chain.
1218                                  */
1219
1220                                 if (p->pindex >= backing_offset_index &&
1221                                         new_pindex <= size) {
1222
1223                                         pp = vm_page_lookup(object, new_pindex);
1224
1225                                         if ((pp == NULL) || (pp->flags & PG_BUSY) || pp->busy) {
1226                                                 vm_page_wakeup(p);
1227                                                 return;
1228                                         }
1229
1230                                         vm_page_busy(pp);
1231                                         if ((pp->valid == 0) &&
1232                                             !vm_pager_has_page(object, OFF_TO_IDX(object->paging_offset) + new_pindex, NULL, NULL)) {
1233                                                 /*
1234                                                  * Page still needed. Can't go any
1235                                                  * further.
1236                                                  */
1237                                                 vm_page_wakeup(pp);
1238                                                 vm_page_wakeup(p);
1239                                                 return;
1240                                         }
1241                                         vm_page_wakeup(pp);
1242                                 }
1243                                 vm_page_wakeup(p);
1244                         }
1245
1246                         /*
1247                          * Make the parent shadow the next object in the
1248                          * chain.  Deallocating backing_object will not remove
1249                          * it, since its reference count is at least 2.
1250                          */
1251
1252                         TAILQ_REMOVE(&backing_object->shadow_head,
1253                             object, shadow_list);
1254                         backing_object->shadow_count--;
1255                         backing_object->generation++;
1256
1257                         new_backing_object = backing_object->backing_object;
1258                         if (object->backing_object = new_backing_object) {
1259                                 vm_object_reference(new_backing_object);
1260                                 TAILQ_INSERT_TAIL(&new_backing_object->shadow_head,
1261                                     object, shadow_list);
1262                                 new_backing_object->shadow_count++;
1263                                 new_backing_object->generation++;
1264                                 object->backing_object_offset +=
1265                                         backing_object->backing_object_offset;
1266                         }
1267
1268                         /*
1269                          * Drop the reference count on backing_object. Since
1270                          * its ref_count was at least 2, it will not vanish;
1271                          * so we don't need to call vm_object_deallocate, but
1272                          * we do anyway.
1273                          */
1274                         vm_object_deallocate(backing_object);
1275                         object_bypasses++;
1276                 }
1277
1278                 /*
1279                  * Try again with this object's new backing object.
1280                  */
1281         }
1282 }
1283
1284 /*
1285  *      vm_object_page_remove: [internal]
1286  *
1287  *      Removes all physical pages in the specified
1288  *      object range from the object's list of pages.
1289  *
1290  *      The object must be locked.
1291  */
1292 void
1293 vm_object_page_remove(object, start, end, clean_only)
1294         register vm_object_t object;
1295         register vm_pindex_t start;
1296         register vm_pindex_t end;
1297         boolean_t clean_only;
1298 {
1299         register vm_page_t p, next;
1300         unsigned int size;
1301         int all;
1302
1303         if (object == NULL)
1304                 return;
1305
1306         all = ((end == 0) && (start == 0));
1307
1308         vm_object_pip_add(object, 1);
1309 again:
1310         size = end - start;
1311         if (all || size > 4 || size >= object->size / 4) {
1312                 for (p = TAILQ_FIRST(&object->memq); p != NULL; p = next) {
1313                         next = TAILQ_NEXT(p, listq);
1314                         if (all || ((start <= p->pindex) && (p->pindex < end))) {
1315                                 if (p->wire_count != 0) {
1316                                         vm_page_protect(p, VM_PROT_NONE);
1317                                         if (!clean_only)
1318                                                 p->valid = 0;
1319                                         continue;
1320                                 }
1321
1322                                 /*
1323                                  * The busy flags are only cleared at
1324                                  * interrupt -- minimize the spl transitions
1325                                  */
1326
1327                                 if (vm_page_sleep(p, "vmopar", &p->busy))
1328                                         goto again;
1329
1330                                 if (clean_only && p->valid) {
1331                                         vm_page_test_dirty(p);
1332                                         if (p->valid & p->dirty)
1333                                                 continue;
1334                                 }
1335
1336                                 vm_page_busy(p);
1337                                 vm_page_protect(p, VM_PROT_NONE);
1338                                 vm_page_free(p);
1339                         }
1340                 }
1341         } else {
1342                 while (size > 0) {
1343                         if ((p = vm_page_lookup(object, start)) != 0) {
1344
1345                                 if (p->wire_count != 0) {
1346                                         vm_page_protect(p, VM_PROT_NONE);
1347                                         if (!clean_only)
1348                                                 p->valid = 0;
1349                                         start += 1;
1350                                         size -= 1;
1351                                         continue;
1352                                 }
1353
1354                                 /*
1355                                  * The busy flags are only cleared at
1356                                  * interrupt -- minimize the spl transitions
1357                                  */
1358                                 if (vm_page_sleep(p, "vmopar", &p->busy))
1359                                         goto again;
1360
1361                                 if (clean_only && p->valid) {
1362                                         vm_page_test_dirty(p);
1363                                         if (p->valid & p->dirty) {
1364                                                 start += 1;
1365                                                 size -= 1;
1366                                                 continue;
1367                                         }
1368                                 }
1369
1370                                 vm_page_busy(p);
1371                                 vm_page_protect(p, VM_PROT_NONE);
1372                                 vm_page_free(p);
1373                         }
1374                         start += 1;
1375                         size -= 1;
1376                 }
1377         }
1378         vm_object_pip_wakeup(object);
1379 }
1380
1381 /*
1382  *      Routine:        vm_object_coalesce
1383  *      Function:       Coalesces two objects backing up adjoining
1384  *                      regions of memory into a single object.
1385  *
1386  *      returns TRUE if objects were combined.
1387  *
1388  *      NOTE:   Only works at the moment if the second object is NULL -
1389  *              if it's not, which object do we lock first?
1390  *
1391  *      Parameters:
1392  *              prev_object     First object to coalesce
1393  *              prev_offset     Offset into prev_object
1394  *              next_object     Second object into coalesce
1395  *              next_offset     Offset into next_object
1396  *
1397  *              prev_size       Size of reference to prev_object
1398  *              next_size       Size of reference to next_object
1399  *
1400  *      Conditions:
1401  *      The object must *not* be locked.
1402  */
1403 boolean_t
1404 vm_object_coalesce(prev_object, prev_pindex, prev_size, next_size)
1405         register vm_object_t prev_object;
1406         vm_pindex_t prev_pindex;
1407         vm_size_t prev_size, next_size;
1408 {
1409         vm_size_t newsize;
1410
1411         if (prev_object == NULL) {
1412                 return (TRUE);
1413         }
1414
1415         if (prev_object->type != OBJT_DEFAULT) {
1416                 return (FALSE);
1417         }
1418
1419         /*
1420          * Try to collapse the object first
1421          */
1422         vm_object_collapse(prev_object);
1423
1424         /*
1425          * Can't coalesce if: . more than one reference . paged out . shadows
1426          * another object . has a copy elsewhere (any of which mean that the
1427          * pages not mapped to prev_entry may be in use anyway)
1428          */
1429
1430         if (prev_object->backing_object != NULL) {
1431                 return (FALSE);
1432         }
1433
1434         prev_size >>= PAGE_SHIFT;
1435         next_size >>= PAGE_SHIFT;
1436
1437         if ((prev_object->ref_count > 1) &&
1438             (prev_object->size != prev_pindex + prev_size)) {
1439                 return (FALSE);
1440         }
1441
1442         /*
1443          * Remove any pages that may still be in the object from a previous
1444          * deallocation.
1445          */
1446
1447         vm_object_page_remove(prev_object,
1448             prev_pindex + prev_size,
1449             prev_pindex + prev_size + next_size, FALSE);
1450
1451         /*
1452          * Extend the object if necessary.
1453          */
1454         newsize = prev_pindex + prev_size + next_size;
1455         if (newsize > prev_object->size)
1456                 prev_object->size = newsize;
1457
1458         return (TRUE);
1459 }
1460
1461 #include "opt_ddb.h"
1462 #ifdef DDB
1463 #include <sys/kernel.h>
1464
1465 #include <machine/cons.h>
1466
1467 #include <ddb/ddb.h>
1468
1469 static int      _vm_object_in_map __P((vm_map_t map, vm_object_t object,
1470                                        vm_map_entry_t entry));
1471 static int      vm_object_in_map __P((vm_object_t object));
1472
1473 static int
1474 _vm_object_in_map(map, object, entry)
1475         vm_map_t map;
1476         vm_object_t object;
1477         vm_map_entry_t entry;
1478 {
1479         vm_map_t tmpm;
1480         vm_map_entry_t tmpe;
1481         vm_object_t obj;
1482         int entcount;
1483
1484         if (map == 0)
1485                 return 0;
1486
1487         if (entry == 0) {
1488                 tmpe = map->header.next;
1489                 entcount = map->nentries;
1490                 while (entcount-- && (tmpe != &map->header)) {
1491                         if( _vm_object_in_map(map, object, tmpe)) {
1492                                 return 1;
1493                         }
1494                         tmpe = tmpe->next;
1495                 }
1496         } else if (entry->eflags & (MAP_ENTRY_IS_A_MAP|MAP_ENTRY_IS_SUB_MAP)) {
1497                 tmpm = entry->object.share_map;
1498                 tmpe = tmpm->header.next;
1499                 entcount = tmpm->nentries;
1500                 while (entcount-- && tmpe != &tmpm->header) {
1501                         if( _vm_object_in_map(tmpm, object, tmpe)) {
1502                                 return 1;
1503                         }
1504                         tmpe = tmpe->next;
1505                 }
1506         } else if (obj = entry->object.vm_object) {
1507                 for(; obj; obj=obj->backing_object)
1508                         if( obj == object) {
1509                                 return 1;
1510                         }
1511         }
1512         return 0;
1513 }
1514
1515 static int
1516 vm_object_in_map( object)
1517         vm_object_t object;
1518 {
1519         struct proc *p;
1520         for (p = allproc.lh_first; p != 0; p = p->p_list.le_next) {
1521                 if( !p->p_vmspace /* || (p->p_flag & (P_SYSTEM|P_WEXIT)) */)
1522                         continue;
1523                 if( _vm_object_in_map(&p->p_vmspace->vm_map, object, 0))
1524                         return 1;
1525         }
1526         if( _vm_object_in_map( kernel_map, object, 0))
1527                 return 1;
1528         if( _vm_object_in_map( kmem_map, object, 0))
1529                 return 1;
1530         if( _vm_object_in_map( pager_map, object, 0))
1531                 return 1;
1532         if( _vm_object_in_map( buffer_map, object, 0))
1533                 return 1;
1534         if( _vm_object_in_map( io_map, object, 0))
1535                 return 1;
1536         if( _vm_object_in_map( phys_map, object, 0))
1537                 return 1;
1538         if( _vm_object_in_map( mb_map, object, 0))
1539                 return 1;
1540         if( _vm_object_in_map( u_map, object, 0))
1541                 return 1;
1542         return 0;
1543 }
1544
1545 DB_SHOW_COMMAND(vmochk, vm_object_check)
1546 {
1547         vm_object_t object;
1548
1549         /*
1550          * make sure that internal objs are in a map somewhere
1551          * and none have zero ref counts.
1552          */
1553         for (object = TAILQ_FIRST(&vm_object_list);
1554                         object != NULL;
1555                         object = TAILQ_NEXT(object, object_list)) {
1556                 if (object->handle == NULL &&
1557                     (object->type == OBJT_DEFAULT || object->type == OBJT_SWAP)) {
1558                         if (object->ref_count == 0) {
1559                                 db_printf("vmochk: internal obj has zero ref count: %d\n",
1560                                         object->size);
1561                         }
1562                         if (!vm_object_in_map(object)) {
1563                                 db_printf(
1564                         "vmochk: internal obj is not in a map: "
1565                         "ref: %d, size: %lu: 0x%lx, backing_object: %p\n",
1566                                     object->ref_count, (u_long)object->size,
1567                                     (u_long)object->size,
1568                                     (void *)object->backing_object);
1569                         }
1570                 }
1571         }
1572 }
1573
1574 /*
1575  *      vm_object_print:        [ debug ]
1576  */
1577 DB_SHOW_COMMAND(object, vm_object_print_static)
1578 {
1579         /* XXX convert args. */
1580         vm_object_t object = (vm_object_t)addr;
1581         boolean_t full = have_addr;
1582
1583         register vm_page_t p;
1584
1585         /* XXX count is an (unused) arg.  Avoid shadowing it. */
1586 #define count   was_count
1587
1588         register int count;
1589
1590         if (object == NULL)
1591                 return;
1592
1593         db_iprintf(
1594             "Object %p: type=%d, size=0x%lx, res=%d, ref=%d, flags=0x%x\n",
1595             object, (int)object->type, (u_long)object->size,
1596             object->resident_page_count, object->ref_count, object->flags);
1597         /*
1598          * XXX no %qd in kernel.  Truncate object->paging_offset and
1599          * object->backing_object_offset.
1600          */
1601         db_iprintf(" sref=%d, offset=0x%lx, backing_object(%d)=(%p)+0x%lx\n",
1602             object->shadow_count, (long)object->paging_offset,
1603             object->backing_object ? object->backing_object->ref_count : 0,
1604             object->backing_object, (long)object->backing_object_offset);
1605
1606         if (!full)
1607                 return;
1608
1609         db_indent += 2;
1610         count = 0;
1611         for (p = TAILQ_FIRST(&object->memq); p != NULL; p = TAILQ_NEXT(p, listq)) {
1612                 if (count == 0)
1613                         db_iprintf("memory:=");
1614                 else if (count == 6) {
1615                         db_printf("\n");
1616                         db_iprintf(" ...");
1617                         count = 0;
1618                 } else
1619                         db_printf(",");
1620                 count++;
1621
1622                 db_printf("(off=0x%lx,page=0x%lx)",
1623                     (u_long) p->pindex, (u_long) VM_PAGE_TO_PHYS(p));
1624         }
1625         if (count != 0)
1626                 db_printf("\n");
1627         db_indent -= 2;
1628 }
1629
1630 /* XXX. */
1631 #undef count
1632
1633 /* XXX need this non-static entry for calling from vm_map_print. */
1634 void
1635 vm_object_print(addr, have_addr, count, modif)
1636         /* db_expr_t */ long addr;
1637         boolean_t have_addr;
1638         /* db_expr_t */ long count;
1639         char *modif;
1640 {
1641         vm_object_print_static(addr, have_addr, count, modif);
1642 }
1643
1644 DB_SHOW_COMMAND(vmopag, vm_object_print_pages)
1645 {
1646         vm_object_t object;
1647         int nl = 0;
1648         int c;
1649         for (object = TAILQ_FIRST(&vm_object_list);
1650                         object != NULL;
1651                         object = TAILQ_NEXT(object, object_list)) {
1652                 vm_pindex_t idx, fidx;
1653                 vm_pindex_t osize;
1654                 vm_offset_t pa = -1, padiff;
1655                 int rcount;
1656                 vm_page_t m;
1657
1658                 db_printf("new object: %p\n", (void *)object);
1659                 if ( nl > 18) {
1660                         c = cngetc();
1661                         if (c != ' ')
1662                                 return;
1663                         nl = 0;
1664                 }
1665                 nl++;
1666                 rcount = 0;
1667                 fidx = 0;
1668                 osize = object->size;
1669                 if (osize > 128)
1670                         osize = 128;
1671                 for(idx=0;idx<osize;idx++) {
1672                         m = vm_page_lookup(object, idx);
1673                         if (m == NULL) {
1674                                 if (rcount) {
1675                                         db_printf(" index(%d)run(%d)pa(0x%x)\n",
1676                                                 fidx, rcount, pa);
1677                                         if ( nl > 18) {
1678                                                 c = cngetc();
1679                                                 if (c != ' ')
1680                                                         return;
1681                                                 nl = 0;
1682                                         }
1683                                         nl++;
1684                                         rcount = 0;
1685                                 }
1686                                 continue;
1687                         }
1688
1689
1690                         if (rcount &&
1691                                 (VM_PAGE_TO_PHYS(m) == pa + rcount * PAGE_SIZE)) {
1692                                 ++rcount;
1693                                 continue;
1694                         }
1695                         if (rcount) {
1696                                 padiff = pa + rcount * PAGE_SIZE - VM_PAGE_TO_PHYS(m);
1697                                 padiff >>= PAGE_SHIFT;
1698                                 padiff &= PQ_L2_MASK;
1699                                 if (padiff == 0) {
1700                                         pa = VM_PAGE_TO_PHYS(m) - rcount * PAGE_SIZE;
1701                                         ++rcount;
1702                                         continue;
1703                                 }
1704                                 db_printf(" index(%d)run(%d)pa(0x%x)", fidx, rcount, pa);
1705                                 db_printf("pd(%d)\n", padiff);
1706                                 if ( nl > 18) {
1707                                         c = cngetc();
1708                                         if (c != ' ')
1709                                                 return;
1710                                         nl = 0;
1711                                 }
1712                                 nl++;
1713                         }
1714                         fidx = idx;
1715                         pa = VM_PAGE_TO_PHYS(m);
1716                         rcount = 1;
1717                 }
1718                 if (rcount) {
1719                         db_printf(" index(%d)run(%d)pa(0x%x)\n", fidx, rcount, pa);
1720                         if ( nl > 18) {
1721                                 c = cngetc();
1722                                 if (c != ' ')
1723                                         return;
1724                                 nl = 0;
1725                         }
1726                         nl++;
1727                 }
1728         }
1729 }
1730 #endif /* DDB */