sys/vm/vnode_pager.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-4-Clause
   3  *
   4  * Copyright (c) 1990 University of Utah.
   5  * Copyright (c) 1991 The Regents of the University of California.
   6  * All rights reserved.
   7  * Copyright (c) 1993, 1994 John S. Dyson
   8  * Copyright (c) 1995, David Greenman
   9  *
  10  * This code is derived from software contributed to Berkeley by
  11  * the Systems Programming Group of the University of Utah Computer
  12  * Science Department.
  13  *
  14  * Redistribution and use in source and binary forms, with or without
  15  * modification, are permitted provided that the following conditions
  16  * are met:
  17  * 1. Redistributions of source code must retain the above copyright
  18  *    notice, this list of conditions and the following disclaimer.
  19  * 2. Redistributions in binary form must reproduce the above copyright
  20  *    notice, this list of conditions and the following disclaimer in the
  21  *    documentation and/or other materials provided with the distribution.
  22  * 3. All advertising materials mentioning features or use of this software
  23  *    must display the following acknowledgement:
  24  *      This product includes software developed by the University of
  25  *      California, Berkeley and its contributors.
  26  * 4. Neither the name of the University nor the names of its contributors
  27  *    may be used to endorse or promote products derived from this software
  28  *    without specific prior written permission.
  29  *
  30  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  31  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  32  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  33  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  34  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  35  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  36  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  37  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  38  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  39  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  40  * SUCH DAMAGE.
  41  */
  42
  43 /*
  44  * Page to/from files (vnodes).
  45  */
  46
  47 /*
  48  * TODO:
  49  *      Implement VOP_GETPAGES/PUTPAGES interface for filesystems. Will
  50  *      greatly re-simplify the vnode_pager.
  51  */
  52
  53 #include <sys/cdefs.h>
  54 #include "opt_vm.h"
  55
  56 #include <sys/param.h>
  57 #include <sys/kernel.h>
  58 #include <sys/systm.h>
  59 #include <sys/sysctl.h>
  60 #include <sys/proc.h>
  61 #include <sys/vnode.h>
  62 #include <sys/mount.h>
  63 #include <sys/bio.h>
  64 #include <sys/buf.h>
  65 #include <sys/vmmeter.h>
  66 #include <sys/ktr.h>
  67 #include <sys/limits.h>
  68 #include <sys/conf.h>
  69 #include <sys/refcount.h>
  70 #include <sys/rwlock.h>
  71 #include <sys/sf_buf.h>
  72 #include <sys/domainset.h>
  73 #include <sys/user.h>
  74
  75 #include <machine/atomic.h>
  76
  77 #include <vm/vm.h>
  78 #include <vm/vm_param.h>
  79 #include <vm/vm_object.h>
  80 #include <vm/vm_page.h>
  81 #include <vm/vm_pager.h>
  82 #include <vm/vm_map.h>
  83 #include <vm/vnode_pager.h>
  84 #include <vm/vm_extern.h>
  85 #include <vm/uma.h>
  86
  87 static int vnode_pager_addr(struct vnode *vp, vm_ooffset_t address,
  88     daddr_t *rtaddress, int *run);
  89 static int vnode_pager_input_smlfs(vm_object_t object, vm_page_t m);
  90 static int vnode_pager_input_old(vm_object_t object, vm_page_t m);
  91 static void vnode_pager_dealloc(vm_object_t);
  92 static int vnode_pager_getpages(vm_object_t, vm_page_t *, int, int *, int *);
  93 static int vnode_pager_getpages_async(vm_object_t, vm_page_t *, int, int *,
  94     int *, vop_getpages_iodone_t, void *);
  95 static void vnode_pager_putpages(vm_object_t, vm_page_t *, int, int, int *);
  96 static boolean_t vnode_pager_haspage(vm_object_t, vm_pindex_t, int *, int *);
  97 static vm_object_t vnode_pager_alloc(void *, vm_ooffset_t, vm_prot_t,
  98     vm_ooffset_t, struct ucred *cred);
  99 static int vnode_pager_generic_getpages_done(struct buf *);
 100 static void vnode_pager_generic_getpages_done_async(struct buf *);
 101 static void vnode_pager_update_writecount(vm_object_t, vm_offset_t,
 102     vm_offset_t);
 103 static void vnode_pager_release_writecount(vm_object_t, vm_offset_t,
 104     vm_offset_t);
 105 static void vnode_pager_getvp(vm_object_t, struct vnode **, bool *);
 106
 107 const struct pagerops vnodepagerops = {
 108         .pgo_kvme_type = KVME_TYPE_VNODE,
 109         .pgo_alloc =    vnode_pager_alloc,
 110         .pgo_dealloc =  vnode_pager_dealloc,
 111         .pgo_getpages = vnode_pager_getpages,
 112         .pgo_getpages_async = vnode_pager_getpages_async,
 113         .pgo_putpages = vnode_pager_putpages,
 114         .pgo_haspage =  vnode_pager_haspage,
 115         .pgo_update_writecount = vnode_pager_update_writecount,
 116         .pgo_release_writecount = vnode_pager_release_writecount,
 117         .pgo_set_writeable_dirty = vm_object_set_writeable_dirty_,
 118         .pgo_mightbedirty = vm_object_mightbedirty_,
 119         .pgo_getvp = vnode_pager_getvp,
 120 };
 121
 122 static struct domainset *vnode_domainset = NULL;
 123
 124 SYSCTL_PROC(_debug, OID_AUTO, vnode_domainset,
 125     CTLTYPE_STRING | CTLFLAG_MPSAFE | CTLFLAG_RW, &vnode_domainset, 0,
 126     sysctl_handle_domainset, "A", "Default vnode NUMA policy");
 127
 128 static int nvnpbufs;
 129 SYSCTL_INT(_vm, OID_AUTO, vnode_pbufs, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
 130     &nvnpbufs, 0, "number of physical buffers allocated for vnode pager");
 131
 132 static uma_zone_t vnode_pbuf_zone;
 133
 134 static void
 135 vnode_pager_init(void *dummy)
 136 {
 137
 138 #ifdef __LP64__
 139         nvnpbufs = nswbuf * 2;
 140 #else
 141         nvnpbufs = nswbuf / 2;
 142 #endif
 143         TUNABLE_INT_FETCH("vm.vnode_pbufs", &nvnpbufs);
 144         vnode_pbuf_zone = pbuf_zsecond_create("vnpbuf", nvnpbufs);
 145 }
 146 SYSINIT(vnode_pager, SI_SUB_CPU, SI_ORDER_ANY, vnode_pager_init, NULL);
 147
 148 /* Create the VM system backing object for this vnode */
 149 int
 150 vnode_create_vobject(struct vnode *vp, off_t isize, struct thread *td)
 151 {
 152         vm_object_t object;
 153         vm_ooffset_t size = isize;
 154         bool last;
 155
 156         if (!vn_isdisk(vp) && vn_canvmio(vp) == FALSE)
 157                 return (0);
 158
 159         object = vp->v_object;
 160         if (object != NULL)
 161                 return (0);
 162
 163         if (size == 0) {
 164                 if (vn_isdisk(vp)) {
 165                         size = IDX_TO_OFF(INT_MAX);
 166                 } else {
 167                         if (vn_getsize_locked(vp, &size, td->td_ucred) != 0)
 168                                 return (0);
 169                 }
 170         }
 171
 172         object = vnode_pager_alloc(vp, size, 0, 0, td->td_ucred);
 173         /*
 174          * Dereference the reference we just created.  This assumes
 175          * that the object is associated with the vp.  We still have
 176          * to serialize with vnode_pager_dealloc() for the last
 177          * potential reference.
 178          */
 179         VM_OBJECT_RLOCK(object);
 180         last = refcount_release(&object->ref_count);
 181         VM_OBJECT_RUNLOCK(object);
 182         if (last)
 183                 vrele(vp);
 184
 185         KASSERT(vp->v_object != NULL, ("vnode_create_vobject: NULL object"));
 186
 187         return (0);
 188 }
 189
 190 void
 191 vnode_destroy_vobject(struct vnode *vp)
 192 {
 193         struct vm_object *obj;
 194
 195         obj = vp->v_object;
 196         if (obj == NULL || obj->handle != vp)
 197                 return;
 198         ASSERT_VOP_ELOCKED(vp, "vnode_destroy_vobject");
 199         VM_OBJECT_WLOCK(obj);
 200         MPASS(obj->type == OBJT_VNODE);
 201         umtx_shm_object_terminated(obj);
 202         if (obj->ref_count == 0) {
 203                 KASSERT((obj->flags & OBJ_DEAD) == 0,
 204                    ("vnode_destroy_vobject: Terminating dead object"));
 205                 vm_object_set_flag(obj, OBJ_DEAD);
 206
 207                 /*
 208                  * Clean pages and flush buffers.
 209                  */
 210                 vm_object_page_clean(obj, 0, 0, OBJPC_SYNC);
 211                 VM_OBJECT_WUNLOCK(obj);
 212
 213                 vinvalbuf(vp, V_SAVE, 0, 0);
 214
 215                 BO_LOCK(&vp->v_bufobj);
 216                 vp->v_bufobj.bo_flag |= BO_DEAD;
 217                 BO_UNLOCK(&vp->v_bufobj);
 218
 219                 VM_OBJECT_WLOCK(obj);
 220                 vm_object_terminate(obj);
 221         } else {
 222                 /*
 223                  * Woe to the process that tries to page now :-).
 224                  */
 225                 vm_pager_deallocate(obj);
 226                 VM_OBJECT_WUNLOCK(obj);
 227         }
 228         KASSERT(vp->v_object == NULL, ("vp %p obj %p", vp, vp->v_object));
 229 }
 230
 231 /*
 232  * Allocate (or lookup) pager for a vnode.
 233  * Handle is a vnode pointer.
 234  */
 235 vm_object_t
 236 vnode_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
 237     vm_ooffset_t offset, struct ucred *cred)
 238 {
 239         vm_object_t object;
 240         struct vnode *vp;
 241
 242         /*
 243          * Pageout to vnode, no can do yet.
 244          */
 245         if (handle == NULL)
 246                 return (NULL);
 247
 248         vp = (struct vnode *)handle;
 249         ASSERT_VOP_LOCKED(vp, "vnode_pager_alloc");
 250         VNPASS(vp->v_usecount > 0, vp);
 251 retry:
 252         object = vp->v_object;
 253
 254         if (object == NULL) {
 255                 /*
 256                  * Add an object of the appropriate size
 257                  */
 258                 object = vm_object_allocate(OBJT_VNODE,
 259                     OFF_TO_IDX(round_page(size)));
 260
 261                 object->un_pager.vnp.vnp_size = size;
 262                 object->un_pager.vnp.writemappings = 0;
 263                 object->domain.dr_policy = vnode_domainset;
 264                 object->handle = handle;
 265                 if ((vp->v_vflag & VV_VMSIZEVNLOCK) != 0) {
 266                         VM_OBJECT_WLOCK(object);
 267                         vm_object_set_flag(object, OBJ_SIZEVNLOCK);
 268                         VM_OBJECT_WUNLOCK(object);
 269                 }
 270                 VI_LOCK(vp);
 271                 if (vp->v_object != NULL) {
 272                         /*
 273                          * Object has been created while we were allocating.
 274                          */
 275                         VI_UNLOCK(vp);
 276                         VM_OBJECT_WLOCK(object);
 277                         KASSERT(object->ref_count == 1,
 278                             ("leaked ref %p %d", object, object->ref_count));
 279                         object->type = OBJT_DEAD;
 280                         refcount_init(&object->ref_count, 0);
 281                         VM_OBJECT_WUNLOCK(object);
 282                         vm_object_destroy(object);
 283                         goto retry;
 284                 }
 285                 vp->v_object = object;
 286                 VI_UNLOCK(vp);
 287                 vrefact(vp);
 288         } else {
 289                 vm_object_reference(object);
 290 #if VM_NRESERVLEVEL > 0
 291                 if ((object->flags & OBJ_COLORED) == 0) {
 292                         VM_OBJECT_WLOCK(object);
 293                         vm_object_color(object, 0);
 294                         VM_OBJECT_WUNLOCK(object);
 295                 }
 296 #endif
 297         }
 298         return (object);
 299 }
 300
 301 /*
 302  *      The object must be locked.
 303  */
 304 static void
 305 vnode_pager_dealloc(vm_object_t object)
 306 {
 307         struct vnode *vp;
 308         int refs;
 309
 310         vp = object->handle;
 311         if (vp == NULL)
 312                 panic("vnode_pager_dealloc: pager already dealloced");
 313
 314         VM_OBJECT_ASSERT_WLOCKED(object);
 315         vm_object_pip_wait(object, "vnpdea");
 316         refs = object->ref_count;
 317
 318         object->handle = NULL;
 319         object->type = OBJT_DEAD;
 320         ASSERT_VOP_ELOCKED(vp, "vnode_pager_dealloc");
 321         if (object->un_pager.vnp.writemappings > 0) {
 322                 object->un_pager.vnp.writemappings = 0;
 323                 VOP_ADD_WRITECOUNT_CHECKED(vp, -1);
 324                 CTR3(KTR_VFS, "%s: vp %p v_writecount decreased to %d",
 325                     __func__, vp, vp->v_writecount);
 326         }
 327         vp->v_object = NULL;
 328         VI_LOCK(vp);
 329
 330         /*
 331          * vm_map_entry_set_vnode_text() cannot reach this vnode by
 332          * following object->handle.  Clear all text references now.
 333          * This also clears the transient references from
 334          * kern_execve(), which is fine because dead_vnodeops uses nop
 335          * for VOP_UNSET_TEXT().
 336          */
 337         if (vp->v_writecount < 0)
 338                 vp->v_writecount = 0;
 339         VI_UNLOCK(vp);
 340         VM_OBJECT_WUNLOCK(object);
 341         if (refs > 0)
 342                 vunref(vp);
 343         VM_OBJECT_WLOCK(object);
 344 }
 345
 346 static boolean_t
 347 vnode_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before,
 348     int *after)
 349 {
 350         struct vnode *vp = object->handle;
 351         daddr_t bn;
 352         uintptr_t lockstate;
 353         int err;
 354         daddr_t reqblock;
 355         int poff;
 356         int bsize;
 357         int pagesperblock, blocksperpage;
 358
 359         VM_OBJECT_ASSERT_LOCKED(object);
 360         /*
 361          * If no vp or vp is doomed or marked transparent to VM, we do not
 362          * have the page.
 363          */
 364         if (vp == NULL || VN_IS_DOOMED(vp))
 365                 return FALSE;
 366         /*
 367          * If the offset is beyond end of file we do
 368          * not have the page.
 369          */
 370         if (IDX_TO_OFF(pindex) >= object->un_pager.vnp.vnp_size)
 371                 return FALSE;
 372
 373         bsize = vp->v_mount->mnt_stat.f_iosize;
 374         pagesperblock = bsize / PAGE_SIZE;
 375         blocksperpage = 0;
 376         if (pagesperblock > 0) {
 377                 reqblock = pindex / pagesperblock;
 378         } else {
 379                 blocksperpage = (PAGE_SIZE / bsize);
 380                 reqblock = pindex * blocksperpage;
 381         }
 382         lockstate = VM_OBJECT_DROP(object);
 383         err = VOP_BMAP(vp, reqblock, NULL, &bn, after, before);
 384         VM_OBJECT_PICKUP(object, lockstate);
 385         if (err)
 386                 return TRUE;
 387         if (bn == -1)
 388                 return FALSE;
 389         if (pagesperblock > 0) {
 390                 poff = pindex - (reqblock * pagesperblock);
 391                 if (before) {
 392                         *before *= pagesperblock;
 393                         *before += poff;
 394                 }
 395                 if (after) {
 396                         /*
 397                          * The BMAP vop can report a partial block in the
 398                          * 'after', but must not report blocks after EOF.
 399                          * Assert the latter, and truncate 'after' in case
 400                          * of the former.
 401                          */
 402                         KASSERT((reqblock + *after) * pagesperblock <
 403                             roundup2(object->size, pagesperblock),
 404                             ("%s: reqblock %jd after %d size %ju", __func__,
 405                             (intmax_t )reqblock, *after,
 406                             (uintmax_t )object->size));
 407                         *after *= pagesperblock;
 408                         *after += pagesperblock - (poff + 1);
 409                         if (pindex + *after >= object->size)
 410                                 *after = object->size - 1 - pindex;
 411                 }
 412         } else {
 413                 if (before) {
 414                         *before /= blocksperpage;
 415                 }
 416
 417                 if (after) {
 418                         *after /= blocksperpage;
 419                 }
 420         }
 421         return TRUE;
 422 }
 423
 424 /*
 425  * Internal routine clearing partial-page content
 426  */
 427 static void
 428 vnode_pager_subpage_purge(struct vm_page *m, int base, int end)
 429 {
 430         int size;
 431
 432         KASSERT(end > base && end <= PAGE_SIZE,
 433             ("%s: start %d end %d", __func__, base, end));
 434         size = end - base;
 435
 436         /*
 437          * Clear out partial-page garbage in case
 438          * the page has been mapped.
 439          */
 440         pmap_zero_page_area(m, base, size);
 441
 442         /*
 443          * Update the valid bits to reflect the blocks
 444          * that have been zeroed.  Some of these valid
 445          * bits may have already been set.
 446          */
 447         vm_page_set_valid_range(m, base, size);
 448
 449         /*
 450          * Round up "base" to the next block boundary so
 451          * that the dirty bit for a partially zeroed
 452          * block is not cleared.
 453          */
 454         base = roundup2(base, DEV_BSIZE);
 455         end = rounddown2(end, DEV_BSIZE);
 456
 457         if (end > base) {
 458                 /*
 459                  * Clear out partial-page dirty bits.
 460                  *
 461                  * note that we do not clear out the
 462                  * valid bits.  This would prevent
 463                  * bogus_page replacement from working
 464                  * properly.
 465                  */
 466                 vm_page_clear_dirty(m, base, end - base);
 467         }
 468
 469 }
 470
 471 /*
 472  * Lets the VM system know about a change in size for a file.
 473  * We adjust our own internal size and flush any cached pages in
 474  * the associated object that are affected by the size change.
 475  *
 476  * Note: this routine may be invoked as a result of a pager put
 477  * operation (possibly at object termination time), so we must be careful.
 478  */
 479 void
 480 vnode_pager_setsize(struct vnode *vp, vm_ooffset_t nsize)
 481 {
 482         vm_object_t object;
 483         vm_page_t m;
 484         vm_pindex_t nobjsize;
 485
 486         if ((object = vp->v_object) == NULL)
 487                 return;
 488 #ifdef DEBUG_VFS_LOCKS
 489         {
 490                 struct mount *mp;
 491
 492                 mp = vp->v_mount;
 493                 if (mp != NULL && (mp->mnt_kern_flag & MNTK_VMSETSIZE_BUG) == 0)
 494                         assert_vop_elocked(vp,
 495                             "vnode_pager_setsize and not locked vnode");
 496         }
 497 #endif
 498         VM_OBJECT_WLOCK(object);
 499         if (object->type == OBJT_DEAD) {
 500                 VM_OBJECT_WUNLOCK(object);
 501                 return;
 502         }
 503         KASSERT(object->type == OBJT_VNODE,
 504             ("not vnode-backed object %p", object));
 505         if (nsize == object->un_pager.vnp.vnp_size) {
 506                 /*
 507                  * Hasn't changed size
 508                  */
 509                 VM_OBJECT_WUNLOCK(object);
 510                 return;
 511         }
 512         nobjsize = OFF_TO_IDX(nsize + PAGE_MASK);
 513         if (nsize < object->un_pager.vnp.vnp_size) {
 514                 /*
 515                  * File has shrunk. Toss any cached pages beyond the new EOF.
 516                  */
 517                 if (nobjsize < object->size)
 518                         vm_object_page_remove(object, nobjsize, object->size,
 519                             0);
 520                 /*
 521                  * this gets rid of garbage at the end of a page that is now
 522                  * only partially backed by the vnode.
 523                  *
 524                  * XXX for some reason (I don't know yet), if we take a
 525                  * completely invalid page and mark it partially valid
 526                  * it can screw up NFS reads, so we don't allow the case.
 527                  */
 528                 if (!(nsize & PAGE_MASK))
 529                         goto out;
 530                 m = vm_page_grab(object, OFF_TO_IDX(nsize), VM_ALLOC_NOCREAT);
 531                 if (m == NULL)
 532                         goto out;
 533                 if (!vm_page_none_valid(m))
 534                         vnode_pager_subpage_purge(m, (int)nsize & PAGE_MASK,
 535                             PAGE_SIZE);
 536                 vm_page_xunbusy(m);
 537         }
 538 out:
 539 #if defined(__powerpc__) && !defined(__powerpc64__)
 540         object->un_pager.vnp.vnp_size = nsize;
 541 #else
 542         atomic_store_64(&object->un_pager.vnp.vnp_size, nsize);
 543 #endif
 544         object->size = nobjsize;
 545         VM_OBJECT_WUNLOCK(object);
 546 }
 547
 548 /*
 549  * Lets the VM system know about the purged range for a file. We toss away any
 550  * cached pages in the associated object that are affected by the purge
 551  * operation. Partial-page area not aligned to page boundaries will be zeroed
 552  * and the dirty blocks in DEV_BSIZE unit within a page will not be flushed.
 553  */
 554 void
 555 vnode_pager_purge_range(struct vnode *vp, vm_ooffset_t start, vm_ooffset_t end)
 556 {
 557         struct vm_page *m;
 558         struct vm_object *object;
 559         vm_pindex_t pi, pistart, piend;
 560         bool same_page;
 561         int base, pend;
 562
 563         ASSERT_VOP_LOCKED(vp, "vnode_pager_purge_range");
 564
 565         object = vp->v_object;
 566         pi = start + PAGE_MASK < start ? OBJ_MAX_SIZE :
 567             OFF_TO_IDX(start + PAGE_MASK);
 568         pistart = OFF_TO_IDX(start);
 569         piend = end == 0 ? OBJ_MAX_SIZE : OFF_TO_IDX(end);
 570         same_page = pistart == piend;
 571         if ((end != 0 && end <= start) || object == NULL)
 572                 return;
 573
 574         VM_OBJECT_WLOCK(object);
 575
 576         if (pi < piend)
 577                 vm_object_page_remove(object, pi, piend, 0);
 578
 579         if ((start & PAGE_MASK) != 0) {
 580                 base = (int)start & PAGE_MASK;
 581                 pend = same_page ? (int)end & PAGE_MASK : PAGE_SIZE;
 582                 m = vm_page_grab(object, pistart, VM_ALLOC_NOCREAT);
 583                 if (m != NULL) {
 584                         if (!vm_page_none_valid(m))
 585                                 vnode_pager_subpage_purge(m, base, pend);
 586                         vm_page_xunbusy(m);
 587                 }
 588                 if (same_page)
 589                         goto out;
 590         }
 591         if ((end & PAGE_MASK) != 0) {
 592                 base = same_page ? (int)start & PAGE_MASK : 0 ;
 593                 pend = (int)end & PAGE_MASK;
 594                 m = vm_page_grab(object, piend, VM_ALLOC_NOCREAT);
 595                 if (m != NULL) {
 596                         if (!vm_page_none_valid(m))
 597                                 vnode_pager_subpage_purge(m, base, pend);
 598                         vm_page_xunbusy(m);
 599                 }
 600         }
 601 out:
 602         VM_OBJECT_WUNLOCK(object);
 603 }
 604
 605 /*
 606  * calculate the linear (byte) disk address of specified virtual
 607  * file address
 608  */
 609 static int
 610 vnode_pager_addr(struct vnode *vp, vm_ooffset_t address, daddr_t *rtaddress,
 611     int *run)
 612 {
 613         int bsize;
 614         int err;
 615         daddr_t vblock;
 616         daddr_t voffset;
 617
 618         if (VN_IS_DOOMED(vp))
 619                 return -1;
 620
 621         bsize = vp->v_mount->mnt_stat.f_iosize;
 622         vblock = address / bsize;
 623         voffset = address % bsize;
 624
 625         err = VOP_BMAP(vp, vblock, NULL, rtaddress, run, NULL);
 626         if (err == 0) {
 627                 if (*rtaddress != -1)
 628                         *rtaddress += voffset / DEV_BSIZE;
 629                 if (run) {
 630                         *run += 1;
 631                         *run *= bsize / PAGE_SIZE;
 632                         *run -= voffset / PAGE_SIZE;
 633                 }
 634         }
 635
 636         return (err);
 637 }
 638
 639 static void
 640 vnode_pager_input_bdone(struct buf *bp)
 641 {
 642         runningbufwakeup(bp);
 643         bdone(bp);
 644 }
 645
 646 /*
 647  * small block filesystem vnode pager input
 648  */
 649 static int
 650 vnode_pager_input_smlfs(vm_object_t object, vm_page_t m)
 651 {
 652         struct vnode *vp;
 653         struct bufobj *bo;
 654         struct buf *bp;
 655         struct sf_buf *sf;
 656         daddr_t fileaddr;
 657         vm_offset_t bsize;
 658         vm_page_bits_t bits;
 659         int error, i;
 660
 661         error = 0;
 662         vp = object->handle;
 663         if (VN_IS_DOOMED(vp))
 664                 return VM_PAGER_BAD;
 665
 666         bsize = vp->v_mount->mnt_stat.f_iosize;
 667
 668         VOP_BMAP(vp, 0, &bo, 0, NULL, NULL);
 669
 670         sf = sf_buf_alloc(m, 0);
 671
 672         for (i = 0; i < PAGE_SIZE / bsize; i++) {
 673                 vm_ooffset_t address;
 674
 675                 bits = vm_page_bits(i * bsize, bsize);
 676                 if (m->valid & bits)
 677                         continue;
 678
 679                 address = IDX_TO_OFF(m->pindex) + i * bsize;
 680                 if (address >= object->un_pager.vnp.vnp_size) {
 681                         fileaddr = -1;
 682                 } else {
 683                         error = vnode_pager_addr(vp, address, &fileaddr, NULL);
 684                         if (error)
 685                                 break;
 686                 }
 687                 if (fileaddr != -1) {
 688                         bp = uma_zalloc(vnode_pbuf_zone, M_WAITOK);
 689
 690                         /* build a minimal buffer header */
 691                         bp->b_iocmd = BIO_READ;
 692                         bp->b_iodone = vnode_pager_input_bdone;
 693                         KASSERT(bp->b_rcred == NOCRED, ("leaking read ucred"));
 694                         KASSERT(bp->b_wcred == NOCRED, ("leaking write ucred"));
 695                         bp->b_rcred = crhold(curthread->td_ucred);
 696                         bp->b_wcred = crhold(curthread->td_ucred);
 697                         bp->b_data = (caddr_t)sf_buf_kva(sf) + i * bsize;
 698                         bp->b_blkno = fileaddr;
 699                         pbgetbo(bo, bp);
 700                         bp->b_vp = vp;
 701                         bp->b_bcount = bsize;
 702                         bp->b_bufsize = bsize;
 703                         bp->b_runningbufspace = bp->b_bufsize;
 704                         atomic_add_long(&runningbufspace, bp->b_runningbufspace);
 705
 706                         /* do the input */
 707                         bp->b_iooffset = dbtob(bp->b_blkno);
 708                         bstrategy(bp);
 709
 710                         bwait(bp, PVM, "vnsrd");
 711
 712                         if ((bp->b_ioflags & BIO_ERROR) != 0) {
 713                                 KASSERT(bp->b_error != 0,
 714                                     ("%s: buf error but b_error == 0\n", __func__));
 715                                 error = bp->b_error;
 716                         }
 717
 718                         /*
 719                          * free the buffer header back to the swap buffer pool
 720                          */
 721                         bp->b_vp = NULL;
 722                         pbrelbo(bp);
 723                         uma_zfree(vnode_pbuf_zone, bp);
 724                         if (error)
 725                                 break;
 726                 } else
 727                         bzero((caddr_t)sf_buf_kva(sf) + i * bsize, bsize);
 728                 KASSERT((m->dirty & bits) == 0,
 729                     ("vnode_pager_input_smlfs: page %p is dirty", m));
 730                 vm_page_bits_set(m, &m->valid, bits);
 731         }
 732         sf_buf_free(sf);
 733         if (error) {
 734                 return VM_PAGER_ERROR;
 735         }
 736         return VM_PAGER_OK;
 737 }
 738
 739 /*
 740  * old style vnode pager input routine
 741  */
 742 static int
 743 vnode_pager_input_old(vm_object_t object, vm_page_t m)
 744 {
 745         struct uio auio;
 746         struct iovec aiov;
 747         int error;
 748         int size;
 749         struct sf_buf *sf;
 750         struct vnode *vp;
 751
 752         VM_OBJECT_ASSERT_WLOCKED(object);
 753         error = 0;
 754
 755         /*
 756          * Return failure if beyond current EOF
 757          */
 758         if (IDX_TO_OFF(m->pindex) >= object->un_pager.vnp.vnp_size) {
 759                 return VM_PAGER_BAD;
 760         } else {
 761                 size = PAGE_SIZE;
 762                 if (IDX_TO_OFF(m->pindex) + size > object->un_pager.vnp.vnp_size)
 763                         size = object->un_pager.vnp.vnp_size - IDX_TO_OFF(m->pindex);
 764                 vp = object->handle;
 765                 VM_OBJECT_WUNLOCK(object);
 766
 767                 /*
 768                  * Allocate a kernel virtual address and initialize so that
 769                  * we can use VOP_READ/WRITE routines.
 770                  */
 771                 sf = sf_buf_alloc(m, 0);
 772
 773                 aiov.iov_base = (caddr_t)sf_buf_kva(sf);
 774                 aiov.iov_len = size;
 775                 auio.uio_iov = &aiov;
 776                 auio.uio_iovcnt = 1;
 777                 auio.uio_offset = IDX_TO_OFF(m->pindex);
 778                 auio.uio_segflg = UIO_SYSSPACE;
 779                 auio.uio_rw = UIO_READ;
 780                 auio.uio_resid = size;
 781                 auio.uio_td = curthread;
 782
 783                 error = VOP_READ(vp, &auio, 0, curthread->td_ucred);
 784                 if (!error) {
 785                         int count = size - auio.uio_resid;
 786
 787                         if (count == 0)
 788                                 error = EINVAL;
 789                         else if (count != PAGE_SIZE)
 790                                 bzero((caddr_t)sf_buf_kva(sf) + count,
 791                                     PAGE_SIZE - count);
 792                 }
 793                 sf_buf_free(sf);
 794
 795                 VM_OBJECT_WLOCK(object);
 796         }
 797         KASSERT(m->dirty == 0, ("vnode_pager_input_old: page %p is dirty", m));
 798         if (!error)
 799                 vm_page_valid(m);
 800         return error ? VM_PAGER_ERROR : VM_PAGER_OK;
 801 }
 802
 803 /*
 804  * generic vnode pager input routine
 805  */
 806
 807 /*
 808  * Local media VFS's that do not implement their own VOP_GETPAGES
 809  * should have their VOP_GETPAGES call to vnode_pager_generic_getpages()
 810  * to implement the previous behaviour.
 811  *
 812  * All other FS's should use the bypass to get to the local media
 813  * backing vp's VOP_GETPAGES.
 814  */
 815 static int
 816 vnode_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind,
 817     int *rahead)
 818 {
 819         struct vnode *vp;
 820         int rtval;
 821
 822         /* Handle is stable with paging in progress. */
 823         vp = object->handle;
 824         rtval = VOP_GETPAGES(vp, m, count, rbehind, rahead);
 825         KASSERT(rtval != EOPNOTSUPP,
 826             ("vnode_pager: FS getpages not implemented\n"));
 827         return rtval;
 828 }
 829
 830 static int
 831 vnode_pager_getpages_async(vm_object_t object, vm_page_t *m, int count,
 832     int *rbehind, int *rahead, vop_getpages_iodone_t iodone, void *arg)
 833 {
 834         struct vnode *vp;
 835         int rtval;
 836
 837         vp = object->handle;
 838         rtval = VOP_GETPAGES_ASYNC(vp, m, count, rbehind, rahead, iodone, arg);
 839         KASSERT(rtval != EOPNOTSUPP,
 840             ("vnode_pager: FS getpages_async not implemented\n"));
 841         return (rtval);
 842 }
 843
 844 /*
 845  * The implementation of VOP_GETPAGES() and VOP_GETPAGES_ASYNC() for
 846  * local filesystems, where partially valid pages can only occur at
 847  * the end of file.
 848  */
 849 int
 850 vnode_pager_local_getpages(struct vop_getpages_args *ap)
 851 {
 852
 853         return (vnode_pager_generic_getpages(ap->a_vp, ap->a_m, ap->a_count,
 854             ap->a_rbehind, ap->a_rahead, NULL, NULL));
 855 }
 856
 857 int
 858 vnode_pager_local_getpages_async(struct vop_getpages_async_args *ap)
 859 {
 860         int error;
 861
 862         error = vnode_pager_generic_getpages(ap->a_vp, ap->a_m, ap->a_count,
 863             ap->a_rbehind, ap->a_rahead, ap->a_iodone, ap->a_arg);
 864         if (error != 0 && ap->a_iodone != NULL)
 865                 ap->a_iodone(ap->a_arg, ap->a_m, ap->a_count, error);
 866         return (error);
 867 }
 868
 869 /*
 870  * This is now called from local media FS's to operate against their
 871  * own vnodes if they fail to implement VOP_GETPAGES.
 872  */
 873 int
 874 vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int count,
 875     int *a_rbehind, int *a_rahead, vop_getpages_iodone_t iodone, void *arg)
 876 {
 877         vm_object_t object;
 878         struct bufobj *bo;
 879         struct buf *bp;
 880         off_t foff;
 881 #ifdef INVARIANTS
 882         off_t blkno0;
 883 #endif
 884         int bsize, pagesperblock;
 885         int error, before, after, rbehind, rahead, poff, i;
 886         int bytecount, secmask;
 887
 888         KASSERT(vp->v_type != VCHR && vp->v_type != VBLK,
 889             ("%s does not support devices", __func__));
 890
 891         if (VN_IS_DOOMED(vp))
 892                 return (VM_PAGER_BAD);
 893
 894         object = vp->v_object;
 895         foff = IDX_TO_OFF(m[0]->pindex);
 896         bsize = vp->v_mount->mnt_stat.f_iosize;
 897         pagesperblock = bsize / PAGE_SIZE;
 898
 899         KASSERT(foff < object->un_pager.vnp.vnp_size,
 900             ("%s: page %p offset beyond vp %p size", __func__, m[0], vp));
 901         KASSERT(count <= atop(maxphys),
 902             ("%s: requested %d pages", __func__, count));
 903
 904         /*
 905          * The last page has valid blocks.  Invalid part can only
 906          * exist at the end of file, and the page is made fully valid
 907          * by zeroing in vm_pager_get_pages().
 908          */
 909         if (!vm_page_none_valid(m[count - 1]) && --count == 0) {
 910                 if (iodone != NULL)
 911                         iodone(arg, m, 1, 0);
 912                 return (VM_PAGER_OK);
 913         }
 914
 915         bp = uma_zalloc(vnode_pbuf_zone, M_WAITOK);
 916         MPASS((bp->b_flags & B_MAXPHYS) != 0);
 917
 918         /*
 919          * Get the underlying device blocks for the file with VOP_BMAP().
 920          * If the file system doesn't support VOP_BMAP, use old way of
 921          * getting pages via VOP_READ.
 922          */
 923         error = VOP_BMAP(vp, foff / bsize, &bo, &bp->b_blkno, &after, &before);
 924         if (error == EOPNOTSUPP) {
 925                 uma_zfree(vnode_pbuf_zone, bp);
 926                 VM_OBJECT_WLOCK(object);
 927                 for (i = 0; i < count; i++) {
 928                         VM_CNT_INC(v_vnodein);
 929                         VM_CNT_INC(v_vnodepgsin);
 930                         error = vnode_pager_input_old(object, m[i]);
 931                         if (error)
 932                                 break;
 933                 }
 934                 VM_OBJECT_WUNLOCK(object);
 935                 return (error);
 936         } else if (error != 0) {
 937                 uma_zfree(vnode_pbuf_zone, bp);
 938                 return (VM_PAGER_ERROR);
 939         }
 940
 941         /*
 942          * If the file system supports BMAP, but blocksize is smaller
 943          * than a page size, then use special small filesystem code.
 944          */
 945         if (pagesperblock == 0) {
 946                 uma_zfree(vnode_pbuf_zone, bp);
 947                 for (i = 0; i < count; i++) {
 948                         VM_CNT_INC(v_vnodein);
 949                         VM_CNT_INC(v_vnodepgsin);
 950                         error = vnode_pager_input_smlfs(object, m[i]);
 951                         if (error)
 952                                 break;
 953                 }
 954                 return (error);
 955         }
 956
 957         /*
 958          * A sparse file can be encountered only for a single page request,
 959          * which may not be preceded by call to vm_pager_haspage().
 960          */
 961         if (bp->b_blkno == -1) {
 962                 KASSERT(count == 1,
 963                     ("%s: array[%d] request to a sparse file %p", __func__,
 964                     count, vp));
 965                 uma_zfree(vnode_pbuf_zone, bp);
 966                 pmap_zero_page(m[0]);
 967                 KASSERT(m[0]->dirty == 0, ("%s: page %p is dirty",
 968                     __func__, m[0]));
 969                 vm_page_valid(m[0]);
 970                 return (VM_PAGER_OK);
 971         }
 972
 973 #ifdef INVARIANTS
 974         blkno0 = bp->b_blkno;
 975 #endif
 976         bp->b_blkno += (foff % bsize) / DEV_BSIZE;
 977
 978         /* Recalculate blocks available after/before to pages. */
 979         poff = (foff % bsize) / PAGE_SIZE;
 980         before *= pagesperblock;
 981         before += poff;
 982         after *= pagesperblock;
 983         after += pagesperblock - (poff + 1);
 984         if (m[0]->pindex + after >= object->size)
 985                 after = object->size - 1 - m[0]->pindex;
 986         KASSERT(count <= after + 1, ("%s: %d pages asked, can do only %d",
 987             __func__, count, after + 1));
 988         after -= count - 1;
 989
 990         /* Trim requested rbehind/rahead to possible values. */
 991         rbehind = a_rbehind ? *a_rbehind : 0;
 992         rahead = a_rahead ? *a_rahead : 0;
 993         rbehind = min(rbehind, before);
 994         rbehind = min(rbehind, m[0]->pindex);
 995         rahead = min(rahead, after);
 996         rahead = min(rahead, object->size - m[count - 1]->pindex);
 997         /*
 998          * Check that total amount of pages fit into buf.  Trim rbehind and
 999          * rahead evenly if not.
1000          */
1001         if (rbehind + rahead + count > atop(maxphys)) {
1002                 int trim, sum;
1003
1004                 trim = rbehind + rahead + count - atop(maxphys) + 1;
1005                 sum = rbehind + rahead;
1006                 if (rbehind == before) {
1007                         /* Roundup rbehind trim to block size. */
1008                         rbehind -= roundup(trim * rbehind / sum, pagesperblock);
1009                         if (rbehind < 0)
1010                                 rbehind = 0;
1011                 } else
1012                         rbehind -= trim * rbehind / sum;
1013                 rahead -= trim * rahead / sum;
1014         }
1015         KASSERT(rbehind + rahead + count <= atop(maxphys),
1016             ("%s: behind %d ahead %d count %d maxphys %lu", __func__,
1017             rbehind, rahead, count, maxphys));
1018
1019         /*
1020          * Fill in the bp->b_pages[] array with requested and optional
1021          * read behind or read ahead pages.  Read behind pages are looked
1022          * up in a backward direction, down to a first cached page.  Same
1023          * for read ahead pages, but there is no need to shift the array
1024          * in case of encountering a cached page.
1025          */
1026         i = bp->b_npages = 0;
1027         if (rbehind) {
1028                 vm_pindex_t startpindex, tpindex;
1029                 vm_page_t p;
1030
1031                 VM_OBJECT_WLOCK(object);
1032                 startpindex = m[0]->pindex - rbehind;
1033                 if ((p = TAILQ_PREV(m[0], pglist, listq)) != NULL &&
1034                     p->pindex >= startpindex)
1035                         startpindex = p->pindex + 1;
1036
1037                 /* tpindex is unsigned; beware of numeric underflow. */
1038                 for (tpindex = m[0]->pindex - 1;
1039                     tpindex >= startpindex && tpindex < m[0]->pindex;
1040                     tpindex--, i++) {
1041                         p = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL);
1042                         if (p == NULL) {
1043                                 /* Shift the array. */
1044                                 for (int j = 0; j < i; j++)
1045                                         bp->b_pages[j] = bp->b_pages[j +
1046                                             tpindex + 1 - startpindex];
1047                                 break;
1048                         }
1049                         bp->b_pages[tpindex - startpindex] = p;
1050                 }
1051
1052                 bp->b_pgbefore = i;
1053                 bp->b_npages += i;
1054                 bp->b_blkno -= IDX_TO_OFF(i) / DEV_BSIZE;
1055         } else
1056                 bp->b_pgbefore = 0;
1057
1058         /* Requested pages. */
1059         for (int j = 0; j < count; j++, i++)
1060                 bp->b_pages[i] = m[j];
1061         bp->b_npages += count;
1062
1063         if (rahead) {
1064                 vm_pindex_t endpindex, tpindex;
1065                 vm_page_t p;
1066
1067                 if (!VM_OBJECT_WOWNED(object))
1068                         VM_OBJECT_WLOCK(object);
1069                 endpindex = m[count - 1]->pindex + rahead + 1;
1070                 if ((p = TAILQ_NEXT(m[count - 1], listq)) != NULL &&
1071                     p->pindex < endpindex)
1072                         endpindex = p->pindex;
1073                 if (endpindex > object->size)
1074                         endpindex = object->size;
1075
1076                 for (tpindex = m[count - 1]->pindex + 1;
1077                     tpindex < endpindex; i++, tpindex++) {
1078                         p = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL);
1079                         if (p == NULL)
1080                                 break;
1081                         bp->b_pages[i] = p;
1082                 }
1083
1084                 bp->b_pgafter = i - bp->b_npages;
1085                 bp->b_npages = i;
1086         } else
1087                 bp->b_pgafter = 0;
1088
1089         if (VM_OBJECT_WOWNED(object))
1090                 VM_OBJECT_WUNLOCK(object);
1091
1092         /* Report back actual behind/ahead read. */
1093         if (a_rbehind)
1094                 *a_rbehind = bp->b_pgbefore;
1095         if (a_rahead)
1096                 *a_rahead = bp->b_pgafter;
1097
1098 #ifdef INVARIANTS
1099         KASSERT(bp->b_npages <= atop(maxphys),
1100             ("%s: buf %p overflowed", __func__, bp));
1101         for (int j = 1, prev = 0; j < bp->b_npages; j++) {
1102                 if (bp->b_pages[j] == bogus_page)
1103                         continue;
1104                 KASSERT(bp->b_pages[j]->pindex - bp->b_pages[prev]->pindex ==
1105                     j - prev, ("%s: pages array not consecutive, bp %p",
1106                      __func__, bp));
1107                 prev = j;
1108         }
1109 #endif
1110
1111         /*
1112          * Recalculate first offset and bytecount with regards to read behind.
1113          * Truncate bytecount to vnode real size and round up physical size
1114          * for real devices.
1115          */
1116         foff = IDX_TO_OFF(bp->b_pages[0]->pindex);
1117         bytecount = bp->b_npages << PAGE_SHIFT;
1118         if ((foff + bytecount) > object->un_pager.vnp.vnp_size)
1119                 bytecount = object->un_pager.vnp.vnp_size - foff;
1120         secmask = bo->bo_bsize - 1;
1121         KASSERT(secmask < PAGE_SIZE && secmask > 0,
1122             ("%s: sector size %d too large", __func__, secmask + 1));
1123         bytecount = (bytecount + secmask) & ~secmask;
1124
1125         /*
1126          * And map the pages to be read into the kva, if the filesystem
1127          * requires mapped buffers.
1128          */
1129         if ((vp->v_mount->mnt_kern_flag & MNTK_UNMAPPED_BUFS) != 0 &&
1130             unmapped_buf_allowed) {
1131                 bp->b_data = unmapped_buf;
1132                 bp->b_offset = 0;
1133         } else {
1134                 bp->b_data = bp->b_kvabase;
1135                 pmap_qenter((vm_offset_t)bp->b_data, bp->b_pages, bp->b_npages);
1136         }
1137
1138         /* Build a minimal buffer header. */
1139         bp->b_iocmd = BIO_READ;
1140         KASSERT(bp->b_rcred == NOCRED, ("leaking read ucred"));
1141         KASSERT(bp->b_wcred == NOCRED, ("leaking write ucred"));
1142         bp->b_rcred = crhold(curthread->td_ucred);
1143         bp->b_wcred = crhold(curthread->td_ucred);
1144         pbgetbo(bo, bp);
1145         bp->b_vp = vp;
1146         bp->b_bcount = bp->b_bufsize = bp->b_runningbufspace = bytecount;
1147         bp->b_iooffset = dbtob(bp->b_blkno);
1148         KASSERT(IDX_TO_OFF(m[0]->pindex - bp->b_pages[0]->pindex) ==
1149             (blkno0 - bp->b_blkno) * DEV_BSIZE +
1150             IDX_TO_OFF(m[0]->pindex) % bsize,
1151             ("wrong offsets bsize %d m[0] %ju b_pages[0] %ju "
1152             "blkno0 %ju b_blkno %ju", bsize,
1153             (uintmax_t)m[0]->pindex, (uintmax_t)bp->b_pages[0]->pindex,
1154             (uintmax_t)blkno0, (uintmax_t)bp->b_blkno));
1155
1156         atomic_add_long(&runningbufspace, bp->b_runningbufspace);
1157         VM_CNT_INC(v_vnodein);
1158         VM_CNT_ADD(v_vnodepgsin, bp->b_npages);
1159
1160         if (iodone != NULL) { /* async */
1161                 bp->b_pgiodone = iodone;
1162                 bp->b_caller1 = arg;
1163                 bp->b_iodone = vnode_pager_generic_getpages_done_async;
1164                 bp->b_flags |= B_ASYNC;
1165                 BUF_KERNPROC(bp);
1166                 bstrategy(bp);
1167                 return (VM_PAGER_OK);
1168         } else {
1169                 bp->b_iodone = bdone;
1170                 bstrategy(bp);
1171                 bwait(bp, PVM, "vnread");
1172                 error = vnode_pager_generic_getpages_done(bp);
1173                 for (i = 0; i < bp->b_npages; i++)
1174                         bp->b_pages[i] = NULL;
1175                 bp->b_vp = NULL;
1176                 pbrelbo(bp);
1177                 uma_zfree(vnode_pbuf_zone, bp);
1178                 return (error != 0 ? VM_PAGER_ERROR : VM_PAGER_OK);
1179         }
1180 }
1181
1182 static void
1183 vnode_pager_generic_getpages_done_async(struct buf *bp)
1184 {
1185         int error;
1186
1187         error = vnode_pager_generic_getpages_done(bp);
1188         /* Run the iodone upon the requested range. */
1189         bp->b_pgiodone(bp->b_caller1, bp->b_pages + bp->b_pgbefore,
1190             bp->b_npages - bp->b_pgbefore - bp->b_pgafter, error);
1191         for (int i = 0; i < bp->b_npages; i++)
1192                 bp->b_pages[i] = NULL;
1193         bp->b_vp = NULL;
1194         pbrelbo(bp);
1195         uma_zfree(vnode_pbuf_zone, bp);
1196 }
1197
1198 static int
1199 vnode_pager_generic_getpages_done(struct buf *bp)
1200 {
1201         vm_object_t object;
1202         off_t tfoff, nextoff;
1203         int i, error;
1204
1205         KASSERT((bp->b_ioflags & BIO_ERROR) == 0 || bp->b_error != 0,
1206             ("%s: buf error but b_error == 0\n", __func__));
1207         error = (bp->b_ioflags & BIO_ERROR) != 0 ? bp->b_error : 0;
1208         object = bp->b_vp->v_object;
1209
1210         runningbufwakeup(bp);
1211
1212         if (error == 0 && bp->b_bcount != bp->b_npages * PAGE_SIZE) {
1213                 if (!buf_mapped(bp)) {
1214                         bp->b_data = bp->b_kvabase;
1215                         pmap_qenter((vm_offset_t)bp->b_data, bp->b_pages,
1216                             bp->b_npages);
1217                 }
1218                 bzero(bp->b_data + bp->b_bcount,
1219                     PAGE_SIZE * bp->b_npages - bp->b_bcount);
1220         }
1221         if (buf_mapped(bp)) {
1222                 pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages);
1223                 bp->b_data = unmapped_buf;
1224         }
1225
1226         /*
1227          * If the read failed, we must free any read ahead/behind pages here.
1228          * The requested pages are freed by the caller (for sync requests)
1229          * or by the bp->b_pgiodone callback (for async requests).
1230          */
1231         if (error != 0) {
1232                 VM_OBJECT_WLOCK(object);
1233                 for (i = 0; i < bp->b_pgbefore; i++)
1234                         vm_page_free_invalid(bp->b_pages[i]);
1235                 for (i = bp->b_npages - bp->b_pgafter; i < bp->b_npages; i++)
1236                         vm_page_free_invalid(bp->b_pages[i]);
1237                 VM_OBJECT_WUNLOCK(object);
1238                 return (error);
1239         }
1240
1241         /* Read lock to protect size. */
1242         VM_OBJECT_RLOCK(object);
1243         for (i = 0, tfoff = IDX_TO_OFF(bp->b_pages[0]->pindex);
1244             i < bp->b_npages; i++, tfoff = nextoff) {
1245                 vm_page_t mt;
1246
1247                 nextoff = tfoff + PAGE_SIZE;
1248                 mt = bp->b_pages[i];
1249                 if (mt == bogus_page)
1250                         continue;
1251
1252                 if (nextoff <= object->un_pager.vnp.vnp_size) {
1253                         /*
1254                          * Read filled up entire page.
1255                          */
1256                         vm_page_valid(mt);
1257                         KASSERT(mt->dirty == 0,
1258                             ("%s: page %p is dirty", __func__, mt));
1259                         KASSERT(!pmap_page_is_mapped(mt),
1260                             ("%s: page %p is mapped", __func__, mt));
1261                 } else {
1262                         /*
1263                          * Read did not fill up entire page.
1264                          *
1265                          * Currently we do not set the entire page valid,
1266                          * we just try to clear the piece that we couldn't
1267                          * read.
1268                          */
1269                         vm_page_set_valid_range(mt, 0,
1270                             object->un_pager.vnp.vnp_size - tfoff);
1271                         KASSERT((mt->dirty & vm_page_bits(0,
1272                             object->un_pager.vnp.vnp_size - tfoff)) == 0,
1273                             ("%s: page %p is dirty", __func__, mt));
1274                 }
1275
1276                 if (i < bp->b_pgbefore || i >= bp->b_npages - bp->b_pgafter)
1277                         vm_page_readahead_finish(mt);
1278         }
1279         VM_OBJECT_RUNLOCK(object);
1280
1281         return (error);
1282 }
1283
1284 /*
1285  * EOPNOTSUPP is no longer legal.  For local media VFS's that do not
1286  * implement their own VOP_PUTPAGES, their VOP_PUTPAGES should call to
1287  * vnode_pager_generic_putpages() to implement the previous behaviour.
1288  *
1289  * All other FS's should use the bypass to get to the local media
1290  * backing vp's VOP_PUTPAGES.
1291  */
1292 static void
1293 vnode_pager_putpages(vm_object_t object, vm_page_t *m, int count,
1294     int flags, int *rtvals)
1295 {
1296         int rtval __diagused;
1297         struct vnode *vp;
1298         int bytes = count * PAGE_SIZE;
1299
1300         /*
1301          * Force synchronous operation if we are extremely low on memory
1302          * to prevent a low-memory deadlock.  VOP operations often need to
1303          * allocate more memory to initiate the I/O ( i.e. do a BMAP
1304          * operation ).  The swapper handles the case by limiting the amount
1305          * of asynchronous I/O, but that sort of solution doesn't scale well
1306          * for the vnode pager without a lot of work.
1307          *
1308          * Also, the backing vnode's iodone routine may not wake the pageout
1309          * daemon up.  This should be probably be addressed XXX.
1310          */
1311
1312         if (vm_page_count_min())
1313                 flags |= VM_PAGER_PUT_SYNC;
1314
1315         /*
1316          * Call device-specific putpages function
1317          */
1318         vp = object->handle;
1319         VM_OBJECT_WUNLOCK(object);
1320         rtval = VOP_PUTPAGES(vp, m, bytes, flags, rtvals);
1321         KASSERT(rtval != EOPNOTSUPP,
1322             ("vnode_pager: stale FS putpages\n"));
1323         VM_OBJECT_WLOCK(object);
1324 }
1325
1326 static int
1327 vn_off2bidx(vm_ooffset_t offset)
1328 {
1329
1330         return ((offset & PAGE_MASK) / DEV_BSIZE);
1331 }
1332
1333 static bool
1334 vn_dirty_blk(vm_page_t m, vm_ooffset_t offset)
1335 {
1336
1337         KASSERT(IDX_TO_OFF(m->pindex) <= offset &&
1338             offset < IDX_TO_OFF(m->pindex + 1),
1339             ("page %p pidx %ju offset %ju", m, (uintmax_t)m->pindex,
1340             (uintmax_t)offset));
1341         return ((m->dirty & ((vm_page_bits_t)1 << vn_off2bidx(offset))) != 0);
1342 }
1343
1344 /*
1345  * This is now called from local media FS's to operate against their
1346  * own vnodes if they fail to implement VOP_PUTPAGES.
1347  *
1348  * This is typically called indirectly via the pageout daemon and
1349  * clustering has already typically occurred, so in general we ask the
1350  * underlying filesystem to write the data out asynchronously rather
1351  * then delayed.
1352  */
1353 int
1354 vnode_pager_generic_putpages(struct vnode *vp, vm_page_t *ma, int bytecount,
1355     int flags, int *rtvals)
1356 {
1357         vm_object_t object;
1358         vm_page_t m;
1359         vm_ooffset_t max_offset, next_offset, poffset, prev_offset;
1360         struct uio auio;
1361         struct iovec aiov;
1362         off_t prev_resid, wrsz;
1363         int count, error, i, maxsize, ncount, pgoff, ppscheck;
1364         bool in_hole;
1365         static struct timeval lastfail;
1366         static int curfail;
1367
1368         object = vp->v_object;
1369         count = bytecount / PAGE_SIZE;
1370
1371         for (i = 0; i < count; i++)
1372                 rtvals[i] = VM_PAGER_ERROR;
1373
1374         if ((int64_t)ma[0]->pindex < 0) {
1375                 printf("vnode_pager_generic_putpages: "
1376                     "attempt to write meta-data 0x%jx(%lx)\n",
1377                     (uintmax_t)ma[0]->pindex, (u_long)ma[0]->dirty);
1378                 rtvals[0] = VM_PAGER_BAD;
1379                 return (VM_PAGER_BAD);
1380         }
1381
1382         maxsize = count * PAGE_SIZE;
1383         ncount = count;
1384
1385         poffset = IDX_TO_OFF(ma[0]->pindex);
1386
1387         /*
1388          * If the page-aligned write is larger then the actual file we
1389          * have to invalidate pages occurring beyond the file EOF.  However,
1390          * there is an edge case where a file may not be page-aligned where
1391          * the last page is partially invalid.  In this case the filesystem
1392          * may not properly clear the dirty bits for the entire page (which
1393          * could be VM_PAGE_BITS_ALL due to the page having been mmap()d).
1394          * With the page busied we are free to fix up the dirty bits here.
1395          *
1396          * We do not under any circumstances truncate the valid bits, as
1397          * this will screw up bogus page replacement.
1398          */
1399         VM_OBJECT_RLOCK(object);
1400         if (maxsize + poffset > object->un_pager.vnp.vnp_size) {
1401                 if (object->un_pager.vnp.vnp_size > poffset) {
1402                         maxsize = object->un_pager.vnp.vnp_size - poffset;
1403                         ncount = btoc(maxsize);
1404                         if ((pgoff = (int)maxsize & PAGE_MASK) != 0) {
1405                                 pgoff = roundup2(pgoff, DEV_BSIZE);
1406
1407                                 /*
1408                                  * If the page is busy and the following
1409                                  * conditions hold, then the page's dirty
1410                                  * field cannot be concurrently changed by a
1411                                  * pmap operation.
1412                                  */
1413                                 m = ma[ncount - 1];
1414                                 vm_page_assert_sbusied(m);
1415                                 KASSERT(!pmap_page_is_write_mapped(m),
1416                 ("vnode_pager_generic_putpages: page %p is not read-only", m));
1417                                 MPASS(m->dirty != 0);
1418                                 vm_page_clear_dirty(m, pgoff, PAGE_SIZE -
1419                                     pgoff);
1420                         }
1421                 } else {
1422                         maxsize = 0;
1423                         ncount = 0;
1424                 }
1425                 for (i = ncount; i < count; i++)
1426                         rtvals[i] = VM_PAGER_BAD;
1427         }
1428         VM_OBJECT_RUNLOCK(object);
1429
1430         auio.uio_iov = &aiov;
1431         auio.uio_segflg = UIO_NOCOPY;
1432         auio.uio_rw = UIO_WRITE;
1433         auio.uio_td = NULL;
1434         max_offset = roundup2(poffset + maxsize, DEV_BSIZE);
1435
1436         for (prev_offset = poffset; prev_offset < max_offset;) {
1437                 /* Skip clean blocks. */
1438                 for (in_hole = true; in_hole && prev_offset < max_offset;) {
1439                         m = ma[OFF_TO_IDX(prev_offset - poffset)];
1440                         for (i = vn_off2bidx(prev_offset);
1441                             i < sizeof(vm_page_bits_t) * NBBY &&
1442                             prev_offset < max_offset; i++) {
1443                                 if (vn_dirty_blk(m, prev_offset)) {
1444                                         in_hole = false;
1445                                         break;
1446                                 }
1447                                 prev_offset += DEV_BSIZE;
1448                         }
1449                 }
1450                 if (in_hole)
1451                         goto write_done;
1452
1453                 /* Find longest run of dirty blocks. */
1454                 for (next_offset = prev_offset; next_offset < max_offset;) {
1455                         m = ma[OFF_TO_IDX(next_offset - poffset)];
1456                         for (i = vn_off2bidx(next_offset);
1457                             i < sizeof(vm_page_bits_t) * NBBY &&
1458                             next_offset < max_offset; i++) {
1459                                 if (!vn_dirty_blk(m, next_offset))
1460                                         goto start_write;
1461                                 next_offset += DEV_BSIZE;
1462                         }
1463                 }
1464 start_write:
1465                 if (next_offset > poffset + maxsize)
1466                         next_offset = poffset + maxsize;
1467                 if (prev_offset == next_offset)
1468                         goto write_done;
1469
1470                 /*
1471                  * Getting here requires finding a dirty block in the
1472                  * 'skip clean blocks' loop.
1473                  */
1474
1475                 aiov.iov_base = NULL;
1476                 auio.uio_iovcnt = 1;
1477                 auio.uio_offset = prev_offset;
1478                 prev_resid = auio.uio_resid = aiov.iov_len = next_offset -
1479                     prev_offset;
1480                 error = VOP_WRITE(vp, &auio,
1481                     vnode_pager_putpages_ioflags(flags), curthread->td_ucred);
1482
1483                 wrsz = prev_resid - auio.uio_resid;
1484                 if (wrsz == 0) {
1485                         if (ppsratecheck(&lastfail, &curfail, 1) != 0) {
1486                                 vn_printf(vp, "vnode_pager_putpages: "
1487                                     "zero-length write at %ju resid %zd\n",
1488                                     auio.uio_offset, auio.uio_resid);
1489                         }
1490                         break;
1491                 }
1492
1493                 /* Adjust the starting offset for next iteration. */
1494                 prev_offset += wrsz;
1495                 MPASS(auio.uio_offset == prev_offset);
1496
1497                 ppscheck = 0;
1498                 if (error != 0 && (ppscheck = ppsratecheck(&lastfail,
1499                     &curfail, 1)) != 0)
1500                         vn_printf(vp, "vnode_pager_putpages: I/O error %d\n",
1501                             error);
1502                 if (auio.uio_resid != 0 && (ppscheck != 0 ||
1503                     ppsratecheck(&lastfail, &curfail, 1) != 0))
1504                         vn_printf(vp, "vnode_pager_putpages: residual I/O %zd "
1505                             "at %ju\n", auio.uio_resid,
1506                             (uintmax_t)ma[0]->pindex);
1507                 if (error != 0 || auio.uio_resid != 0)
1508                         break;
1509         }
1510 write_done:
1511         /* Mark completely processed pages. */
1512         for (i = 0; i < OFF_TO_IDX(prev_offset - poffset); i++)
1513                 rtvals[i] = VM_PAGER_OK;
1514         /* Mark partial EOF page. */
1515         if (prev_offset == poffset + maxsize && (prev_offset & PAGE_MASK) != 0)
1516                 rtvals[i++] = VM_PAGER_OK;
1517         /* Unwritten pages in range, free bonus if the page is clean. */
1518         for (; i < ncount; i++)
1519                 rtvals[i] = ma[i]->dirty == 0 ? VM_PAGER_OK : VM_PAGER_ERROR;
1520         VM_CNT_ADD(v_vnodepgsout, i);
1521         VM_CNT_INC(v_vnodeout);
1522         return (rtvals[0]);
1523 }
1524
1525 int
1526 vnode_pager_putpages_ioflags(int pager_flags)
1527 {
1528         int ioflags;
1529
1530         /*
1531          * Pageouts are already clustered, use IO_ASYNC to force a
1532          * bawrite() rather then a bdwrite() to prevent paging I/O
1533          * from saturating the buffer cache.  Dummy-up the sequential
1534          * heuristic to cause large ranges to cluster.  If neither
1535          * IO_SYNC or IO_ASYNC is set, the system decides how to
1536          * cluster.
1537          */
1538         ioflags = IO_VMIO;
1539         if ((pager_flags & (VM_PAGER_PUT_SYNC | VM_PAGER_PUT_INVAL)) != 0)
1540                 ioflags |= IO_SYNC;
1541         else if ((pager_flags & VM_PAGER_CLUSTER_OK) == 0)
1542                 ioflags |= IO_ASYNC;
1543         ioflags |= (pager_flags & VM_PAGER_PUT_INVAL) != 0 ? IO_INVAL: 0;
1544         ioflags |= (pager_flags & VM_PAGER_PUT_NOREUSE) != 0 ? IO_NOREUSE : 0;
1545         ioflags |= IO_SEQMAX << IO_SEQSHIFT;
1546         return (ioflags);
1547 }
1548
1549 /*
1550  * vnode_pager_undirty_pages().
1551  *
1552  * A helper to mark pages as clean after pageout that was possibly
1553  * done with a short write.  The lpos argument specifies the page run
1554  * length in bytes, and the written argument specifies how many bytes
1555  * were actually written.  eof is the offset past the last valid byte
1556  * in the vnode using the absolute file position of the first byte in
1557  * the run as the base from which it is computed.
1558  */
1559 void
1560 vnode_pager_undirty_pages(vm_page_t *ma, int *rtvals, int written, off_t eof,
1561     int lpos)
1562 {
1563         int i, pos, pos_devb;
1564
1565         if (written == 0 && eof >= lpos)
1566                 return;
1567         for (i = 0, pos = 0; pos < written; i++, pos += PAGE_SIZE) {
1568                 if (pos < trunc_page(written)) {
1569                         rtvals[i] = VM_PAGER_OK;
1570                         vm_page_undirty(ma[i]);
1571                 } else {
1572                         /* Partially written page. */
1573                         rtvals[i] = VM_PAGER_AGAIN;
1574                         vm_page_clear_dirty(ma[i], 0, written & PAGE_MASK);
1575                 }
1576         }
1577         if (eof >= lpos) /* avoid truncation */
1578                 return;
1579         for (pos = eof, i = OFF_TO_IDX(trunc_page(pos)); pos < lpos; i++) {
1580                 if (pos != trunc_page(pos)) {
1581                         /*
1582                          * The page contains the last valid byte in
1583                          * the vnode, mark the rest of the page as
1584                          * clean, potentially making the whole page
1585                          * clean.
1586                          */
1587                         pos_devb = roundup2(pos & PAGE_MASK, DEV_BSIZE);
1588                         vm_page_clear_dirty(ma[i], pos_devb, PAGE_SIZE -
1589                             pos_devb);
1590
1591                         /*
1592                          * If the page was cleaned, report the pageout
1593                          * on it as successful.  msync() no longer
1594                          * needs to write out the page, endlessly
1595                          * creating write requests and dirty buffers.
1596                          */
1597                         if (ma[i]->dirty == 0)
1598                                 rtvals[i] = VM_PAGER_OK;
1599
1600                         pos = round_page(pos);
1601                 } else {
1602                         /* vm_pageout_flush() clears dirty */
1603                         rtvals[i] = VM_PAGER_BAD;
1604                         pos += PAGE_SIZE;
1605                 }
1606         }
1607 }
1608
1609 static void
1610 vnode_pager_update_writecount(vm_object_t object, vm_offset_t start,
1611     vm_offset_t end)
1612 {
1613         struct vnode *vp;
1614         vm_ooffset_t old_wm;
1615
1616         VM_OBJECT_WLOCK(object);
1617         if (object->type != OBJT_VNODE) {
1618                 VM_OBJECT_WUNLOCK(object);
1619                 return;
1620         }
1621         old_wm = object->un_pager.vnp.writemappings;
1622         object->un_pager.vnp.writemappings += (vm_ooffset_t)end - start;
1623         vp = object->handle;
1624         if (old_wm == 0 && object->un_pager.vnp.writemappings != 0) {
1625                 ASSERT_VOP_LOCKED(vp, "v_writecount inc");
1626                 VOP_ADD_WRITECOUNT_CHECKED(vp, 1);
1627                 CTR3(KTR_VFS, "%s: vp %p v_writecount increased to %d",
1628                     __func__, vp, vp->v_writecount);
1629         } else if (old_wm != 0 && object->un_pager.vnp.writemappings == 0) {
1630                 ASSERT_VOP_LOCKED(vp, "v_writecount dec");
1631                 VOP_ADD_WRITECOUNT_CHECKED(vp, -1);
1632                 CTR3(KTR_VFS, "%s: vp %p v_writecount decreased to %d",
1633                     __func__, vp, vp->v_writecount);
1634         }
1635         VM_OBJECT_WUNLOCK(object);
1636 }
1637
1638 static void
1639 vnode_pager_release_writecount(vm_object_t object, vm_offset_t start,
1640     vm_offset_t end)
1641 {
1642         struct vnode *vp;
1643         struct mount *mp;
1644         vm_offset_t inc;
1645
1646         VM_OBJECT_WLOCK(object);
1647
1648         /*
1649          * First, recheck the object type to account for the race when
1650          * the vnode is reclaimed.
1651          */
1652         if (object->type != OBJT_VNODE) {
1653                 VM_OBJECT_WUNLOCK(object);
1654                 return;
1655         }
1656
1657         /*
1658          * Optimize for the case when writemappings is not going to
1659          * zero.
1660          */
1661         inc = end - start;
1662         if (object->un_pager.vnp.writemappings != inc) {
1663                 object->un_pager.vnp.writemappings -= inc;
1664                 VM_OBJECT_WUNLOCK(object);
1665                 return;
1666         }
1667
1668         vp = object->handle;
1669         vhold(vp);
1670         VM_OBJECT_WUNLOCK(object);
1671         mp = NULL;
1672         vn_start_write(vp, &mp, V_WAIT);
1673         vn_lock(vp, LK_SHARED | LK_RETRY);
1674
1675         /*
1676          * Decrement the object's writemappings, by swapping the start
1677          * and end arguments for vnode_pager_update_writecount().  If
1678          * there was not a race with vnode reclaimation, then the
1679          * vnode's v_writecount is decremented.
1680          */
1681         vnode_pager_update_writecount(object, end, start);
1682         VOP_UNLOCK(vp);
1683         vdrop(vp);
1684         if (mp != NULL)
1685                 vn_finished_write(mp);
1686 }
1687
1688 static void
1689 vnode_pager_getvp(vm_object_t object, struct vnode **vpp, bool *vp_heldp)
1690 {
1691         *vpp = object->handle;
1692 }
1693
1694 static void
1695 vnode_pager_clean1(struct vnode *vp, int sync_flags)
1696 {
1697         struct vm_object *obj;
1698
1699         ASSERT_VOP_LOCKED(vp, "needs lock for writes");
1700         obj = vp->v_object;
1701         if (obj == NULL)
1702                 return;
1703
1704         VM_OBJECT_WLOCK(obj);
1705         vm_object_page_clean(obj, 0, 0, sync_flags);
1706         VM_OBJECT_WUNLOCK(obj);
1707 }
1708
1709 void
1710 vnode_pager_clean_sync(struct vnode *vp)
1711 {
1712         vnode_pager_clean1(vp, OBJPC_SYNC);
1713 }
1714
1715 void
1716 vnode_pager_clean_async(struct vnode *vp)
1717 {
1718         vnode_pager_clean1(vp, 0);
1719 }