sys/arm64/arm64/pmap.c

   1 /*-
   2  * Copyright (c) 1991 Regents of the University of California.
   3  * All rights reserved.
   4  * Copyright (c) 1994 John S. Dyson
   5  * All rights reserved.
   6  * Copyright (c) 1994 David Greenman
   7  * All rights reserved.
   8  * Copyright (c) 2003 Peter Wemm
   9  * All rights reserved.
  10  * Copyright (c) 2005-2010 Alan L. Cox <alc@cs.rice.edu>
  11  * All rights reserved.
  12  * Copyright (c) 2014 Andrew Turner
  13  * All rights reserved.
  14  * Copyright (c) 2014-2016 The FreeBSD Foundation
  15  * All rights reserved.
  16  *
  17  * This code is derived from software contributed to Berkeley by
  18  * the Systems Programming Group of the University of Utah Computer
  19  * Science Department and William Jolitz of UUNET Technologies Inc.
  20  *
  21  * This software was developed by Andrew Turner under sponsorship from
  22  * the FreeBSD Foundation.
  23  *
  24  * Redistribution and use in source and binary forms, with or without
  25  * modification, are permitted provided that the following conditions
  26  * are met:
  27  * 1. Redistributions of source code must retain the above copyright
  28  *    notice, this list of conditions and the following disclaimer.
  29  * 2. Redistributions in binary form must reproduce the above copyright
  30  *    notice, this list of conditions and the following disclaimer in the
  31  *    documentation and/or other materials provided with the distribution.
  32  * 3. All advertising materials mentioning features or use of this software
  33  *    must display the following acknowledgement:
  34  *      This product includes software developed by the University of
  35  *      California, Berkeley and its contributors.
  36  * 4. Neither the name of the University nor the names of its contributors
  37  *    may be used to endorse or promote products derived from this software
  38  *    without specific prior written permission.
  39  *
  40  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  41  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  42  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  43  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  44  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  45  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  46  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  47  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  48  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  49  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  50  * SUCH DAMAGE.
  51  *
  52  *      from:   @(#)pmap.c      7.7 (Berkeley)  5/12/91
  53  */
  54 /*-
  55  * Copyright (c) 2003 Networks Associates Technology, Inc.
  56  * All rights reserved.
  57  *
  58  * This software was developed for the FreeBSD Project by Jake Burkholder,
  59  * Safeport Network Services, and Network Associates Laboratories, the
  60  * Security Research Division of Network Associates, Inc. under
  61  * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA
  62  * CHATS research program.
  63  *
  64  * Redistribution and use in source and binary forms, with or without
  65  * modification, are permitted provided that the following conditions
  66  * are met:
  67  * 1. Redistributions of source code must retain the above copyright
  68  *    notice, this list of conditions and the following disclaimer.
  69  * 2. Redistributions in binary form must reproduce the above copyright
  70  *    notice, this list of conditions and the following disclaimer in the
  71  *    documentation and/or other materials provided with the distribution.
  72  *
  73  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  74  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  75  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  76  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  77  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  78  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  79  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  80  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  81  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  82  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  83  * SUCH DAMAGE.
  84  */
  85
  86 #include <sys/cdefs.h>
  87 __FBSDID("$FreeBSD$");
  88
  89 /*
  90  *      Manages physical address maps.
  91  *
  92  *      Since the information managed by this module is
  93  *      also stored by the logical address mapping module,
  94  *      this module may throw away valid virtual-to-physical
  95  *      mappings at almost any time.  However, invalidations
  96  *      of virtual-to-physical mappings must be done as
  97  *      requested.
  98  *
  99  *      In order to cope with hardware architectures which
 100  *      make virtual-to-physical map invalidates expensive,
 101  *      this module may delay invalidate or reduced protection
 102  *      operations until such time as they are actually
 103  *      necessary.  This module is given full information as
 104  *      to which processors are currently using which maps,
 105  *      and to when physical maps must be made correct.
 106  */
 107
 108 #include "opt_vm.h"
 109
 110 #include <sys/param.h>
 111 #include <sys/bitstring.h>
 112 #include <sys/bus.h>
 113 #include <sys/systm.h>
 114 #include <sys/kernel.h>
 115 #include <sys/ktr.h>
 116 #include <sys/lock.h>
 117 #include <sys/malloc.h>
 118 #include <sys/mman.h>
 119 #include <sys/msgbuf.h>
 120 #include <sys/mutex.h>
 121 #include <sys/proc.h>
 122 #include <sys/rwlock.h>
 123 #include <sys/sx.h>
 124 #include <sys/vmem.h>
 125 #include <sys/vmmeter.h>
 126 #include <sys/sched.h>
 127 #include <sys/sysctl.h>
 128 #include <sys/_unrhdr.h>
 129 #include <sys/smp.h>
 130
 131 #include <vm/vm.h>
 132 #include <vm/vm_param.h>
 133 #include <vm/vm_kern.h>
 134 #include <vm/vm_page.h>
 135 #include <vm/vm_map.h>
 136 #include <vm/vm_object.h>
 137 #include <vm/vm_extern.h>
 138 #include <vm/vm_pageout.h>
 139 #include <vm/vm_pager.h>
 140 #include <vm/vm_phys.h>
 141 #include <vm/vm_radix.h>
 142 #include <vm/vm_reserv.h>
 143 #include <vm/uma.h>
 144
 145 #include <machine/machdep.h>
 146 #include <machine/md_var.h>
 147 #include <machine/pcb.h>
 148
 149 #include <arm/include/physmem.h>
 150
 151 #define NL0PG           (PAGE_SIZE/(sizeof (pd_entry_t)))
 152 #define NL1PG           (PAGE_SIZE/(sizeof (pd_entry_t)))
 153 #define NL2PG           (PAGE_SIZE/(sizeof (pd_entry_t)))
 154 #define NL3PG           (PAGE_SIZE/(sizeof (pt_entry_t)))
 155
 156 #define NUL0E           L0_ENTRIES
 157 #define NUL1E           (NUL0E * NL1PG)
 158 #define NUL2E           (NUL1E * NL2PG)
 159
 160 #if !defined(DIAGNOSTIC)
 161 #ifdef __GNUC_GNU_INLINE__
 162 #define PMAP_INLINE     __attribute__((__gnu_inline__)) inline
 163 #else
 164 #define PMAP_INLINE     extern inline
 165 #endif
 166 #else
 167 #define PMAP_INLINE
 168 #endif
 169
 170 /*
 171  * These are configured by the mair_el1 register. This is set up in locore.S
 172  */
 173 #define DEVICE_MEMORY   0
 174 #define UNCACHED_MEMORY 1
 175 #define CACHED_MEMORY   2
 176
 177
 178 #ifdef PV_STATS
 179 #define PV_STAT(x)      do { x ; } while (0)
 180 #else
 181 #define PV_STAT(x)      do { } while (0)
 182 #endif
 183
 184 #define pmap_l2_pindex(v)       ((v) >> L2_SHIFT)
 185 #define pa_to_pvh(pa)           (&pv_table[pmap_l2_pindex(pa)])
 186
 187 #define NPV_LIST_LOCKS  MAXCPU
 188
 189 #define PHYS_TO_PV_LIST_LOCK(pa)        \
 190                         (&pv_list_locks[pa_index(pa) % NPV_LIST_LOCKS])
 191
 192 #define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa)  do {    \
 193         struct rwlock **_lockp = (lockp);               \
 194         struct rwlock *_new_lock;                       \
 195                                                         \
 196         _new_lock = PHYS_TO_PV_LIST_LOCK(pa);           \
 197         if (_new_lock != *_lockp) {                     \
 198                 if (*_lockp != NULL)                    \
 199                         rw_wunlock(*_lockp);            \
 200                 *_lockp = _new_lock;                    \
 201                 rw_wlock(*_lockp);                      \
 202         }                                               \
 203 } while (0)
 204
 205 #define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m)        \
 206                         CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, VM_PAGE_TO_PHYS(m))
 207
 208 #define RELEASE_PV_LIST_LOCK(lockp)             do {    \
 209         struct rwlock **_lockp = (lockp);               \
 210                                                         \
 211         if (*_lockp != NULL) {                          \
 212                 rw_wunlock(*_lockp);                    \
 213                 *_lockp = NULL;                         \
 214         }                                               \
 215 } while (0)
 216
 217 #define VM_PAGE_TO_PV_LIST_LOCK(m)      \
 218                         PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m))
 219
 220 struct pmap kernel_pmap_store;
 221
 222 /* Used for mapping ACPI memory before VM is initialized */
 223 #define PMAP_PREINIT_MAPPING_COUNT      32
 224 #define PMAP_PREINIT_MAPPING_SIZE       (PMAP_PREINIT_MAPPING_COUNT * L2_SIZE)
 225 static vm_offset_t preinit_map_va;      /* Start VA of pre-init mapping space */
 226 static int vm_initialized = 0;          /* No need to use pre-init maps when set */
 227
 228 /*
 229  * Reserve a few L2 blocks starting from 'preinit_map_va' pointer.
 230  * Always map entire L2 block for simplicity.
 231  * VA of L2 block = preinit_map_va + i * L2_SIZE
 232  */
 233 static struct pmap_preinit_mapping {
 234         vm_paddr_t      pa;
 235         vm_offset_t     va;
 236         vm_size_t       size;
 237 } pmap_preinit_mapping[PMAP_PREINIT_MAPPING_COUNT];
 238
 239 vm_offset_t virtual_avail;      /* VA of first avail page (after kernel bss) */
 240 vm_offset_t virtual_end;        /* VA of last avail page (end of kernel AS) */
 241 vm_offset_t kernel_vm_end = 0;
 242
 243 /*
 244  * Data for the pv entry allocation mechanism.
 245  */
 246 static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks);
 247 static struct mtx pv_chunks_mutex;
 248 static struct rwlock pv_list_locks[NPV_LIST_LOCKS];
 249 static struct md_page *pv_table;
 250 static struct md_page pv_dummy;
 251
 252 vm_paddr_t dmap_phys_base;      /* The start of the dmap region */
 253 vm_paddr_t dmap_phys_max;       /* The limit of the dmap region */
 254 vm_offset_t dmap_max_addr;      /* The virtual address limit of the dmap */
 255
 256 /* This code assumes all L1 DMAP entries will be used */
 257 CTASSERT((DMAP_MIN_ADDRESS  & ~L0_OFFSET) == DMAP_MIN_ADDRESS);
 258 CTASSERT((DMAP_MAX_ADDRESS  & ~L0_OFFSET) == DMAP_MAX_ADDRESS);
 259
 260 #define DMAP_TABLES     ((DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) >> L0_SHIFT)
 261 extern pt_entry_t pagetable_dmap[];
 262
 263 #define PHYSMAP_SIZE    (2 * (VM_PHYSSEG_MAX - 1))
 264 static vm_paddr_t physmap[PHYSMAP_SIZE];
 265 static u_int physmap_idx;
 266
 267 static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");
 268
 269 static int superpages_enabled = 1;
 270 SYSCTL_INT(_vm_pmap, OID_AUTO, superpages_enabled,
 271     CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &superpages_enabled, 0,
 272     "Are large page mappings enabled?");
 273
 274 /*
 275  * Internal flags for pmap_enter()'s helper functions.
 276  */
 277 #define PMAP_ENTER_NORECLAIM    0x1000000       /* Don't reclaim PV entries. */
 278 #define PMAP_ENTER_NOREPLACE    0x2000000       /* Don't replace mappings. */
 279
 280 static void     free_pv_chunk(struct pv_chunk *pc);
 281 static void     free_pv_entry(pmap_t pmap, pv_entry_t pv);
 282 static pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp);
 283 static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp);
 284 static void     pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);
 285 static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,
 286                     vm_offset_t va);
 287
 288 static int pmap_change_attr(vm_offset_t va, vm_size_t size, int mode);
 289 static int pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode);
 290 static pt_entry_t *pmap_demote_l1(pmap_t pmap, pt_entry_t *l1, vm_offset_t va);
 291 static pt_entry_t *pmap_demote_l2_locked(pmap_t pmap, pt_entry_t *l2,
 292     vm_offset_t va, struct rwlock **lockp);
 293 static pt_entry_t *pmap_demote_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t va);
 294 static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
 295     vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp);
 296 static int pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t new_l2,
 297     u_int flags, vm_page_t m, struct rwlock **lockp);
 298 static int pmap_remove_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t sva,
 299     pd_entry_t l1e, struct spglist *free, struct rwlock **lockp);
 300 static int pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t sva,
 301     pd_entry_t l2e, struct spglist *free, struct rwlock **lockp);
 302 static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,
 303     vm_page_t m, struct rwlock **lockp);
 304
 305 static vm_page_t _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex,
 306                 struct rwlock **lockp);
 307
 308 static void _pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m,
 309     struct spglist *free);
 310 static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, struct spglist *);
 311 static __inline vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va);
 312
 313 /*
 314  * These load the old table data and store the new value.
 315  * They need to be atomic as the System MMU may write to the table at
 316  * the same time as the CPU.
 317  */
 318 #define pmap_load_store(table, entry) atomic_swap_64(table, entry)
 319 #define pmap_set(table, mask) atomic_set_64(table, mask)
 320 #define pmap_load_clear(table) atomic_swap_64(table, 0)
 321 #define pmap_load(table) (*table)
 322
 323 /********************/
 324 /* Inline functions */
 325 /********************/
 326
 327 static __inline void
 328 pagecopy(void *s, void *d)
 329 {
 330
 331         memcpy(d, s, PAGE_SIZE);
 332 }
 333
 334 static __inline pd_entry_t *
 335 pmap_l0(pmap_t pmap, vm_offset_t va)
 336 {
 337
 338         return (&pmap->pm_l0[pmap_l0_index(va)]);
 339 }
 340
 341 static __inline pd_entry_t *
 342 pmap_l0_to_l1(pd_entry_t *l0, vm_offset_t va)
 343 {
 344         pd_entry_t *l1;
 345
 346         l1 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l0) & ~ATTR_MASK);
 347         return (&l1[pmap_l1_index(va)]);
 348 }
 349
 350 static __inline pd_entry_t *
 351 pmap_l1(pmap_t pmap, vm_offset_t va)
 352 {
 353         pd_entry_t *l0;
 354
 355         l0 = pmap_l0(pmap, va);
 356         if ((pmap_load(l0) & ATTR_DESCR_MASK) != L0_TABLE)
 357                 return (NULL);
 358
 359         return (pmap_l0_to_l1(l0, va));
 360 }
 361
 362 static __inline pd_entry_t *
 363 pmap_l1_to_l2(pd_entry_t *l1, vm_offset_t va)
 364 {
 365         pd_entry_t *l2;
 366
 367         l2 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l1) & ~ATTR_MASK);
 368         return (&l2[pmap_l2_index(va)]);
 369 }
 370
 371 static __inline pd_entry_t *
 372 pmap_l2(pmap_t pmap, vm_offset_t va)
 373 {
 374         pd_entry_t *l1;
 375
 376         l1 = pmap_l1(pmap, va);
 377         if ((pmap_load(l1) & ATTR_DESCR_MASK) != L1_TABLE)
 378                 return (NULL);
 379
 380         return (pmap_l1_to_l2(l1, va));
 381 }
 382
 383 static __inline pt_entry_t *
 384 pmap_l2_to_l3(pd_entry_t *l2, vm_offset_t va)
 385 {
 386         pt_entry_t *l3;
 387
 388         l3 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l2) & ~ATTR_MASK);
 389         return (&l3[pmap_l3_index(va)]);
 390 }
 391
 392 /*
 393  * Returns the lowest valid pde for a given virtual address.
 394  * The next level may or may not point to a valid page or block.
 395  */
 396 static __inline pd_entry_t *
 397 pmap_pde(pmap_t pmap, vm_offset_t va, int *level)
 398 {
 399         pd_entry_t *l0, *l1, *l2, desc;
 400
 401         l0 = pmap_l0(pmap, va);
 402         desc = pmap_load(l0) & ATTR_DESCR_MASK;
 403         if (desc != L0_TABLE) {
 404                 *level = -1;
 405                 return (NULL);
 406         }
 407
 408         l1 = pmap_l0_to_l1(l0, va);
 409         desc = pmap_load(l1) & ATTR_DESCR_MASK;
 410         if (desc != L1_TABLE) {
 411                 *level = 0;
 412                 return (l0);
 413         }
 414
 415         l2 = pmap_l1_to_l2(l1, va);
 416         desc = pmap_load(l2) & ATTR_DESCR_MASK;
 417         if (desc != L2_TABLE) {
 418                 *level = 1;
 419                 return (l1);
 420         }
 421
 422         *level = 2;
 423         return (l2);
 424 }
 425
 426 /*
 427  * Returns the lowest valid pte block or table entry for a given virtual
 428  * address. If there are no valid entries return NULL and set the level to
 429  * the first invalid level.
 430  */
 431 static __inline pt_entry_t *
 432 pmap_pte(pmap_t pmap, vm_offset_t va, int *level)
 433 {
 434         pd_entry_t *l1, *l2, desc;
 435         pt_entry_t *l3;
 436
 437         l1 = pmap_l1(pmap, va);
 438         if (l1 == NULL) {
 439                 *level = 0;
 440                 return (NULL);
 441         }
 442         desc = pmap_load(l1) & ATTR_DESCR_MASK;
 443         if (desc == L1_BLOCK) {
 444                 *level = 1;
 445                 return (l1);
 446         }
 447
 448         if (desc != L1_TABLE) {
 449                 *level = 1;
 450                 return (NULL);
 451         }
 452
 453         l2 = pmap_l1_to_l2(l1, va);
 454         desc = pmap_load(l2) & ATTR_DESCR_MASK;
 455         if (desc == L2_BLOCK) {
 456                 *level = 2;
 457                 return (l2);
 458         }
 459
 460         if (desc != L2_TABLE) {
 461                 *level = 2;
 462                 return (NULL);
 463         }
 464
 465         *level = 3;
 466         l3 = pmap_l2_to_l3(l2, va);
 467         if ((pmap_load(l3) & ATTR_DESCR_MASK) != L3_PAGE)
 468                 return (NULL);
 469
 470         return (l3);
 471 }
 472
 473 bool
 474 pmap_ps_enabled(pmap_t pmap __unused)
 475 {
 476
 477         return (superpages_enabled != 0);
 478 }
 479
 480 bool
 481 pmap_get_tables(pmap_t pmap, vm_offset_t va, pd_entry_t **l0, pd_entry_t **l1,
 482     pd_entry_t **l2, pt_entry_t **l3)
 483 {
 484         pd_entry_t *l0p, *l1p, *l2p;
 485
 486         if (pmap->pm_l0 == NULL)
 487                 return (false);
 488
 489         l0p = pmap_l0(pmap, va);
 490         *l0 = l0p;
 491
 492         if ((pmap_load(l0p) & ATTR_DESCR_MASK) != L0_TABLE)
 493                 return (false);
 494
 495         l1p = pmap_l0_to_l1(l0p, va);
 496         *l1 = l1p;
 497
 498         if ((pmap_load(l1p) & ATTR_DESCR_MASK) == L1_BLOCK) {
 499                 *l2 = NULL;
 500                 *l3 = NULL;
 501                 return (true);
 502         }
 503
 504         if ((pmap_load(l1p) & ATTR_DESCR_MASK) != L1_TABLE)
 505                 return (false);
 506
 507         l2p = pmap_l1_to_l2(l1p, va);
 508         *l2 = l2p;
 509
 510         if ((pmap_load(l2p) & ATTR_DESCR_MASK) == L2_BLOCK) {
 511                 *l3 = NULL;
 512                 return (true);
 513         }
 514
 515         if ((pmap_load(l2p) & ATTR_DESCR_MASK) != L2_TABLE)
 516                 return (false);
 517
 518         *l3 = pmap_l2_to_l3(l2p, va);
 519
 520         return (true);
 521 }
 522
 523 static __inline int
 524 pmap_l3_valid(pt_entry_t l3)
 525 {
 526
 527         return ((l3 & ATTR_DESCR_MASK) == L3_PAGE);
 528 }
 529
 530
 531 CTASSERT(L1_BLOCK == L2_BLOCK);
 532
 533 /*
 534  * Checks if the page is dirty. We currently lack proper tracking of this on
 535  * arm64 so for now assume is a page mapped as rw was accessed it is.
 536  */
 537 static inline int
 538 pmap_page_dirty(pt_entry_t pte)
 539 {
 540
 541         return ((pte & (ATTR_AF | ATTR_AP_RW_BIT)) ==
 542             (ATTR_AF | ATTR_AP(ATTR_AP_RW)));
 543 }
 544
 545 static __inline void
 546 pmap_resident_count_inc(pmap_t pmap, int count)
 547 {
 548
 549         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 550         pmap->pm_stats.resident_count += count;
 551 }
 552
 553 static __inline void
 554 pmap_resident_count_dec(pmap_t pmap, int count)
 555 {
 556
 557         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 558         KASSERT(pmap->pm_stats.resident_count >= count,
 559             ("pmap %p resident count underflow %ld %d", pmap,
 560             pmap->pm_stats.resident_count, count));
 561         pmap->pm_stats.resident_count -= count;
 562 }
 563
 564 static pt_entry_t *
 565 pmap_early_page_idx(vm_offset_t l1pt, vm_offset_t va, u_int *l1_slot,
 566     u_int *l2_slot)
 567 {
 568         pt_entry_t *l2;
 569         pd_entry_t *l1;
 570
 571         l1 = (pd_entry_t *)l1pt;
 572         *l1_slot = (va >> L1_SHIFT) & Ln_ADDR_MASK;
 573
 574         /* Check locore has used a table L1 map */
 575         KASSERT((l1[*l1_slot] & ATTR_DESCR_MASK) == L1_TABLE,
 576            ("Invalid bootstrap L1 table"));
 577         /* Find the address of the L2 table */
 578         l2 = (pt_entry_t *)init_pt_va;
 579         *l2_slot = pmap_l2_index(va);
 580
 581         return (l2);
 582 }
 583
 584 static vm_paddr_t
 585 pmap_early_vtophys(vm_offset_t l1pt, vm_offset_t va)
 586 {
 587         u_int l1_slot, l2_slot;
 588         pt_entry_t *l2;
 589
 590         l2 = pmap_early_page_idx(l1pt, va, &l1_slot, &l2_slot);
 591
 592         return ((l2[l2_slot] & ~ATTR_MASK) + (va & L2_OFFSET));
 593 }
 594
 595 static vm_offset_t
 596 pmap_bootstrap_dmap(vm_offset_t kern_l1, vm_paddr_t min_pa,
 597     vm_offset_t freemempos)
 598 {
 599         pt_entry_t *l2;
 600         vm_offset_t va;
 601         vm_paddr_t l2_pa, pa;
 602         u_int l1_slot, l2_slot, prev_l1_slot;
 603         int i;
 604
 605         dmap_phys_base = min_pa & ~L1_OFFSET;
 606         dmap_phys_max = 0;
 607         dmap_max_addr = 0;
 608         l2 = NULL;
 609         prev_l1_slot = -1;
 610
 611 #define DMAP_TABLES     ((DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) >> L0_SHIFT)
 612         memset(pagetable_dmap, 0, PAGE_SIZE * DMAP_TABLES);
 613
 614         for (i = 0; i < (physmap_idx * 2); i += 2) {
 615                 pa = physmap[i] & ~L2_OFFSET;
 616                 va = pa - dmap_phys_base + DMAP_MIN_ADDRESS;
 617
 618                 /* Create L2 mappings at the start of the region */
 619                 if ((pa & L1_OFFSET) != 0) {
 620                         l1_slot = ((va - DMAP_MIN_ADDRESS) >> L1_SHIFT);
 621                         if (l1_slot != prev_l1_slot) {
 622                                 prev_l1_slot = l1_slot;
 623                                 l2 = (pt_entry_t *)freemempos;
 624                                 l2_pa = pmap_early_vtophys(kern_l1,
 625                                     (vm_offset_t)l2);
 626                                 freemempos += PAGE_SIZE;
 627
 628                                 pmap_load_store(&pagetable_dmap[l1_slot],
 629                                     (l2_pa & ~Ln_TABLE_MASK) | L1_TABLE);
 630
 631                                 memset(l2, 0, PAGE_SIZE);
 632                         }
 633                         KASSERT(l2 != NULL,
 634                             ("pmap_bootstrap_dmap: NULL l2 map"));
 635                         for (; va < DMAP_MAX_ADDRESS && pa < physmap[i + 1];
 636                             pa += L2_SIZE, va += L2_SIZE) {
 637                                 /*
 638                                  * We are on a boundary, stop to
 639                                  * create a level 1 block
 640                                  */
 641                                 if ((pa & L1_OFFSET) == 0)
 642                                         break;
 643
 644                                 l2_slot = pmap_l2_index(va);
 645                                 KASSERT(l2_slot != 0, ("..."));
 646                                 pmap_load_store(&l2[l2_slot],
 647                                     (pa & ~L2_OFFSET) | ATTR_DEFAULT | ATTR_XN |
 648                                     ATTR_IDX(CACHED_MEMORY) | L2_BLOCK);
 649                         }
 650                         KASSERT(va == (pa - dmap_phys_base + DMAP_MIN_ADDRESS),
 651                             ("..."));
 652                 }
 653
 654                 for (; va < DMAP_MAX_ADDRESS && pa < physmap[i + 1] &&
 655                     (physmap[i + 1] - pa) >= L1_SIZE;
 656                     pa += L1_SIZE, va += L1_SIZE) {
 657                         l1_slot = ((va - DMAP_MIN_ADDRESS) >> L1_SHIFT);
 658                         pmap_load_store(&pagetable_dmap[l1_slot],
 659                             (pa & ~L1_OFFSET) | ATTR_DEFAULT | ATTR_XN |
 660                             ATTR_IDX(CACHED_MEMORY) | L1_BLOCK);
 661                 }
 662
 663                 /* Create L2 mappings at the end of the region */
 664                 if (pa < physmap[i + 1]) {
 665                         l1_slot = ((va - DMAP_MIN_ADDRESS) >> L1_SHIFT);
 666                         if (l1_slot != prev_l1_slot) {
 667                                 prev_l1_slot = l1_slot;
 668                                 l2 = (pt_entry_t *)freemempos;
 669                                 l2_pa = pmap_early_vtophys(kern_l1,
 670                                     (vm_offset_t)l2);
 671                                 freemempos += PAGE_SIZE;
 672
 673                                 pmap_load_store(&pagetable_dmap[l1_slot],
 674                                     (l2_pa & ~Ln_TABLE_MASK) | L1_TABLE);
 675
 676                                 memset(l2, 0, PAGE_SIZE);
 677                         }
 678                         KASSERT(l2 != NULL,
 679                             ("pmap_bootstrap_dmap: NULL l2 map"));
 680                         for (; va < DMAP_MAX_ADDRESS && pa < physmap[i + 1];
 681                             pa += L2_SIZE, va += L2_SIZE) {
 682                                 l2_slot = pmap_l2_index(va);
 683                                 pmap_load_store(&l2[l2_slot],
 684                                     (pa & ~L2_OFFSET) | ATTR_DEFAULT | ATTR_XN |
 685                                     ATTR_IDX(CACHED_MEMORY) | L2_BLOCK);
 686                         }
 687                 }
 688
 689                 if (pa > dmap_phys_max) {
 690                         dmap_phys_max = pa;
 691                         dmap_max_addr = va;
 692                 }
 693         }
 694
 695         cpu_tlb_flushID();
 696
 697         return (freemempos);
 698 }
 699
 700 static vm_offset_t
 701 pmap_bootstrap_l2(vm_offset_t l1pt, vm_offset_t va, vm_offset_t l2_start)
 702 {
 703         vm_offset_t l2pt;
 704         vm_paddr_t pa;
 705         pd_entry_t *l1;
 706         u_int l1_slot;
 707
 708         KASSERT((va & L1_OFFSET) == 0, ("Invalid virtual address"));
 709
 710         l1 = (pd_entry_t *)l1pt;
 711         l1_slot = pmap_l1_index(va);
 712         l2pt = l2_start;
 713
 714         for (; va < VM_MAX_KERNEL_ADDRESS; l1_slot++, va += L1_SIZE) {
 715                 KASSERT(l1_slot < Ln_ENTRIES, ("Invalid L1 index"));
 716
 717                 pa = pmap_early_vtophys(l1pt, l2pt);
 718                 pmap_load_store(&l1[l1_slot],
 719                     (pa & ~Ln_TABLE_MASK) | L1_TABLE);
 720                 l2pt += PAGE_SIZE;
 721         }
 722
 723         /* Clean the L2 page table */
 724         memset((void *)l2_start, 0, l2pt - l2_start);
 725
 726         return l2pt;
 727 }
 728
 729 static vm_offset_t
 730 pmap_bootstrap_l3(vm_offset_t l1pt, vm_offset_t va, vm_offset_t l3_start)
 731 {
 732         vm_offset_t l3pt;
 733         vm_paddr_t pa;
 734         pd_entry_t *l2;
 735         u_int l2_slot;
 736
 737         KASSERT((va & L2_OFFSET) == 0, ("Invalid virtual address"));
 738
 739         l2 = pmap_l2(kernel_pmap, va);
 740         l2 = (pd_entry_t *)rounddown2((uintptr_t)l2, PAGE_SIZE);
 741         l2_slot = pmap_l2_index(va);
 742         l3pt = l3_start;
 743
 744         for (; va < VM_MAX_KERNEL_ADDRESS; l2_slot++, va += L2_SIZE) {
 745                 KASSERT(l2_slot < Ln_ENTRIES, ("Invalid L2 index"));
 746
 747                 pa = pmap_early_vtophys(l1pt, l3pt);
 748                 pmap_load_store(&l2[l2_slot],
 749                     (pa & ~Ln_TABLE_MASK) | L2_TABLE);
 750                 l3pt += PAGE_SIZE;
 751         }
 752
 753         /* Clean the L2 page table */
 754         memset((void *)l3_start, 0, l3pt - l3_start);
 755
 756         return l3pt;
 757 }
 758
 759 /*
 760  *      Bootstrap the system enough to run with virtual memory.
 761  */
 762 void
 763 pmap_bootstrap(vm_offset_t l0pt, vm_offset_t l1pt, vm_paddr_t kernstart,
 764     vm_size_t kernlen)
 765 {
 766         u_int l1_slot, l2_slot;
 767         uint64_t kern_delta;
 768         pt_entry_t *l2;
 769         vm_offset_t va, freemempos;
 770         vm_offset_t dpcpu, msgbufpv;
 771         vm_paddr_t start_pa, pa, min_pa;
 772         int i;
 773
 774         kern_delta = KERNBASE - kernstart;
 775
 776         printf("pmap_bootstrap %lx %lx %lx\n", l1pt, kernstart, kernlen);
 777         printf("%lx\n", l1pt);
 778         printf("%lx\n", (KERNBASE >> L1_SHIFT) & Ln_ADDR_MASK);
 779
 780         /* Set this early so we can use the pagetable walking functions */
 781         kernel_pmap_store.pm_l0 = (pd_entry_t *)l0pt;
 782         PMAP_LOCK_INIT(kernel_pmap);
 783
 784         /* Assume the address we were loaded to is a valid physical address */
 785         min_pa = KERNBASE - kern_delta;
 786
 787         physmap_idx = arm_physmem_avail(physmap, nitems(physmap));
 788         physmap_idx /= 2;
 789
 790         /*
 791          * Find the minimum physical address. physmap is sorted,
 792          * but may contain empty ranges.
 793          */
 794         for (i = 0; i < (physmap_idx * 2); i += 2) {
 795                 if (physmap[i] == physmap[i + 1])
 796                         continue;
 797                 if (physmap[i] <= min_pa)
 798                         min_pa = physmap[i];
 799         }
 800
 801         freemempos = KERNBASE + kernlen;
 802         freemempos = roundup2(freemempos, PAGE_SIZE);
 803
 804         /* Create a direct map region early so we can use it for pa -> va */
 805         freemempos = pmap_bootstrap_dmap(l1pt, min_pa, freemempos);
 806
 807         va = KERNBASE;
 808         start_pa = pa = KERNBASE - kern_delta;
 809
 810         /*
 811          * Read the page table to find out what is already mapped.
 812          * This assumes we have mapped a block of memory from KERNBASE
 813          * using a single L1 entry.
 814          */
 815         l2 = pmap_early_page_idx(l1pt, KERNBASE, &l1_slot, &l2_slot);
 816
 817         /* Sanity check the index, KERNBASE should be the first VA */
 818         KASSERT(l2_slot == 0, ("The L2 index is non-zero"));
 819
 820         /* Find how many pages we have mapped */
 821         for (; l2_slot < Ln_ENTRIES; l2_slot++) {
 822                 if ((l2[l2_slot] & ATTR_DESCR_MASK) == 0)
 823                         break;
 824
 825                 /* Check locore used L2 blocks */
 826                 KASSERT((l2[l2_slot] & ATTR_DESCR_MASK) == L2_BLOCK,
 827                     ("Invalid bootstrap L2 table"));
 828                 KASSERT((l2[l2_slot] & ~ATTR_MASK) == pa,
 829                     ("Incorrect PA in L2 table"));
 830
 831                 va += L2_SIZE;
 832                 pa += L2_SIZE;
 833         }
 834
 835         va = roundup2(va, L1_SIZE);
 836
 837         /* Create the l2 tables up to VM_MAX_KERNEL_ADDRESS */
 838         freemempos = pmap_bootstrap_l2(l1pt, va, freemempos);
 839         /* And the l3 tables for the early devmap */
 840         freemempos = pmap_bootstrap_l3(l1pt,
 841             VM_MAX_KERNEL_ADDRESS - (PMAP_MAPDEV_EARLY_SIZE), freemempos);
 842
 843         cpu_tlb_flushID();
 844
 845 #define alloc_pages(var, np)                                            \
 846         (var) = freemempos;                                             \
 847         freemempos += (np * PAGE_SIZE);                                 \
 848         memset((char *)(var), 0, ((np) * PAGE_SIZE));
 849
 850         /* Allocate dynamic per-cpu area. */
 851         alloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE);
 852         dpcpu_init((void *)dpcpu, 0);
 853
 854         /* Allocate memory for the msgbuf, e.g. for /sbin/dmesg */
 855         alloc_pages(msgbufpv, round_page(msgbufsize) / PAGE_SIZE);
 856         msgbufp = (void *)msgbufpv;
 857
 858         /* Reserve some VA space for early BIOS/ACPI mapping */
 859         preinit_map_va = roundup2(freemempos, L2_SIZE);
 860
 861         virtual_avail = preinit_map_va + PMAP_PREINIT_MAPPING_SIZE;
 862         virtual_avail = roundup2(virtual_avail, L1_SIZE);
 863         virtual_end = VM_MAX_KERNEL_ADDRESS - (PMAP_MAPDEV_EARLY_SIZE);
 864         kernel_vm_end = virtual_avail;
 865
 866         pa = pmap_early_vtophys(l1pt, freemempos);
 867
 868         arm_physmem_exclude_region(start_pa, pa - start_pa, EXFLAG_NOALLOC);
 869
 870         cpu_tlb_flushID();
 871 }
 872
 873 /*
 874  *      Initialize a vm_page's machine-dependent fields.
 875  */
 876 void
 877 pmap_page_init(vm_page_t m)
 878 {
 879
 880         TAILQ_INIT(&m->md.pv_list);
 881         m->md.pv_memattr = VM_MEMATTR_WRITE_BACK;
 882 }
 883
 884 /*
 885  *      Initialize the pmap module.
 886  *      Called by vm_init, to initialize any structures that the pmap
 887  *      system needs to map virtual memory.
 888  */
 889 void
 890 pmap_init(void)
 891 {
 892         vm_size_t s;
 893         int i, pv_npg;
 894
 895         /*
 896          * Are large page mappings enabled?
 897          */
 898         TUNABLE_INT_FETCH("vm.pmap.superpages_enabled", &superpages_enabled);
 899         if (superpages_enabled) {
 900                 KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0,
 901                     ("pmap_init: can't assign to pagesizes[1]"));
 902                 pagesizes[1] = L2_SIZE;
 903         }
 904
 905         /*
 906          * Initialize the pv chunk list mutex.
 907          */
 908         mtx_init(&pv_chunks_mutex, "pmap pv chunk list", NULL, MTX_DEF);
 909
 910         /*
 911          * Initialize the pool of pv list locks.
 912          */
 913         for (i = 0; i < NPV_LIST_LOCKS; i++)
 914                 rw_init(&pv_list_locks[i], "pmap pv list");
 915
 916         /*
 917          * Calculate the size of the pv head table for superpages.
 918          */
 919         pv_npg = howmany(vm_phys_segs[vm_phys_nsegs - 1].end, L2_SIZE);
 920
 921         /*
 922          * Allocate memory for the pv head table for superpages.
 923          */
 924         s = (vm_size_t)(pv_npg * sizeof(struct md_page));
 925         s = round_page(s);
 926         pv_table = (struct md_page *)kmem_malloc(s, M_WAITOK | M_ZERO);
 927         for (i = 0; i < pv_npg; i++)
 928                 TAILQ_INIT(&pv_table[i].pv_list);
 929         TAILQ_INIT(&pv_dummy.pv_list);
 930
 931         vm_initialized = 1;
 932 }
 933
 934 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l2, CTLFLAG_RD, 0,
 935     "2MB page mapping counters");
 936
 937 static u_long pmap_l2_demotions;
 938 SYSCTL_ULONG(_vm_pmap_l2, OID_AUTO, demotions, CTLFLAG_RD,
 939     &pmap_l2_demotions, 0, "2MB page demotions");
 940
 941 static u_long pmap_l2_mappings;
 942 SYSCTL_ULONG(_vm_pmap_l2, OID_AUTO, mappings, CTLFLAG_RD,
 943     &pmap_l2_mappings, 0, "2MB page mappings");
 944
 945 static u_long pmap_l2_p_failures;
 946 SYSCTL_ULONG(_vm_pmap_l2, OID_AUTO, p_failures, CTLFLAG_RD,
 947     &pmap_l2_p_failures, 0, "2MB page promotion failures");
 948
 949 static u_long pmap_l2_promotions;
 950 SYSCTL_ULONG(_vm_pmap_l2, OID_AUTO, promotions, CTLFLAG_RD,
 951     &pmap_l2_promotions, 0, "2MB page promotions");
 952
 953 /*
 954  * Invalidate a single TLB entry.
 955  */
 956 static __inline void
 957 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
 958 {
 959
 960         sched_pin();
 961         __asm __volatile(
 962             "dsb  ishst         \n"
 963             "tlbi vaae1is, %0   \n"
 964             "dsb  ish           \n"
 965             "isb                \n"
 966             : : "r"(va >> PAGE_SHIFT));
 967         sched_unpin();
 968 }
 969
 970 static __inline void
 971 pmap_invalidate_range_nopin(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 972 {
 973         vm_offset_t addr;
 974
 975         dsb(ishst);
 976         for (addr = sva; addr < eva; addr += PAGE_SIZE) {
 977                 __asm __volatile(
 978                     "tlbi vaae1is, %0" : : "r"(addr >> PAGE_SHIFT));
 979         }
 980         __asm __volatile(
 981             "dsb  ish   \n"
 982             "isb        \n");
 983 }
 984
 985 static __inline void
 986 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 987 {
 988
 989         sched_pin();
 990         pmap_invalidate_range_nopin(pmap, sva, eva);
 991         sched_unpin();
 992 }
 993
 994 static __inline void
 995 pmap_invalidate_all(pmap_t pmap)
 996 {
 997
 998         sched_pin();
 999         __asm __volatile(
1000             "dsb  ishst         \n"
1001             "tlbi vmalle1is     \n"
1002             "dsb  ish           \n"
1003             "isb                \n");
1004         sched_unpin();
1005 }
1006
1007 /*
1008  *      Routine:        pmap_extract
1009  *      Function:
1010  *              Extract the physical page address associated
1011  *              with the given map/virtual_address pair.
1012  */
1013 vm_paddr_t
1014 pmap_extract(pmap_t pmap, vm_offset_t va)
1015 {
1016         pt_entry_t *pte, tpte;
1017         vm_paddr_t pa;
1018         int lvl;
1019
1020         pa = 0;
1021         PMAP_LOCK(pmap);
1022         /*
1023          * Find the block or page map for this virtual address. pmap_pte
1024          * will return either a valid block/page entry, or NULL.
1025          */
1026         pte = pmap_pte(pmap, va, &lvl);
1027         if (pte != NULL) {
1028                 tpte = pmap_load(pte);
1029                 pa = tpte & ~ATTR_MASK;
1030                 switch(lvl) {
1031                 case 1:
1032                         KASSERT((tpte & ATTR_DESCR_MASK) == L1_BLOCK,
1033                             ("pmap_extract: Invalid L1 pte found: %lx",
1034                             tpte & ATTR_DESCR_MASK));
1035                         pa |= (va & L1_OFFSET);
1036                         break;
1037                 case 2:
1038                         KASSERT((tpte & ATTR_DESCR_MASK) == L2_BLOCK,
1039                             ("pmap_extract: Invalid L2 pte found: %lx",
1040                             tpte & ATTR_DESCR_MASK));
1041                         pa |= (va & L2_OFFSET);
1042                         break;
1043                 case 3:
1044                         KASSERT((tpte & ATTR_DESCR_MASK) == L3_PAGE,
1045                             ("pmap_extract: Invalid L3 pte found: %lx",
1046                             tpte & ATTR_DESCR_MASK));
1047                         pa |= (va & L3_OFFSET);
1048                         break;
1049                 }
1050         }
1051         PMAP_UNLOCK(pmap);
1052         return (pa);
1053 }
1054
1055 /*
1056  *      Routine:        pmap_extract_and_hold
1057  *      Function:
1058  *              Atomically extract and hold the physical page
1059  *              with the given pmap and virtual address pair
1060  *              if that mapping permits the given protection.
1061  */
1062 vm_page_t
1063 pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
1064 {
1065         pt_entry_t *pte, tpte;
1066         vm_offset_t off;
1067         vm_paddr_t pa;
1068         vm_page_t m;
1069         int lvl;
1070
1071         pa = 0;
1072         m = NULL;
1073         PMAP_LOCK(pmap);
1074 retry:
1075         pte = pmap_pte(pmap, va, &lvl);
1076         if (pte != NULL) {
1077                 tpte = pmap_load(pte);
1078
1079                 KASSERT(lvl > 0 && lvl <= 3,
1080                     ("pmap_extract_and_hold: Invalid level %d", lvl));
1081                 CTASSERT(L1_BLOCK == L2_BLOCK);
1082                 KASSERT((lvl == 3 && (tpte & ATTR_DESCR_MASK) == L3_PAGE) ||
1083                     (lvl < 3 && (tpte & ATTR_DESCR_MASK) == L1_BLOCK),
1084                     ("pmap_extract_and_hold: Invalid pte at L%d: %lx", lvl,
1085                      tpte & ATTR_DESCR_MASK));
1086                 if (((tpte & ATTR_AP_RW_BIT) == ATTR_AP(ATTR_AP_RW)) ||
1087                     ((prot & VM_PROT_WRITE) == 0)) {
1088                         switch(lvl) {
1089                         case 1:
1090                                 off = va & L1_OFFSET;
1091                                 break;
1092                         case 2:
1093                                 off = va & L2_OFFSET;
1094                                 break;
1095                         case 3:
1096                         default:
1097                                 off = 0;
1098                         }
1099                         if (vm_page_pa_tryrelock(pmap,
1100                             (tpte & ~ATTR_MASK) | off, &pa))
1101                                 goto retry;
1102                         m = PHYS_TO_VM_PAGE((tpte & ~ATTR_MASK) | off);
1103                         vm_page_hold(m);
1104                 }
1105         }
1106         PA_UNLOCK_COND(pa);
1107         PMAP_UNLOCK(pmap);
1108         return (m);
1109 }
1110
1111 vm_paddr_t
1112 pmap_kextract(vm_offset_t va)
1113 {
1114         pt_entry_t *pte, tpte;
1115         vm_paddr_t pa;
1116         int lvl;
1117
1118         if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) {
1119                 pa = DMAP_TO_PHYS(va);
1120         } else {
1121                 pa = 0;
1122                 pte = pmap_pte(kernel_pmap, va, &lvl);
1123                 if (pte != NULL) {
1124                         tpte = pmap_load(pte);
1125                         pa = tpte & ~ATTR_MASK;
1126                         switch(lvl) {
1127                         case 1:
1128                                 KASSERT((tpte & ATTR_DESCR_MASK) == L1_BLOCK,
1129                                     ("pmap_kextract: Invalid L1 pte found: %lx",
1130                                     tpte & ATTR_DESCR_MASK));
1131                                 pa |= (va & L1_OFFSET);
1132                                 break;
1133                         case 2:
1134                                 KASSERT((tpte & ATTR_DESCR_MASK) == L2_BLOCK,
1135                                     ("pmap_kextract: Invalid L2 pte found: %lx",
1136                                     tpte & ATTR_DESCR_MASK));
1137                                 pa |= (va & L2_OFFSET);
1138                                 break;
1139                         case 3:
1140                                 KASSERT((tpte & ATTR_DESCR_MASK) == L3_PAGE,
1141                                     ("pmap_kextract: Invalid L3 pte found: %lx",
1142                                     tpte & ATTR_DESCR_MASK));
1143                                 pa |= (va & L3_OFFSET);
1144                                 break;
1145                         }
1146                 }
1147         }
1148         return (pa);
1149 }
1150
1151 /***************************************************
1152  * Low level mapping routines.....
1153  ***************************************************/
1154
1155 void
1156 pmap_kenter(vm_offset_t sva, vm_size_t size, vm_paddr_t pa, int mode)
1157 {
1158         pd_entry_t *pde;
1159         pt_entry_t *pte, attr;
1160         vm_offset_t va;
1161         int lvl;
1162
1163         KASSERT((pa & L3_OFFSET) == 0,
1164            ("pmap_kenter: Invalid physical address"));
1165         KASSERT((sva & L3_OFFSET) == 0,
1166            ("pmap_kenter: Invalid virtual address"));
1167         KASSERT((size & PAGE_MASK) == 0,
1168             ("pmap_kenter: Mapping is not page-sized"));
1169
1170         attr = ATTR_DEFAULT | ATTR_IDX(mode) | L3_PAGE;
1171         if (mode == DEVICE_MEMORY)
1172                 attr |= ATTR_XN;
1173
1174         va = sva;
1175         while (size != 0) {
1176                 pde = pmap_pde(kernel_pmap, va, &lvl);
1177                 KASSERT(pde != NULL,
1178                     ("pmap_kenter: Invalid page entry, va: 0x%lx", va));
1179                 KASSERT(lvl == 2, ("pmap_kenter: Invalid level %d", lvl));
1180
1181                 pte = pmap_l2_to_l3(pde, va);
1182                 pmap_load_store(pte, (pa & ~L3_OFFSET) | attr);
1183
1184                 va += PAGE_SIZE;
1185                 pa += PAGE_SIZE;
1186                 size -= PAGE_SIZE;
1187         }
1188         pmap_invalidate_range(kernel_pmap, sva, va);
1189 }
1190
1191 void
1192 pmap_kenter_device(vm_offset_t sva, vm_size_t size, vm_paddr_t pa)
1193 {
1194
1195         pmap_kenter(sva, size, pa, DEVICE_MEMORY);
1196 }
1197
1198 /*
1199  * Remove a page from the kernel pagetables.
1200  */
1201 PMAP_INLINE void
1202 pmap_kremove(vm_offset_t va)
1203 {
1204         pt_entry_t *pte;
1205         int lvl;
1206
1207         pte = pmap_pte(kernel_pmap, va, &lvl);
1208         KASSERT(pte != NULL, ("pmap_kremove: Invalid address"));
1209         KASSERT(lvl == 3, ("pmap_kremove: Invalid pte level %d", lvl));
1210
1211         pmap_load_clear(pte);
1212         pmap_invalidate_page(kernel_pmap, va);
1213 }
1214
1215 void
1216 pmap_kremove_device(vm_offset_t sva, vm_size_t size)
1217 {
1218         pt_entry_t *pte;
1219         vm_offset_t va;
1220         int lvl;
1221
1222         KASSERT((sva & L3_OFFSET) == 0,
1223            ("pmap_kremove_device: Invalid virtual address"));
1224         KASSERT((size & PAGE_MASK) == 0,
1225             ("pmap_kremove_device: Mapping is not page-sized"));
1226
1227         va = sva;
1228         while (size != 0) {
1229                 pte = pmap_pte(kernel_pmap, va, &lvl);
1230                 KASSERT(pte != NULL, ("Invalid page table, va: 0x%lx", va));
1231                 KASSERT(lvl == 3,
1232                     ("Invalid device pagetable level: %d != 3", lvl));
1233                 pmap_load_clear(pte);
1234
1235                 va += PAGE_SIZE;
1236                 size -= PAGE_SIZE;
1237         }
1238         pmap_invalidate_range(kernel_pmap, sva, va);
1239 }
1240
1241 /*
1242  *      Used to map a range of physical addresses into kernel
1243  *      virtual address space.
1244  *
1245  *      The value passed in '*virt' is a suggested virtual address for
1246  *      the mapping. Architectures which can support a direct-mapped
1247  *      physical to virtual region can return the appropriate address
1248  *      within that region, leaving '*virt' unchanged. Other
1249  *      architectures should map the pages starting at '*virt' and
1250  *      update '*virt' with the first usable address after the mapped
1251  *      region.
1252  */
1253 vm_offset_t
1254 pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot)
1255 {
1256         return PHYS_TO_DMAP(start);
1257 }
1258
1259
1260 /*
1261  * Add a list of wired pages to the kva
1262  * this routine is only used for temporary
1263  * kernel mappings that do not need to have
1264  * page modification or references recorded.
1265  * Note that old mappings are simply written
1266  * over.  The page *must* be wired.
1267  * Note: SMP coherent.  Uses a ranged shootdown IPI.
1268  */
1269 void
1270 pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count)
1271 {
1272         pd_entry_t *pde;
1273         pt_entry_t *pte, pa;
1274         vm_offset_t va;
1275         vm_page_t m;
1276         int i, lvl;
1277
1278         va = sva;
1279         for (i = 0; i < count; i++) {
1280                 pde = pmap_pde(kernel_pmap, va, &lvl);
1281                 KASSERT(pde != NULL,
1282                     ("pmap_qenter: Invalid page entry, va: 0x%lx", va));
1283                 KASSERT(lvl == 2,
1284                     ("pmap_qenter: Invalid level %d", lvl));
1285
1286                 m = ma[i];
1287                 pa = VM_PAGE_TO_PHYS(m) | ATTR_DEFAULT | ATTR_AP(ATTR_AP_RW) |
1288                     ATTR_IDX(m->md.pv_memattr) | L3_PAGE;
1289                 if (m->md.pv_memattr == DEVICE_MEMORY)
1290                         pa |= ATTR_XN;
1291                 pte = pmap_l2_to_l3(pde, va);
1292                 pmap_load_store(pte, pa);
1293
1294                 va += L3_SIZE;
1295         }
1296         pmap_invalidate_range(kernel_pmap, sva, va);
1297 }
1298
1299 /*
1300  * This routine tears out page mappings from the
1301  * kernel -- it is meant only for temporary mappings.
1302  */
1303 void
1304 pmap_qremove(vm_offset_t sva, int count)
1305 {
1306         pt_entry_t *pte;
1307         vm_offset_t va;
1308         int lvl;
1309
1310         KASSERT(sva >= VM_MIN_KERNEL_ADDRESS, ("usermode va %lx", sva));
1311
1312         va = sva;
1313         while (count-- > 0) {
1314                 pte = pmap_pte(kernel_pmap, va, &lvl);
1315                 KASSERT(lvl == 3,
1316                     ("Invalid device pagetable level: %d != 3", lvl));
1317                 if (pte != NULL) {
1318                         pmap_load_clear(pte);
1319                 }
1320
1321                 va += PAGE_SIZE;
1322         }
1323         pmap_invalidate_range(kernel_pmap, sva, va);
1324 }
1325
1326 /***************************************************
1327  * Page table page management routines.....
1328  ***************************************************/
1329 /*
1330  * Schedule the specified unused page table page to be freed.  Specifically,
1331  * add the page to the specified list of pages that will be released to the
1332  * physical memory manager after the TLB has been updated.
1333  */
1334 static __inline void
1335 pmap_add_delayed_free_list(vm_page_t m, struct spglist *free,
1336     boolean_t set_PG_ZERO)
1337 {
1338
1339         if (set_PG_ZERO)
1340                 m->flags |= PG_ZERO;
1341         else
1342                 m->flags &= ~PG_ZERO;
1343         SLIST_INSERT_HEAD(free, m, plinks.s.ss);
1344 }
1345
1346 /*
1347  * Decrements a page table page's wire count, which is used to record the
1348  * number of valid page table entries within the page.  If the wire count
1349  * drops to zero, then the page table page is unmapped.  Returns TRUE if the
1350  * page table page was unmapped and FALSE otherwise.
1351  */
1352 static inline boolean_t
1353 pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)
1354 {
1355
1356         --m->wire_count;
1357         if (m->wire_count == 0) {
1358                 _pmap_unwire_l3(pmap, va, m, free);
1359                 return (TRUE);
1360         } else
1361                 return (FALSE);
1362 }
1363
1364 static void
1365 _pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)
1366 {
1367
1368         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1369         /*
1370          * unmap the page table page
1371          */
1372         if (m->pindex >= (NUL2E + NUL1E)) {
1373                 /* l1 page */
1374                 pd_entry_t *l0;
1375
1376                 l0 = pmap_l0(pmap, va);
1377                 pmap_load_clear(l0);
1378         } else if (m->pindex >= NUL2E) {
1379                 /* l2 page */
1380                 pd_entry_t *l1;
1381
1382                 l1 = pmap_l1(pmap, va);
1383                 pmap_load_clear(l1);
1384         } else {
1385                 /* l3 page */
1386                 pd_entry_t *l2;
1387
1388                 l2 = pmap_l2(pmap, va);
1389                 pmap_load_clear(l2);
1390         }
1391         pmap_resident_count_dec(pmap, 1);
1392         if (m->pindex < NUL2E) {
1393                 /* We just released an l3, unhold the matching l2 */
1394                 pd_entry_t *l1, tl1;
1395                 vm_page_t l2pg;
1396
1397                 l1 = pmap_l1(pmap, va);
1398                 tl1 = pmap_load(l1);
1399                 l2pg = PHYS_TO_VM_PAGE(tl1 & ~ATTR_MASK);
1400                 pmap_unwire_l3(pmap, va, l2pg, free);
1401         } else if (m->pindex < (NUL2E + NUL1E)) {
1402                 /* We just released an l2, unhold the matching l1 */
1403                 pd_entry_t *l0, tl0;
1404                 vm_page_t l1pg;
1405
1406                 l0 = pmap_l0(pmap, va);
1407                 tl0 = pmap_load(l0);
1408                 l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK);
1409                 pmap_unwire_l3(pmap, va, l1pg, free);
1410         }
1411         pmap_invalidate_page(pmap, va);
1412
1413         /*
1414          * Put page on a list so that it is released after
1415          * *ALL* TLB shootdown is done
1416          */
1417         pmap_add_delayed_free_list(m, free, TRUE);
1418 }
1419
1420 /*
1421  * After removing a page table entry, this routine is used to
1422  * conditionally free the page, and manage the hold/wire counts.
1423  */
1424 static int
1425 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde,
1426     struct spglist *free)
1427 {
1428         vm_page_t mpte;
1429
1430         if (va >= VM_MAXUSER_ADDRESS)
1431                 return (0);
1432         KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0"));
1433         mpte = PHYS_TO_VM_PAGE(ptepde & ~ATTR_MASK);
1434         return (pmap_unwire_l3(pmap, va, mpte, free));
1435 }
1436
1437 void
1438 pmap_pinit0(pmap_t pmap)
1439 {
1440
1441         PMAP_LOCK_INIT(pmap);
1442         bzero(&pmap->pm_stats, sizeof(pmap->pm_stats));
1443         pmap->pm_l0 = kernel_pmap->pm_l0;
1444         pmap->pm_root.rt_root = 0;
1445 }
1446
1447 int
1448 pmap_pinit(pmap_t pmap)
1449 {
1450         vm_paddr_t l0phys;
1451         vm_page_t l0pt;
1452
1453         /*
1454          * allocate the l0 page
1455          */
1456         while ((l0pt = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL |
1457             VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL)
1458                 vm_wait(NULL);
1459
1460         l0phys = VM_PAGE_TO_PHYS(l0pt);
1461         pmap->pm_l0 = (pd_entry_t *)PHYS_TO_DMAP(l0phys);
1462
1463         if ((l0pt->flags & PG_ZERO) == 0)
1464                 pagezero(pmap->pm_l0);
1465
1466         pmap->pm_root.rt_root = 0;
1467         bzero(&pmap->pm_stats, sizeof(pmap->pm_stats));
1468
1469         return (1);
1470 }
1471
1472 /*
1473  * This routine is called if the desired page table page does not exist.
1474  *
1475  * If page table page allocation fails, this routine may sleep before
1476  * returning NULL.  It sleeps only if a lock pointer was given.
1477  *
1478  * Note: If a page allocation fails at page table level two or three,
1479  * one or two pages may be held during the wait, only to be released
1480  * afterwards.  This conservative approach is easily argued to avoid
1481  * race conditions.
1482  */
1483 static vm_page_t
1484 _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp)
1485 {
1486         vm_page_t m, l1pg, l2pg;
1487
1488         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1489
1490         /*
1491          * Allocate a page table page.
1492          */
1493         if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ |
1494             VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
1495                 if (lockp != NULL) {
1496                         RELEASE_PV_LIST_LOCK(lockp);
1497                         PMAP_UNLOCK(pmap);
1498                         vm_wait(NULL);
1499                         PMAP_LOCK(pmap);
1500                 }
1501
1502                 /*
1503                  * Indicate the need to retry.  While waiting, the page table
1504                  * page may have been allocated.
1505                  */
1506                 return (NULL);
1507         }
1508         if ((m->flags & PG_ZERO) == 0)
1509                 pmap_zero_page(m);
1510
1511         /*
1512          * Map the pagetable page into the process address space, if
1513          * it isn't already there.
1514          */
1515
1516         if (ptepindex >= (NUL2E + NUL1E)) {
1517                 pd_entry_t *l0;
1518                 vm_pindex_t l0index;
1519
1520                 l0index = ptepindex - (NUL2E + NUL1E);
1521                 l0 = &pmap->pm_l0[l0index];
1522                 pmap_load_store(l0, VM_PAGE_TO_PHYS(m) | L0_TABLE);
1523         } else if (ptepindex >= NUL2E) {
1524                 vm_pindex_t l0index, l1index;
1525                 pd_entry_t *l0, *l1;
1526                 pd_entry_t tl0;
1527
1528                 l1index = ptepindex - NUL2E;
1529                 l0index = l1index >> L0_ENTRIES_SHIFT;
1530
1531                 l0 = &pmap->pm_l0[l0index];
1532                 tl0 = pmap_load(l0);
1533                 if (tl0 == 0) {
1534                         /* recurse for allocating page dir */
1535                         if (_pmap_alloc_l3(pmap, NUL2E + NUL1E + l0index,
1536                             lockp) == NULL) {
1537                                 vm_page_unwire_noq(m);
1538                                 vm_page_free_zero(m);
1539                                 return (NULL);
1540                         }
1541                 } else {
1542                         l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK);
1543                         l1pg->wire_count++;
1544                 }
1545
1546                 l1 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l0) & ~ATTR_MASK);
1547                 l1 = &l1[ptepindex & Ln_ADDR_MASK];
1548                 pmap_load_store(l1, VM_PAGE_TO_PHYS(m) | L1_TABLE);
1549         } else {
1550                 vm_pindex_t l0index, l1index;
1551                 pd_entry_t *l0, *l1, *l2;
1552                 pd_entry_t tl0, tl1;
1553
1554                 l1index = ptepindex >> Ln_ENTRIES_SHIFT;
1555                 l0index = l1index >> L0_ENTRIES_SHIFT;
1556
1557                 l0 = &pmap->pm_l0[l0index];
1558                 tl0 = pmap_load(l0);
1559                 if (tl0 == 0) {
1560                         /* recurse for allocating page dir */
1561                         if (_pmap_alloc_l3(pmap, NUL2E + l1index,
1562                             lockp) == NULL) {
1563                                 vm_page_unwire_noq(m);
1564                                 vm_page_free_zero(m);
1565                                 return (NULL);
1566                         }
1567                         tl0 = pmap_load(l0);
1568                         l1 = (pd_entry_t *)PHYS_TO_DMAP(tl0 & ~ATTR_MASK);
1569                         l1 = &l1[l1index & Ln_ADDR_MASK];
1570                 } else {
1571                         l1 = (pd_entry_t *)PHYS_TO_DMAP(tl0 & ~ATTR_MASK);
1572                         l1 = &l1[l1index & Ln_ADDR_MASK];
1573                         tl1 = pmap_load(l1);
1574                         if (tl1 == 0) {
1575                                 /* recurse for allocating page dir */
1576                                 if (_pmap_alloc_l3(pmap, NUL2E + l1index,
1577                                     lockp) == NULL) {
1578                                         vm_page_unwire_noq(m);
1579                                         vm_page_free_zero(m);
1580                                         return (NULL);
1581                                 }
1582                         } else {
1583                                 l2pg = PHYS_TO_VM_PAGE(tl1 & ~ATTR_MASK);
1584                                 l2pg->wire_count++;
1585                         }
1586                 }
1587
1588                 l2 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l1) & ~ATTR_MASK);
1589                 l2 = &l2[ptepindex & Ln_ADDR_MASK];
1590                 pmap_load_store(l2, VM_PAGE_TO_PHYS(m) | L2_TABLE);
1591         }
1592
1593         pmap_resident_count_inc(pmap, 1);
1594
1595         return (m);
1596 }
1597
1598 static vm_page_t
1599 pmap_alloc_l2(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)
1600 {
1601         pd_entry_t *l1;
1602         vm_page_t l2pg;
1603         vm_pindex_t l2pindex;
1604
1605 retry:
1606         l1 = pmap_l1(pmap, va);
1607         if (l1 != NULL && (pmap_load(l1) & ATTR_DESCR_MASK) == L1_TABLE) {
1608                 /* Add a reference to the L2 page. */
1609                 l2pg = PHYS_TO_VM_PAGE(pmap_load(l1) & ~ATTR_MASK);
1610                 l2pg->wire_count++;
1611         } else {
1612                 /* Allocate a L2 page. */
1613                 l2pindex = pmap_l2_pindex(va) >> Ln_ENTRIES_SHIFT;
1614                 l2pg = _pmap_alloc_l3(pmap, NUL2E + l2pindex, lockp);
1615                 if (l2pg == NULL && lockp != NULL)
1616                         goto retry;
1617         }
1618         return (l2pg);
1619 }
1620
1621 static vm_page_t
1622 pmap_alloc_l3(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)
1623 {
1624         vm_pindex_t ptepindex;
1625         pd_entry_t *pde, tpde;
1626 #ifdef INVARIANTS
1627         pt_entry_t *pte;
1628 #endif
1629         vm_page_t m;
1630         int lvl;
1631
1632         /*
1633          * Calculate pagetable page index
1634          */
1635         ptepindex = pmap_l2_pindex(va);
1636 retry:
1637         /*
1638          * Get the page directory entry
1639          */
1640         pde = pmap_pde(pmap, va, &lvl);
1641
1642         /*
1643          * If the page table page is mapped, we just increment the hold count,
1644          * and activate it. If we get a level 2 pde it will point to a level 3
1645          * table.
1646          */
1647         switch (lvl) {
1648         case -1:
1649                 break;
1650         case 0:
1651 #ifdef INVARIANTS
1652                 pte = pmap_l0_to_l1(pde, va);
1653                 KASSERT(pmap_load(pte) == 0,
1654                     ("pmap_alloc_l3: TODO: l0 superpages"));
1655 #endif
1656                 break;
1657         case 1:
1658 #ifdef INVARIANTS
1659                 pte = pmap_l1_to_l2(pde, va);
1660                 KASSERT(pmap_load(pte) == 0,
1661                     ("pmap_alloc_l3: TODO: l1 superpages"));
1662 #endif
1663                 break;
1664         case 2:
1665                 tpde = pmap_load(pde);
1666                 if (tpde != 0) {
1667                         m = PHYS_TO_VM_PAGE(tpde & ~ATTR_MASK);
1668                         m->wire_count++;
1669                         return (m);
1670                 }
1671                 break;
1672         default:
1673                 panic("pmap_alloc_l3: Invalid level %d", lvl);
1674         }
1675
1676         /*
1677          * Here if the pte page isn't mapped, or if it has been deallocated.
1678          */
1679         m = _pmap_alloc_l3(pmap, ptepindex, lockp);
1680         if (m == NULL && lockp != NULL)
1681                 goto retry;
1682
1683         return (m);
1684 }
1685
1686 /***************************************************
1687  * Pmap allocation/deallocation routines.
1688  ***************************************************/
1689
1690 /*
1691  * Release any resources held by the given physical map.
1692  * Called when a pmap initialized by pmap_pinit is being released.
1693  * Should only be called if the map contains no valid mappings.
1694  */
1695 void
1696 pmap_release(pmap_t pmap)
1697 {
1698         vm_page_t m;
1699
1700         KASSERT(pmap->pm_stats.resident_count == 0,
1701             ("pmap_release: pmap resident count %ld != 0",
1702             pmap->pm_stats.resident_count));
1703         KASSERT(vm_radix_is_empty(&pmap->pm_root),
1704             ("pmap_release: pmap has reserved page table page(s)"));
1705
1706         m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pmap->pm_l0));
1707
1708         vm_page_unwire_noq(m);
1709         vm_page_free_zero(m);
1710 }
1711
1712 static int
1713 kvm_size(SYSCTL_HANDLER_ARGS)
1714 {
1715         unsigned long ksize = VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS;
1716
1717         return sysctl_handle_long(oidp, &ksize, 0, req);
1718 }
1719 SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG|CTLFLAG_RD,
1720     0, 0, kvm_size, "LU", "Size of KVM");
1721
1722 static int
1723 kvm_free(SYSCTL_HANDLER_ARGS)
1724 {
1725         unsigned long kfree = VM_MAX_KERNEL_ADDRESS - kernel_vm_end;
1726
1727         return sysctl_handle_long(oidp, &kfree, 0, req);
1728 }
1729 SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG|CTLFLAG_RD,
1730     0, 0, kvm_free, "LU", "Amount of KVM free");
1731
1732 /*
1733  * grow the number of kernel page table entries, if needed
1734  */
1735 void
1736 pmap_growkernel(vm_offset_t addr)
1737 {
1738         vm_paddr_t paddr;
1739         vm_page_t nkpg;
1740         pd_entry_t *l0, *l1, *l2;
1741
1742         mtx_assert(&kernel_map->system_mtx, MA_OWNED);
1743
1744         addr = roundup2(addr, L2_SIZE);
1745         if (addr - 1 >= vm_map_max(kernel_map))
1746                 addr = vm_map_max(kernel_map);
1747         while (kernel_vm_end < addr) {
1748                 l0 = pmap_l0(kernel_pmap, kernel_vm_end);
1749                 KASSERT(pmap_load(l0) != 0,
1750                     ("pmap_growkernel: No level 0 kernel entry"));
1751
1752                 l1 = pmap_l0_to_l1(l0, kernel_vm_end);
1753                 if (pmap_load(l1) == 0) {
1754                         /* We need a new PDP entry */
1755                         nkpg = vm_page_alloc(NULL, kernel_vm_end >> L1_SHIFT,
1756                             VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ |
1757                             VM_ALLOC_WIRED | VM_ALLOC_ZERO);
1758                         if (nkpg == NULL)
1759                                 panic("pmap_growkernel: no memory to grow kernel");
1760                         if ((nkpg->flags & PG_ZERO) == 0)
1761                                 pmap_zero_page(nkpg);
1762                         paddr = VM_PAGE_TO_PHYS(nkpg);
1763                         pmap_load_store(l1, paddr | L1_TABLE);
1764                         continue; /* try again */
1765                 }
1766                 l2 = pmap_l1_to_l2(l1, kernel_vm_end);
1767                 if ((pmap_load(l2) & ATTR_AF) != 0) {
1768                         kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET;
1769                         if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
1770                                 kernel_vm_end = vm_map_max(kernel_map);
1771                                 break;
1772                         }
1773                         continue;
1774                 }
1775
1776                 nkpg = vm_page_alloc(NULL, kernel_vm_end >> L2_SHIFT,
1777                     VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
1778                     VM_ALLOC_ZERO);
1779                 if (nkpg == NULL)
1780                         panic("pmap_growkernel: no memory to grow kernel");
1781                 if ((nkpg->flags & PG_ZERO) == 0)
1782                         pmap_zero_page(nkpg);
1783                 paddr = VM_PAGE_TO_PHYS(nkpg);
1784                 pmap_load_store(l2, paddr | L2_TABLE);
1785                 pmap_invalidate_page(kernel_pmap, kernel_vm_end);
1786
1787                 kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET;
1788                 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
1789                         kernel_vm_end = vm_map_max(kernel_map);
1790                         break;
1791                 }
1792         }
1793 }
1794
1795
1796 /***************************************************
1797  * page management routines.
1798  ***************************************************/
1799
1800 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE);
1801 CTASSERT(_NPCM == 3);
1802 CTASSERT(_NPCPV == 168);
1803
1804 static __inline struct pv_chunk *
1805 pv_to_chunk(pv_entry_t pv)
1806 {
1807
1808         return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK));
1809 }
1810
1811 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
1812
1813 #define PC_FREE0        0xfffffffffffffffful
1814 #define PC_FREE1        0xfffffffffffffffful
1815 #define PC_FREE2        0x000000fffffffffful
1816
1817 static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1, PC_FREE2 };
1818
1819 #if 0
1820 #ifdef PV_STATS
1821 static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail;
1822
1823 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0,
1824         "Current number of pv entry chunks");
1825 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0,
1826         "Current number of pv entry chunks allocated");
1827 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0,
1828         "Current number of pv entry chunks frees");
1829 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, &pc_chunk_tryfail, 0,
1830         "Number of times tried to get a chunk page but failed.");
1831
1832 static long pv_entry_frees, pv_entry_allocs, pv_entry_count;
1833 static int pv_entry_spare;
1834
1835 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, &pv_entry_frees, 0,
1836         "Current number of pv entry frees");
1837 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, &pv_entry_allocs, 0,
1838         "Current number of pv entry allocs");
1839 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0,
1840         "Current number of pv entries");
1841 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0,
1842         "Current number of spare pv entries");
1843 #endif
1844 #endif /* 0 */
1845
1846 /*
1847  * We are in a serious low memory condition.  Resort to
1848  * drastic measures to free some pages so we can allocate
1849  * another pv entry chunk.
1850  *
1851  * Returns NULL if PV entries were reclaimed from the specified pmap.
1852  *
1853  * We do not, however, unmap 2mpages because subsequent accesses will
1854  * allocate per-page pv entries until repromotion occurs, thereby
1855  * exacerbating the shortage of free pv entries.
1856  */
1857 static vm_page_t
1858 reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp)
1859 {
1860         struct pv_chunk *pc, *pc_marker, *pc_marker_end;
1861         struct pv_chunk_header pc_marker_b, pc_marker_end_b;
1862         struct md_page *pvh;
1863         pd_entry_t *pde;
1864         pmap_t next_pmap, pmap;
1865         pt_entry_t *pte, tpte;
1866         pv_entry_t pv;
1867         vm_offset_t va;
1868         vm_page_t m, m_pc;
1869         struct spglist free;
1870         uint64_t inuse;
1871         int bit, field, freed, lvl;
1872         static int active_reclaims = 0;
1873
1874         PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED);
1875         KASSERT(lockp != NULL, ("reclaim_pv_chunk: lockp is NULL"));
1876
1877         pmap = NULL;
1878         m_pc = NULL;
1879         SLIST_INIT(&free);
1880         bzero(&pc_marker_b, sizeof(pc_marker_b));
1881         bzero(&pc_marker_end_b, sizeof(pc_marker_end_b));
1882         pc_marker = (struct pv_chunk *)&pc_marker_b;
1883         pc_marker_end = (struct pv_chunk *)&pc_marker_end_b;
1884
1885         mtx_lock(&pv_chunks_mutex);
1886         active_reclaims++;
1887         TAILQ_INSERT_HEAD(&pv_chunks, pc_marker, pc_lru);
1888         TAILQ_INSERT_TAIL(&pv_chunks, pc_marker_end, pc_lru);
1889         while ((pc = TAILQ_NEXT(pc_marker, pc_lru)) != pc_marker_end &&
1890             SLIST_EMPTY(&free)) {
1891                 next_pmap = pc->pc_pmap;
1892                 if (next_pmap == NULL) {
1893                         /*
1894                          * The next chunk is a marker.  However, it is
1895                          * not our marker, so active_reclaims must be
1896                          * > 1.  Consequently, the next_chunk code
1897                          * will not rotate the pv_chunks list.
1898                          */
1899                         goto next_chunk;
1900                 }
1901                 mtx_unlock(&pv_chunks_mutex);
1902
1903                 /*
1904                  * A pv_chunk can only be removed from the pc_lru list
1905                  * when both pv_chunks_mutex is owned and the
1906                  * corresponding pmap is locked.
1907                  */
1908                 if (pmap != next_pmap) {
1909                         if (pmap != NULL && pmap != locked_pmap)
1910                                 PMAP_UNLOCK(pmap);
1911                         pmap = next_pmap;
1912                         /* Avoid deadlock and lock recursion. */
1913                         if (pmap > locked_pmap) {
1914                                 RELEASE_PV_LIST_LOCK(lockp);
1915                                 PMAP_LOCK(pmap);
1916                                 mtx_lock(&pv_chunks_mutex);
1917                                 continue;
1918                         } else if (pmap != locked_pmap) {
1919                                 if (PMAP_TRYLOCK(pmap)) {
1920                                         mtx_lock(&pv_chunks_mutex);
1921                                         continue;
1922                                 } else {
1923                                         pmap = NULL; /* pmap is not locked */
1924                                         mtx_lock(&pv_chunks_mutex);
1925                                         pc = TAILQ_NEXT(pc_marker, pc_lru);
1926                                         if (pc == NULL ||
1927                                             pc->pc_pmap != next_pmap)
1928                                                 continue;
1929                                         goto next_chunk;
1930                                 }
1931                         }
1932                 }
1933
1934                 /*
1935                  * Destroy every non-wired, 4 KB page mapping in the chunk.
1936                  */
1937                 freed = 0;
1938                 for (field = 0; field < _NPCM; field++) {
1939                         for (inuse = ~pc->pc_map[field] & pc_freemask[field];
1940                             inuse != 0; inuse &= ~(1UL << bit)) {
1941                                 bit = ffsl(inuse) - 1;
1942                                 pv = &pc->pc_pventry[field * 64 + bit];
1943                                 va = pv->pv_va;
1944                                 pde = pmap_pde(pmap, va, &lvl);
1945                                 if (lvl != 2)
1946                                         continue;
1947                                 pte = pmap_l2_to_l3(pde, va);
1948                                 tpte = pmap_load(pte);
1949                                 if ((tpte & ATTR_SW_WIRED) != 0)
1950                                         continue;
1951                                 tpte = pmap_load_clear(pte);
1952                                 pmap_invalidate_page(pmap, va);
1953                                 m = PHYS_TO_VM_PAGE(tpte & ~ATTR_MASK);
1954                                 if (pmap_page_dirty(tpte))
1955                                         vm_page_dirty(m);
1956                                 if ((tpte & ATTR_AF) != 0)
1957                                         vm_page_aflag_set(m, PGA_REFERENCED);
1958                                 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
1959                                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
1960                                 m->md.pv_gen++;
1961                                 if (TAILQ_EMPTY(&m->md.pv_list) &&
1962                                     (m->flags & PG_FICTITIOUS) == 0) {
1963                                         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
1964                                         if (TAILQ_EMPTY(&pvh->pv_list)) {
1965                                                 vm_page_aflag_clear(m,
1966                                                     PGA_WRITEABLE);
1967                                         }
1968                                 }
1969                                 pc->pc_map[field] |= 1UL << bit;
1970                                 pmap_unuse_pt(pmap, va, pmap_load(pde), &free);
1971                                 freed++;
1972                         }
1973                 }
1974                 if (freed == 0) {
1975                         mtx_lock(&pv_chunks_mutex);
1976                         goto next_chunk;
1977                 }
1978                 /* Every freed mapping is for a 4 KB page. */
1979                 pmap_resident_count_dec(pmap, freed);
1980                 PV_STAT(atomic_add_long(&pv_entry_frees, freed));
1981                 PV_STAT(atomic_add_int(&pv_entry_spare, freed));
1982                 PV_STAT(atomic_subtract_long(&pv_entry_count, freed));
1983                 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1984                 if (pc->pc_map[0] == PC_FREE0 && pc->pc_map[1] == PC_FREE1 &&
1985                     pc->pc_map[2] == PC_FREE2) {
1986                         PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV));
1987                         PV_STAT(atomic_subtract_int(&pc_chunk_count, 1));
1988                         PV_STAT(atomic_add_int(&pc_chunk_frees, 1));
1989                         /* Entire chunk is free; return it. */
1990                         m_pc = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc));
1991                         dump_drop_page(m_pc->phys_addr);
1992                         mtx_lock(&pv_chunks_mutex);
1993                         TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
1994                         break;
1995                 }
1996                 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1997                 mtx_lock(&pv_chunks_mutex);
1998                 /* One freed pv entry in locked_pmap is sufficient. */
1999                 if (pmap == locked_pmap)
2000                         break;
2001
2002 next_chunk:
2003                 TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru);
2004                 TAILQ_INSERT_AFTER(&pv_chunks, pc, pc_marker, pc_lru);
2005                 if (active_reclaims == 1 && pmap != NULL) {
2006                         /*
2007                          * Rotate the pv chunks list so that we do not
2008                          * scan the same pv chunks that could not be
2009                          * freed (because they contained a wired
2010                          * and/or superpage mapping) on every
2011                          * invocation of reclaim_pv_chunk().
2012                          */
2013                         while ((pc = TAILQ_FIRST(&pv_chunks)) != pc_marker) {
2014                                 MPASS(pc->pc_pmap != NULL);
2015                                 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
2016                                 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
2017                         }
2018                 }
2019         }
2020         TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru);
2021         TAILQ_REMOVE(&pv_chunks, pc_marker_end, pc_lru);
2022         active_reclaims--;
2023         mtx_unlock(&pv_chunks_mutex);
2024         if (pmap != NULL && pmap != locked_pmap)
2025                 PMAP_UNLOCK(pmap);
2026         if (m_pc == NULL && !SLIST_EMPTY(&free)) {
2027                 m_pc = SLIST_FIRST(&free);
2028                 SLIST_REMOVE_HEAD(&free, plinks.s.ss);
2029                 /* Recycle a freed page table page. */
2030                 m_pc->wire_count = 1;
2031         }
2032         vm_page_free_pages_toq(&free, true);
2033         return (m_pc);
2034 }
2035
2036 /*
2037  * free the pv_entry back to the free list
2038  */
2039 static void
2040 free_pv_entry(pmap_t pmap, pv_entry_t pv)
2041 {
2042         struct pv_chunk *pc;
2043         int idx, field, bit;
2044
2045         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
2046         PV_STAT(atomic_add_long(&pv_entry_frees, 1));
2047         PV_STAT(atomic_add_int(&pv_entry_spare, 1));
2048         PV_STAT(atomic_subtract_long(&pv_entry_count, 1));
2049         pc = pv_to_chunk(pv);
2050         idx = pv - &pc->pc_pventry[0];
2051         field = idx / 64;
2052         bit = idx % 64;
2053         pc->pc_map[field] |= 1ul << bit;
2054         if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1 ||
2055             pc->pc_map[2] != PC_FREE2) {
2056                 /* 98% of the time, pc is already at the head of the list. */
2057                 if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) {
2058                         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
2059                         TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
2060                 }
2061                 return;
2062         }
2063         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
2064         free_pv_chunk(pc);
2065 }
2066
2067 static void
2068 free_pv_chunk(struct pv_chunk *pc)
2069 {
2070         vm_page_t m;
2071
2072         mtx_lock(&pv_chunks_mutex);
2073         TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
2074         mtx_unlock(&pv_chunks_mutex);
2075         PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV));
2076         PV_STAT(atomic_subtract_int(&pc_chunk_count, 1));
2077         PV_STAT(atomic_add_int(&pc_chunk_frees, 1));
2078         /* entire chunk is free, return it */
2079         m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc));
2080         dump_drop_page(m->phys_addr);
2081         vm_page_unwire_noq(m);
2082         vm_page_free(m);
2083 }
2084
2085 /*
2086  * Returns a new PV entry, allocating a new PV chunk from the system when
2087  * needed.  If this PV chunk allocation fails and a PV list lock pointer was
2088  * given, a PV chunk is reclaimed from an arbitrary pmap.  Otherwise, NULL is
2089  * returned.
2090  *
2091  * The given PV list lock may be released.
2092  */
2093 static pv_entry_t
2094 get_pv_entry(pmap_t pmap, struct rwlock **lockp)
2095 {
2096         int bit, field;
2097         pv_entry_t pv;
2098         struct pv_chunk *pc;
2099         vm_page_t m;
2100
2101         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
2102         PV_STAT(atomic_add_long(&pv_entry_allocs, 1));
2103 retry:
2104         pc = TAILQ_FIRST(&pmap->pm_pvchunk);
2105         if (pc != NULL) {
2106                 for (field = 0; field < _NPCM; field++) {
2107                         if (pc->pc_map[field]) {
2108                                 bit = ffsl(pc->pc_map[field]) - 1;
2109                                 break;
2110                         }
2111                 }
2112                 if (field < _NPCM) {
2113                         pv = &pc->pc_pventry[field * 64 + bit];
2114                         pc->pc_map[field] &= ~(1ul << bit);
2115                         /* If this was the last item, move it to tail */
2116                         if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0 &&
2117                             pc->pc_map[2] == 0) {
2118                                 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
2119                                 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc,
2120                                     pc_list);
2121                         }
2122                         PV_STAT(atomic_add_long(&pv_entry_count, 1));
2123                         PV_STAT(atomic_subtract_int(&pv_entry_spare, 1));
2124                         return (pv);
2125                 }
2126         }
2127         /* No free items, allocate another chunk */
2128         m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ |
2129             VM_ALLOC_WIRED);
2130         if (m == NULL) {
2131                 if (lockp == NULL) {
2132                         PV_STAT(pc_chunk_tryfail++);
2133                         return (NULL);
2134                 }
2135                 m = reclaim_pv_chunk(pmap, lockp);
2136                 if (m == NULL)
2137                         goto retry;
2138         }
2139         PV_STAT(atomic_add_int(&pc_chunk_count, 1));
2140         PV_STAT(atomic_add_int(&pc_chunk_allocs, 1));
2141         dump_add_page(m->phys_addr);
2142         pc = (void *)PHYS_TO_DMAP(m->phys_addr);
2143         pc->pc_pmap = pmap;
2144         pc->pc_map[0] = PC_FREE0 & ~1ul;        /* preallocated bit 0 */
2145         pc->pc_map[1] = PC_FREE1;
2146         pc->pc_map[2] = PC_FREE2;
2147         mtx_lock(&pv_chunks_mutex);
2148         TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
2149         mtx_unlock(&pv_chunks_mutex);
2150         pv = &pc->pc_pventry[0];
2151         TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
2152         PV_STAT(atomic_add_long(&pv_entry_count, 1));
2153         PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1));
2154         return (pv);
2155 }
2156
2157 /*
2158  * Ensure that the number of spare PV entries in the specified pmap meets or
2159  * exceeds the given count, "needed".
2160  *
2161  * The given PV list lock may be released.
2162  */
2163 static void
2164 reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp)
2165 {
2166         struct pch new_tail;
2167         struct pv_chunk *pc;
2168         vm_page_t m;
2169         int avail, free;
2170         bool reclaimed;
2171
2172         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
2173         KASSERT(lockp != NULL, ("reserve_pv_entries: lockp is NULL"));
2174
2175         /*
2176          * Newly allocated PV chunks must be stored in a private list until
2177          * the required number of PV chunks have been allocated.  Otherwise,
2178          * reclaim_pv_chunk() could recycle one of these chunks.  In
2179          * contrast, these chunks must be added to the pmap upon allocation.
2180          */
2181         TAILQ_INIT(&new_tail);
2182 retry:
2183         avail = 0;
2184         TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) {
2185                 bit_count((bitstr_t *)pc->pc_map, 0,
2186                     sizeof(pc->pc_map) * NBBY, &free);
2187                 if (free == 0)
2188                         break;
2189                 avail += free;
2190                 if (avail >= needed)
2191                         break;
2192         }
2193         for (reclaimed = false; avail < needed; avail += _NPCPV) {
2194                 m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ |
2195                     VM_ALLOC_WIRED);
2196                 if (m == NULL) {
2197                         m = reclaim_pv_chunk(pmap, lockp);
2198                         if (m == NULL)
2199                                 goto retry;
2200                         reclaimed = true;
2201                 }
2202                 PV_STAT(atomic_add_int(&pc_chunk_count, 1));
2203                 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1));
2204                 dump_add_page(m->phys_addr);
2205                 pc = (void *)PHYS_TO_DMAP(m->phys_addr);
2206                 pc->pc_pmap = pmap;
2207                 pc->pc_map[0] = PC_FREE0;
2208                 pc->pc_map[1] = PC_FREE1;
2209                 pc->pc_map[2] = PC_FREE2;
2210                 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
2211                 TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru);
2212                 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV));
2213
2214                 /*
2215                  * The reclaim might have freed a chunk from the current pmap.
2216                  * If that chunk contained available entries, we need to
2217                  * re-count the number of available entries.
2218                  */
2219                 if (reclaimed)
2220                         goto retry;
2221         }
2222         if (!TAILQ_EMPTY(&new_tail)) {
2223                 mtx_lock(&pv_chunks_mutex);
2224                 TAILQ_CONCAT(&pv_chunks, &new_tail, pc_lru);
2225                 mtx_unlock(&pv_chunks_mutex);
2226         }
2227 }
2228
2229 /*
2230  * First find and then remove the pv entry for the specified pmap and virtual
2231  * address from the specified pv list.  Returns the pv entry if found and NULL
2232  * otherwise.  This operation can be performed on pv lists for either 4KB or
2233  * 2MB page mappings.
2234  */
2235 static __inline pv_entry_t
2236 pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
2237 {
2238         pv_entry_t pv;
2239
2240         TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
2241                 if (pmap == PV_PMAP(pv) && va == pv->pv_va) {
2242                         TAILQ_REMOVE(&pvh->pv_list, pv, pv_next);
2243                         pvh->pv_gen++;
2244                         break;
2245                 }
2246         }
2247         return (pv);
2248 }
2249
2250 /*
2251  * After demotion from a 2MB page mapping to 512 4KB page mappings,
2252  * destroy the pv entry for the 2MB page mapping and reinstantiate the pv
2253  * entries for each of the 4KB page mappings.
2254  */
2255 static void
2256 pmap_pv_demote_l2(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
2257     struct rwlock **lockp)
2258 {
2259         struct md_page *pvh;
2260         struct pv_chunk *pc;
2261         pv_entry_t pv;
2262         vm_offset_t va_last;
2263         vm_page_t m;
2264         int bit, field;
2265
2266         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
2267         KASSERT((va & L2_OFFSET) == 0,
2268             ("pmap_pv_demote_l2: va is not 2mpage aligned"));
2269         KASSERT((pa & L2_OFFSET) == 0,
2270             ("pmap_pv_demote_l2: pa is not 2mpage aligned"));
2271         CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
2272
2273         /*
2274          * Transfer the 2mpage's pv entry for this mapping to the first
2275          * page's pv list.  Once this transfer begins, the pv list lock
2276          * must not be released until the last pv entry is reinstantiated.
2277          */
2278         pvh = pa_to_pvh(pa);
2279         pv = pmap_pvh_remove(pvh, pmap, va);
2280         KASSERT(pv != NULL, ("pmap_pv_demote_l2: pv not found"));
2281         m = PHYS_TO_VM_PAGE(pa);
2282         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
2283         m->md.pv_gen++;
2284         /* Instantiate the remaining Ln_ENTRIES - 1 pv entries. */
2285         PV_STAT(atomic_add_long(&pv_entry_allocs, Ln_ENTRIES - 1));
2286         va_last = va + L2_SIZE - PAGE_SIZE;
2287         for (;;) {
2288                 pc = TAILQ_FIRST(&pmap->pm_pvchunk);
2289                 KASSERT(pc->pc_map[0] != 0 || pc->pc_map[1] != 0 ||
2290                     pc->pc_map[2] != 0, ("pmap_pv_demote_l2: missing spare"));
2291                 for (field = 0; field < _NPCM; field++) {
2292                         while (pc->pc_map[field]) {
2293                                 bit = ffsl(pc->pc_map[field]) - 1;
2294                                 pc->pc_map[field] &= ~(1ul << bit);
2295                                 pv = &pc->pc_pventry[field * 64 + bit];
2296                                 va += PAGE_SIZE;
2297                                 pv->pv_va = va;
2298                                 m++;
2299                                 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2300                             ("pmap_pv_demote_l2: page %p is not managed", m));
2301                                 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
2302                                 m->md.pv_gen++;
2303                                 if (va == va_last)
2304                                         goto out;
2305                         }
2306                 }
2307                 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
2308                 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
2309         }
2310 out:
2311         if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0 && pc->pc_map[2] == 0) {
2312                 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
2313                 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
2314         }
2315         PV_STAT(atomic_add_long(&pv_entry_count, Ln_ENTRIES - 1));
2316         PV_STAT(atomic_subtract_int(&pv_entry_spare, Ln_ENTRIES - 1));
2317 }
2318
2319 /*
2320  * First find and then destroy the pv entry for the specified pmap and virtual
2321  * address.  This operation can be performed on pv lists for either 4KB or 2MB
2322  * page mappings.
2323  */
2324 static void
2325 pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
2326 {
2327         pv_entry_t pv;
2328
2329         pv = pmap_pvh_remove(pvh, pmap, va);
2330         KASSERT(pv != NULL, ("pmap_pvh_free: pv not found"));
2331         free_pv_entry(pmap, pv);
2332 }
2333
2334 /*
2335  * Conditionally create the PV entry for a 4KB page mapping if the required
2336  * memory can be allocated without resorting to reclamation.
2337  */
2338 static boolean_t
2339 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m,
2340     struct rwlock **lockp)
2341 {
2342         pv_entry_t pv;
2343
2344         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
2345         /* Pass NULL instead of the lock pointer to disable reclamation. */
2346         if ((pv = get_pv_entry(pmap, NULL)) != NULL) {
2347                 pv->pv_va = va;
2348                 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
2349                 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
2350                 m->md.pv_gen++;
2351                 return (TRUE);
2352         } else
2353                 return (FALSE);
2354 }
2355
2356 /*
2357  * Create the PV entry for a 2MB page mapping.  Always returns true unless the
2358  * flag PMAP_ENTER_NORECLAIM is specified.  If that flag is specified, returns
2359  * false if the PV entry cannot be allocated without resorting to reclamation.
2360  */
2361 static bool
2362 pmap_pv_insert_l2(pmap_t pmap, vm_offset_t va, pd_entry_t l2e, u_int flags,
2363     struct rwlock **lockp)
2364 {
2365         struct md_page *pvh;
2366         pv_entry_t pv;
2367         vm_paddr_t pa;
2368
2369         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
2370         /* Pass NULL instead of the lock pointer to disable reclamation. */
2371         if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ?
2372             NULL : lockp)) == NULL)
2373                 return (false);
2374         pv->pv_va = va;
2375         pa = l2e & ~ATTR_MASK;
2376         CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
2377         pvh = pa_to_pvh(pa);
2378         TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next);
2379         pvh->pv_gen++;
2380         return (true);
2381 }
2382
2383 static void
2384 pmap_remove_kernel_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t va)
2385 {
2386         pt_entry_t newl2, oldl2;
2387         vm_page_t ml3;
2388         vm_paddr_t ml3pa;
2389
2390         KASSERT(!VIRT_IN_DMAP(va), ("removing direct mapping of %#lx", va));
2391         KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap));
2392         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
2393
2394         ml3 = pmap_remove_pt_page(pmap, va);
2395         if (ml3 == NULL)
2396                 panic("pmap_remove_kernel_l2: Missing pt page");
2397
2398         ml3pa = VM_PAGE_TO_PHYS(ml3);
2399         newl2 = ml3pa | L2_TABLE;
2400
2401         /*
2402          * If this page table page was unmapped by a promotion, then it
2403          * contains valid mappings.  Zero it to invalidate those mappings.
2404          */
2405         if (ml3->valid != 0)
2406                 pagezero((void *)PHYS_TO_DMAP(ml3pa));
2407
2408         /*
2409          * Demote the mapping.  The caller must have already invalidated the
2410          * mapping (i.e., the "break" in break-before-make).
2411          */
2412         oldl2 = pmap_load_store(l2, newl2);
2413         KASSERT(oldl2 == 0, ("%s: found existing mapping at %p: %#lx",
2414             __func__, l2, oldl2));
2415 }
2416
2417 /*
2418  * pmap_remove_l2: Do the things to unmap a level 2 superpage.
2419  */
2420 static int
2421 pmap_remove_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t sva,
2422     pd_entry_t l1e, struct spglist *free, struct rwlock **lockp)
2423 {
2424         struct md_page *pvh;
2425         pt_entry_t old_l2;
2426         vm_offset_t eva, va;
2427         vm_page_t m, ml3;
2428
2429         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
2430         KASSERT((sva & L2_OFFSET) == 0, ("pmap_remove_l2: sva is not aligned"));
2431         old_l2 = pmap_load_clear(l2);
2432         KASSERT((old_l2 & ATTR_DESCR_MASK) == L2_BLOCK,
2433             ("pmap_remove_l2: L2e %lx is not a block mapping", old_l2));
2434
2435         /*
2436          * Since a promotion must break the 4KB page mappings before making
2437          * the 2MB page mapping, a pmap_invalidate_page() suffices.
2438          */
2439         pmap_invalidate_page(pmap, sva);
2440
2441         if (old_l2 & ATTR_SW_WIRED)
2442                 pmap->pm_stats.wired_count -= L2_SIZE / PAGE_SIZE;
2443         pmap_resident_count_dec(pmap, L2_SIZE / PAGE_SIZE);
2444         if (old_l2 & ATTR_SW_MANAGED) {
2445                 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, old_l2 & ~ATTR_MASK);
2446                 pvh = pa_to_pvh(old_l2 & ~ATTR_MASK);
2447                 pmap_pvh_free(pvh, pmap, sva);
2448                 eva = sva + L2_SIZE;
2449                 for (va = sva, m = PHYS_TO_VM_PAGE(old_l2 & ~ATTR_MASK);
2450                     va < eva; va += PAGE_SIZE, m++) {
2451                         if (pmap_page_dirty(old_l2))
2452                                 vm_page_dirty(m);
2453                         if (old_l2 & ATTR_AF)
2454                                 vm_page_aflag_set(m, PGA_REFERENCED);
2455                         if (TAILQ_EMPTY(&m->md.pv_list) &&
2456                             TAILQ_EMPTY(&pvh->pv_list))
2457                                 vm_page_aflag_clear(m, PGA_WRITEABLE);
2458                 }
2459         }
2460         if (pmap == kernel_pmap) {
2461                 pmap_remove_kernel_l2(pmap, l2, sva);
2462         } else {
2463                 ml3 = pmap_remove_pt_page(pmap, sva);
2464                 if (ml3 != NULL) {
2465                         KASSERT(ml3->valid == VM_PAGE_BITS_ALL,
2466                             ("pmap_remove_l2: l3 page not promoted"));
2467                         pmap_resident_count_dec(pmap, 1);
2468                         KASSERT(ml3->wire_count == NL3PG,
2469                             ("pmap_remove_l2: l3 page wire count error"));
2470                         ml3->wire_count = 0;
2471                         pmap_add_delayed_free_list(ml3, free, FALSE);
2472                 }
2473         }
2474         return (pmap_unuse_pt(pmap, sva, l1e, free));
2475 }
2476
2477 /*
2478  * pmap_remove_l3: do the things to unmap a page in a process
2479  */
2480 static int
2481 pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t va,
2482     pd_entry_t l2e, struct spglist *free, struct rwlock **lockp)
2483 {
2484         struct md_page *pvh;
2485         pt_entry_t old_l3;
2486         vm_page_t m;
2487
2488         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
2489         old_l3 = pmap_load_clear(l3);
2490         pmap_invalidate_page(pmap, va);
2491         if (old_l3 & ATTR_SW_WIRED)
2492                 pmap->pm_stats.wired_count -= 1;
2493         pmap_resident_count_dec(pmap, 1);
2494         if (old_l3 & ATTR_SW_MANAGED) {
2495                 m = PHYS_TO_VM_PAGE(old_l3 & ~ATTR_MASK);
2496                 if (pmap_page_dirty(old_l3))
2497                         vm_page_dirty(m);
2498                 if (old_l3 & ATTR_AF)
2499                         vm_page_aflag_set(m, PGA_REFERENCED);
2500                 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
2501                 pmap_pvh_free(&m->md, pmap, va);
2502                 if (TAILQ_EMPTY(&m->md.pv_list) &&
2503                     (m->flags & PG_FICTITIOUS) == 0) {
2504                         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
2505                         if (TAILQ_EMPTY(&pvh->pv_list))
2506                                 vm_page_aflag_clear(m, PGA_WRITEABLE);
2507                 }
2508         }
2509         return (pmap_unuse_pt(pmap, va, l2e, free));
2510 }
2511
2512 /*
2513  * Remove the specified range of addresses from the L3 page table that is
2514  * identified by the given L2 entry.
2515  */
2516 static void
2517 pmap_remove_l3_range(pmap_t pmap, pd_entry_t l2e, vm_offset_t sva,
2518     vm_offset_t eva, struct spglist *free, struct rwlock **lockp)
2519 {
2520         struct md_page *pvh;
2521         struct rwlock *new_lock;
2522         pt_entry_t *l3, old_l3;
2523         vm_offset_t va;
2524         vm_page_t m;
2525
2526         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
2527         KASSERT(rounddown2(sva, L2_SIZE) + L2_SIZE == roundup2(eva, L2_SIZE),
2528             ("pmap_remove_l3_range: range crosses an L3 page table boundary"));
2529         va = eva;
2530         for (l3 = pmap_l2_to_l3(&l2e, sva); sva != eva; l3++, sva += L3_SIZE) {
2531                 if (!pmap_l3_valid(pmap_load(l3))) {
2532                         if (va != eva) {
2533                                 pmap_invalidate_range(pmap, va, sva);
2534                                 va = eva;
2535                         }
2536                         continue;
2537                 }
2538                 old_l3 = pmap_load_clear(l3);
2539                 if ((old_l3 & ATTR_SW_WIRED) != 0)
2540                         pmap->pm_stats.wired_count--;
2541                 pmap_resident_count_dec(pmap, 1);
2542                 if ((old_l3 & ATTR_SW_MANAGED) != 0) {
2543                         m = PHYS_TO_VM_PAGE(old_l3 & ~ATTR_MASK);
2544                         if (pmap_page_dirty(old_l3))
2545                                 vm_page_dirty(m);
2546                         if ((old_l3 & ATTR_AF) != 0)
2547                                 vm_page_aflag_set(m, PGA_REFERENCED);
2548                         new_lock = PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m));
2549                         if (new_lock != *lockp) {
2550                                 if (*lockp != NULL) {
2551                                         /*
2552                                          * Pending TLB invalidations must be
2553                                          * performed before the PV list lock is
2554                                          * released.  Otherwise, a concurrent
2555                                          * pmap_remove_all() on a physical page
2556                                          * could return while a stale TLB entry
2557                                          * still provides access to that page.
2558                                          */
2559                                         if (va != eva) {
2560                                                 pmap_invalidate_range(pmap, va,
2561                                                     sva);
2562                                                 va = eva;
2563                                         }
2564                                         rw_wunlock(*lockp);
2565                                 }
2566                                 *lockp = new_lock;
2567                                 rw_wlock(*lockp);
2568                         }
2569                         pmap_pvh_free(&m->md, pmap, sva);
2570                         if (TAILQ_EMPTY(&m->md.pv_list) &&
2571                             (m->flags & PG_FICTITIOUS) == 0) {
2572                                 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
2573                                 if (TAILQ_EMPTY(&pvh->pv_list))
2574                                         vm_page_aflag_clear(m, PGA_WRITEABLE);
2575                         }
2576                 }
2577                 if (va == eva)
2578                         va = sva;
2579                 if (pmap_unuse_pt(pmap, sva, l2e, free)) {
2580                         sva += L3_SIZE;
2581                         break;
2582                 }
2583         }
2584         if (va != eva)
2585                 pmap_invalidate_range(pmap, va, sva);
2586 }
2587
2588 /*
2589  *      Remove the given range of addresses from the specified map.
2590  *
2591  *      It is assumed that the start and end are properly
2592  *      rounded to the page size.
2593  */
2594 void
2595 pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
2596 {
2597         struct rwlock *lock;
2598         vm_offset_t va_next;
2599         pd_entry_t *l0, *l1, *l2;
2600         pt_entry_t l3_paddr;
2601         struct spglist free;
2602
2603         /*
2604          * Perform an unsynchronized read.  This is, however, safe.
2605          */
2606         if (pmap->pm_stats.resident_count == 0)
2607                 return;
2608
2609         SLIST_INIT(&free);
2610
2611         PMAP_LOCK(pmap);
2612
2613         lock = NULL;
2614         for (; sva < eva; sva = va_next) {
2615
2616                 if (pmap->pm_stats.resident_count == 0)
2617                         break;
2618
2619                 l0 = pmap_l0(pmap, sva);
2620                 if (pmap_load(l0) == 0) {
2621                         va_next = (sva + L0_SIZE) & ~L0_OFFSET;
2622                         if (va_next < sva)
2623                                 va_next = eva;
2624                         continue;
2625                 }
2626
2627                 l1 = pmap_l0_to_l1(l0, sva);
2628                 if (pmap_load(l1) == 0) {
2629                         va_next = (sva + L1_SIZE) & ~L1_OFFSET;
2630                         if (va_next < sva)
2631                                 va_next = eva;
2632                         continue;
2633                 }
2634
2635                 /*
2636                  * Calculate index for next page table.
2637                  */
2638                 va_next = (sva + L2_SIZE) & ~L2_OFFSET;
2639                 if (va_next < sva)
2640                         va_next = eva;
2641
2642                 l2 = pmap_l1_to_l2(l1, sva);
2643                 if (l2 == NULL)
2644                         continue;
2645
2646                 l3_paddr = pmap_load(l2);
2647
2648                 if ((l3_paddr & ATTR_DESCR_MASK) == L2_BLOCK) {
2649                         if (sva + L2_SIZE == va_next && eva >= va_next) {
2650                                 pmap_remove_l2(pmap, l2, sva, pmap_load(l1),
2651                                     &free, &lock);
2652                                 continue;
2653                         } else if (pmap_demote_l2_locked(pmap, l2, sva,
2654                             &lock) == NULL)
2655                                 continue;
2656                         l3_paddr = pmap_load(l2);
2657                 }
2658
2659                 /*
2660                  * Weed out invalid mappings.
2661                  */
2662                 if ((l3_paddr & ATTR_DESCR_MASK) != L2_TABLE)
2663                         continue;
2664
2665                 /*
2666                  * Limit our scan to either the end of the va represented
2667                  * by the current page table page, or to the end of the
2668                  * range being removed.
2669                  */
2670                 if (va_next > eva)
2671                         va_next = eva;
2672
2673                 pmap_remove_l3_range(pmap, l3_paddr, sva, va_next, &free,
2674                     &lock);
2675         }
2676         if (lock != NULL)
2677                 rw_wunlock(lock);
2678         PMAP_UNLOCK(pmap);
2679         vm_page_free_pages_toq(&free, true);
2680 }
2681
2682 /*
2683  *      Routine:        pmap_remove_all
2684  *      Function:
2685  *              Removes this physical page from
2686  *              all physical maps in which it resides.
2687  *              Reflects back modify bits to the pager.
2688  *
2689  *      Notes:
2690  *              Original versions of this routine were very
2691  *              inefficient because they iteratively called
2692  *              pmap_remove (slow...)
2693  */
2694
2695 void
2696 pmap_remove_all(vm_page_t m)
2697 {
2698         struct md_page *pvh;
2699         pv_entry_t pv;
2700         pmap_t pmap;
2701         struct rwlock *lock;
2702         pd_entry_t *pde, tpde;
2703         pt_entry_t *pte, tpte;
2704         vm_offset_t va;
2705         struct spglist free;
2706         int lvl, pvh_gen, md_gen;
2707
2708         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2709             ("pmap_remove_all: page %p is not managed", m));
2710         SLIST_INIT(&free);
2711         lock = VM_PAGE_TO_PV_LIST_LOCK(m);
2712         pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
2713             pa_to_pvh(VM_PAGE_TO_PHYS(m));
2714 retry:
2715         rw_wlock(lock);
2716         while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) {
2717                 pmap = PV_PMAP(pv);
2718                 if (!PMAP_TRYLOCK(pmap)) {
2719                         pvh_gen = pvh->pv_gen;
2720                         rw_wunlock(lock);
2721                         PMAP_LOCK(pmap);
2722                         rw_wlock(lock);
2723                         if (pvh_gen != pvh->pv_gen) {
2724                                 rw_wunlock(lock);
2725                                 PMAP_UNLOCK(pmap);
2726                                 goto retry;
2727                         }
2728                 }
2729                 va = pv->pv_va;
2730                 pte = pmap_pte(pmap, va, &lvl);
2731                 KASSERT(pte != NULL,
2732                     ("pmap_remove_all: no page table entry found"));
2733                 KASSERT(lvl == 2,
2734                     ("pmap_remove_all: invalid pte level %d", lvl));
2735
2736                 pmap_demote_l2_locked(pmap, pte, va, &lock);
2737                 PMAP_UNLOCK(pmap);
2738         }
2739         while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
2740                 pmap = PV_PMAP(pv);
2741                 if (!PMAP_TRYLOCK(pmap)) {
2742                         pvh_gen = pvh->pv_gen;
2743                         md_gen = m->md.pv_gen;
2744                         rw_wunlock(lock);
2745                         PMAP_LOCK(pmap);
2746                         rw_wlock(lock);
2747                         if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
2748                                 rw_wunlock(lock);
2749                                 PMAP_UNLOCK(pmap);
2750                                 goto retry;
2751                         }
2752                 }
2753                 pmap_resident_count_dec(pmap, 1);
2754
2755                 pde = pmap_pde(pmap, pv->pv_va, &lvl);
2756                 KASSERT(pde != NULL,
2757                     ("pmap_remove_all: no page directory entry found"));
2758                 KASSERT(lvl == 2,
2759                     ("pmap_remove_all: invalid pde level %d", lvl));
2760                 tpde = pmap_load(pde);
2761
2762                 pte = pmap_l2_to_l3(pde, pv->pv_va);
2763                 tpte = pmap_load(pte);
2764                 pmap_load_clear(pte);
2765                 pmap_invalidate_page(pmap, pv->pv_va);
2766                 if (tpte & ATTR_SW_WIRED)
2767                         pmap->pm_stats.wired_count--;
2768                 if ((tpte & ATTR_AF) != 0)
2769                         vm_page_aflag_set(m, PGA_REFERENCED);
2770
2771                 /*
2772                  * Update the vm_page_t clean and reference bits.
2773                  */
2774                 if (pmap_page_dirty(tpte))
2775                         vm_page_dirty(m);
2776                 pmap_unuse_pt(pmap, pv->pv_va, tpde, &free);
2777                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
2778                 m->md.pv_gen++;
2779                 free_pv_entry(pmap, pv);
2780                 PMAP_UNLOCK(pmap);
2781         }
2782         vm_page_aflag_clear(m, PGA_WRITEABLE);
2783         rw_wunlock(lock);
2784         vm_page_free_pages_toq(&free, true);
2785 }
2786
2787 /*
2788  * pmap_protect_l2: do the things to protect a 2MB page in a pmap
2789  */
2790 static void
2791 pmap_protect_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t sva, pt_entry_t nbits)
2792 {
2793         pd_entry_t old_l2;
2794         vm_page_t m, mt;
2795
2796         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
2797         KASSERT((sva & L2_OFFSET) == 0,
2798             ("pmap_protect_l2: sva is not 2mpage aligned"));
2799         old_l2 = pmap_load(l2);
2800         KASSERT((old_l2 & ATTR_DESCR_MASK) == L2_BLOCK,
2801             ("pmap_protect_l2: L2e %lx is not a block mapping", old_l2));
2802
2803         /*
2804          * Return if the L2 entry already has the desired access restrictions
2805          * in place.
2806          */
2807         if ((old_l2 | nbits) == old_l2)
2808                 return;
2809
2810         /*
2811          * When a dirty read/write superpage mapping is write protected,
2812          * update the dirty field of each of the superpage's constituent 4KB
2813          * pages.
2814          */
2815         if ((nbits & ATTR_AP(ATTR_AP_RO)) != 0 &&
2816             (old_l2 & ATTR_SW_MANAGED) != 0 &&
2817             pmap_page_dirty(old_l2)) {
2818                 m = PHYS_TO_VM_PAGE(old_l2 & ~ATTR_MASK);
2819                 for (mt = m; mt < &m[L2_SIZE / PAGE_SIZE]; mt++)
2820                         vm_page_dirty(mt);
2821         }
2822
2823         pmap_set(l2, nbits);
2824
2825         /*
2826          * Since a promotion must break the 4KB page mappings before making
2827          * the 2MB page mapping, a pmap_invalidate_page() suffices.
2828          */
2829         pmap_invalidate_page(pmap, sva);
2830 }
2831
2832 /*
2833  *      Set the physical protection on the
2834  *      specified range of this map as requested.
2835  */
2836 void
2837 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
2838 {
2839         vm_offset_t va, va_next;
2840         pd_entry_t *l0, *l1, *l2;
2841         pt_entry_t *l3p, l3, nbits;
2842
2843         KASSERT((prot & ~VM_PROT_ALL) == 0, ("invalid prot %x", prot));
2844         if (prot == VM_PROT_NONE) {
2845                 pmap_remove(pmap, sva, eva);
2846                 return;
2847         }
2848
2849         nbits = 0;
2850         if ((prot & VM_PROT_WRITE) == 0)
2851                 nbits |= ATTR_AP(ATTR_AP_RO);
2852         if ((prot & VM_PROT_EXECUTE) == 0)
2853                 nbits |= ATTR_XN;
2854         if (nbits == 0)
2855                 return;
2856
2857         PMAP_LOCK(pmap);
2858         for (; sva < eva; sva = va_next) {
2859
2860                 l0 = pmap_l0(pmap, sva);
2861                 if (pmap_load(l0) == 0) {
2862                         va_next = (sva + L0_SIZE) & ~L0_OFFSET;
2863                         if (va_next < sva)
2864                                 va_next = eva;
2865                         continue;
2866                 }
2867
2868                 l1 = pmap_l0_to_l1(l0, sva);
2869                 if (pmap_load(l1) == 0) {
2870                         va_next = (sva + L1_SIZE) & ~L1_OFFSET;
2871                         if (va_next < sva)
2872                                 va_next = eva;
2873                         continue;
2874                 }
2875
2876                 va_next = (sva + L2_SIZE) & ~L2_OFFSET;
2877                 if (va_next < sva)
2878                         va_next = eva;
2879
2880                 l2 = pmap_l1_to_l2(l1, sva);
2881                 if (pmap_load(l2) == 0)
2882                         continue;
2883
2884                 if ((pmap_load(l2) & ATTR_DESCR_MASK) == L2_BLOCK) {
2885                         if (sva + L2_SIZE == va_next && eva >= va_next) {
2886                                 pmap_protect_l2(pmap, l2, sva, nbits);
2887                                 continue;
2888                         } else if (pmap_demote_l2(pmap, l2, sva) == NULL)
2889                                 continue;
2890                 }
2891                 KASSERT((pmap_load(l2) & ATTR_DESCR_MASK) == L2_TABLE,
2892                     ("pmap_protect: Invalid L2 entry after demotion"));
2893
2894                 if (va_next > eva)
2895                         va_next = eva;
2896
2897                 va = va_next;
2898                 for (l3p = pmap_l2_to_l3(l2, sva); sva != va_next; l3p++,
2899                     sva += L3_SIZE) {
2900                         /*
2901                          * Go to the next L3 entry if the current one is
2902                          * invalid or already has the desired access
2903                          * restrictions in place.  (The latter case occurs
2904                          * frequently.  For example, in a "buildworld"
2905                          * workload, almost 1 out of 4 L3 entries already
2906                          * have the desired restrictions.)
2907                          */
2908                         l3 = pmap_load(l3p);
2909                         if (!pmap_l3_valid(l3) || (l3 | nbits) == l3) {
2910                                 if (va != va_next) {
2911                                         pmap_invalidate_range(pmap, va, sva);
2912                                         va = va_next;
2913                                 }
2914                                 continue;
2915                         }
2916                         if (va == va_next)
2917                                 va = sva;
2918
2919                         /*
2920                          * When a dirty read/write mapping is write protected,
2921                          * update the page's dirty field.
2922                          */
2923                         if ((nbits & ATTR_AP(ATTR_AP_RO)) != 0 &&
2924                             (l3 & ATTR_SW_MANAGED) != 0 &&
2925                             pmap_page_dirty(l3))
2926                                 vm_page_dirty(PHYS_TO_VM_PAGE(l3 & ~ATTR_MASK));
2927
2928                         pmap_set(l3p, nbits);
2929                 }
2930                 if (va != va_next)
2931                         pmap_invalidate_range(pmap, va, sva);
2932         }
2933         PMAP_UNLOCK(pmap);
2934 }
2935
2936 /*
2937  * Inserts the specified page table page into the specified pmap's collection
2938  * of idle page table pages.  Each of a pmap's page table pages is responsible
2939  * for mapping a distinct range of virtual addresses.  The pmap's collection is
2940  * ordered by this virtual address range.
2941  *
2942  * If "promoted" is false, then the page table page "mpte" must be zero filled.
2943  */
2944 static __inline int
2945 pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted)
2946 {
2947
2948         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
2949         mpte->valid = promoted ? VM_PAGE_BITS_ALL : 0;
2950         return (vm_radix_insert(&pmap->pm_root, mpte));
2951 }
2952
2953 /*
2954  * Removes the page table page mapping the specified virtual address from the
2955  * specified pmap's collection of idle page table pages, and returns it.
2956  * Otherwise, returns NULL if there is no page table page corresponding to the
2957  * specified virtual address.
2958  */
2959 static __inline vm_page_t
2960 pmap_remove_pt_page(pmap_t pmap, vm_offset_t va)
2961 {
2962
2963         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
2964         return (vm_radix_remove(&pmap->pm_root, pmap_l2_pindex(va)));
2965 }
2966
2967 /*
2968  * Performs a break-before-make update of a pmap entry. This is needed when
2969  * either promoting or demoting pages to ensure the TLB doesn't get into an
2970  * inconsistent state.
2971  */
2972 static void
2973 pmap_update_entry(pmap_t pmap, pd_entry_t *pte, pd_entry_t newpte,
2974     vm_offset_t va, vm_size_t size)
2975 {
2976         register_t intr;
2977
2978         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
2979
2980         /*
2981          * Ensure we don't get switched out with the page table in an
2982          * inconsistent state. We also need to ensure no interrupts fire
2983          * as they may make use of an address we are about to invalidate.
2984          */
2985         intr = intr_disable();
2986         critical_enter();
2987
2988         /* Clear the old mapping */
2989         pmap_load_clear(pte);
2990         pmap_invalidate_range_nopin(pmap, va, va + size);
2991
2992         /* Create the new mapping */
2993         pmap_load_store(pte, newpte);
2994         dsb(ishst);
2995
2996         critical_exit();
2997         intr_restore(intr);
2998 }
2999
3000 #if VM_NRESERVLEVEL > 0
3001 /*
3002  * After promotion from 512 4KB page mappings to a single 2MB page mapping,
3003  * replace the many pv entries for the 4KB page mappings by a single pv entry
3004  * for the 2MB page mapping.
3005  */
3006 static void
3007 pmap_pv_promote_l2(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
3008     struct rwlock **lockp)
3009 {
3010         struct md_page *pvh;
3011         pv_entry_t pv;
3012         vm_offset_t va_last;
3013         vm_page_t m;
3014
3015         KASSERT((pa & L2_OFFSET) == 0,
3016             ("pmap_pv_promote_l2: pa is not 2mpage aligned"));
3017         CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
3018
3019         /*
3020          * Transfer the first page's pv entry for this mapping to the 2mpage's
3021          * pv list.  Aside from avoiding the cost of a call to get_pv_entry(),
3022          * a transfer avoids the possibility that get_pv_entry() calls
3023          * reclaim_pv_chunk() and that reclaim_pv_chunk() removes one of the
3024          * mappings that is being promoted.
3025          */
3026         m = PHYS_TO_VM_PAGE(pa);
3027         va = va & ~L2_OFFSET;
3028         pv = pmap_pvh_remove(&m->md, pmap, va);
3029         KASSERT(pv != NULL, ("pmap_pv_promote_l2: pv not found"));
3030         pvh = pa_to_pvh(pa);
3031         TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next);
3032         pvh->pv_gen++;
3033         /* Free the remaining NPTEPG - 1 pv entries. */
3034         va_last = va + L2_SIZE - PAGE_SIZE;
3035         do {
3036                 m++;
3037                 va += PAGE_SIZE;
3038                 pmap_pvh_free(&m->md, pmap, va);
3039         } while (va < va_last);
3040 }
3041
3042 /*
3043  * Tries to promote the 512, contiguous 4KB page mappings that are within a
3044  * single level 2 table entry to a single 2MB page mapping.  For promotion
3045  * to occur, two conditions must be met: (1) the 4KB page mappings must map
3046  * aligned, contiguous physical memory and (2) the 4KB page mappings must have
3047  * identical characteristics.
3048  */
3049 static void
3050 pmap_promote_l2(pmap_t pmap, pd_entry_t *l2, vm_offset_t va,
3051     struct rwlock **lockp)
3052 {
3053         pt_entry_t *firstl3, *l3, newl2, oldl3, pa;
3054         vm_page_t mpte;
3055         vm_offset_t sva;
3056
3057         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3058
3059         sva = va & ~L2_OFFSET;
3060         firstl3 = pmap_l2_to_l3(l2, sva);
3061         newl2 = pmap_load(firstl3);
3062
3063         /* Check the alingment is valid */
3064         if (((newl2 & ~ATTR_MASK) & L2_OFFSET) != 0) {
3065                 atomic_add_long(&pmap_l2_p_failures, 1);
3066                 CTR2(KTR_PMAP, "pmap_promote_l2: failure for va %#lx"
3067                     " in pmap %p", va, pmap);
3068                 return;
3069         }
3070
3071         pa = newl2 + L2_SIZE - PAGE_SIZE;
3072         for (l3 = firstl3 + NL3PG - 1; l3 > firstl3; l3--) {
3073                 oldl3 = pmap_load(l3);
3074                 if (oldl3 != pa) {
3075                         atomic_add_long(&pmap_l2_p_failures, 1);
3076                         CTR2(KTR_PMAP, "pmap_promote_l2: failure for va %#lx"
3077                             " in pmap %p", va, pmap);
3078                         return;
3079                 }
3080                 pa -= PAGE_SIZE;
3081         }
3082
3083         /*
3084          * Save the page table page in its current state until the L2
3085          * mapping the superpage is demoted by pmap_demote_l2() or
3086          * destroyed by pmap_remove_l3().
3087          */
3088         mpte = PHYS_TO_VM_PAGE(pmap_load(l2) & ~ATTR_MASK);
3089         KASSERT(mpte >= vm_page_array &&
3090             mpte < &vm_page_array[vm_page_array_size],
3091             ("pmap_promote_l2: page table page is out of range"));
3092         KASSERT(mpte->pindex == pmap_l2_pindex(va),
3093             ("pmap_promote_l2: page table page's pindex is wrong"));
3094         if (pmap_insert_pt_page(pmap, mpte, true)) {
3095                 atomic_add_long(&pmap_l2_p_failures, 1);
3096                 CTR2(KTR_PMAP,
3097                     "pmap_promote_l2: failure for va %#lx in pmap %p", va,
3098                     pmap);
3099                 return;
3100         }
3101
3102         if ((newl2 & ATTR_SW_MANAGED) != 0)
3103                 pmap_pv_promote_l2(pmap, va, newl2 & ~ATTR_MASK, lockp);
3104
3105         newl2 &= ~ATTR_DESCR_MASK;
3106         newl2 |= L2_BLOCK;
3107
3108         pmap_update_entry(pmap, l2, newl2, sva, L2_SIZE);
3109
3110         atomic_add_long(&pmap_l2_promotions, 1);
3111         CTR2(KTR_PMAP, "pmap_promote_l2: success for va %#lx in pmap %p", va,
3112                     pmap);
3113 }
3114 #endif /* VM_NRESERVLEVEL > 0 */
3115
3116 /*
3117  *      Insert the given physical page (p) at
3118  *      the specified virtual address (v) in the
3119  *      target physical map with the protection requested.
3120  *
3121  *      If specified, the page will be wired down, meaning
3122  *      that the related pte can not be reclaimed.
3123  *
3124  *      NB:  This is the only routine which MAY NOT lazy-evaluate
3125  *      or lose information.  That is, this routine must actually
3126  *      insert this page into the given map NOW.
3127  */
3128 int
3129 pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
3130     u_int flags, int8_t psind)
3131 {
3132         struct rwlock *lock;
3133         pd_entry_t *pde;
3134         pt_entry_t new_l3, orig_l3;
3135         pt_entry_t *l2, *l3;
3136         pv_entry_t pv;
3137         vm_paddr_t opa, pa, l1_pa, l2_pa, l3_pa;
3138         vm_page_t mpte, om, l1_m, l2_m, l3_m;
3139         boolean_t nosleep;
3140         int lvl, rv;
3141
3142         va = trunc_page(va);
3143         if ((m->oflags & VPO_UNMANAGED) == 0 && !vm_page_xbusied(m))
3144                 VM_OBJECT_ASSERT_LOCKED(m->object);
3145         pa = VM_PAGE_TO_PHYS(m);
3146         new_l3 = (pt_entry_t)(pa | ATTR_DEFAULT | ATTR_IDX(m->md.pv_memattr) |
3147             L3_PAGE);
3148         if ((prot & VM_PROT_WRITE) == 0)
3149                 new_l3 |= ATTR_AP(ATTR_AP_RO);
3150         if ((prot & VM_PROT_EXECUTE) == 0 || m->md.pv_memattr == DEVICE_MEMORY)
3151                 new_l3 |= ATTR_XN;
3152         if ((flags & PMAP_ENTER_WIRED) != 0)
3153                 new_l3 |= ATTR_SW_WIRED;
3154         if (va < VM_MAXUSER_ADDRESS)
3155                 new_l3 |= ATTR_AP(ATTR_AP_USER) | ATTR_PXN;
3156         if ((m->oflags & VPO_UNMANAGED) == 0)
3157                 new_l3 |= ATTR_SW_MANAGED;
3158
3159         CTR2(KTR_PMAP, "pmap_enter: %.16lx -> %.16lx", va, pa);
3160
3161         lock = NULL;
3162         mpte = NULL;
3163         PMAP_LOCK(pmap);
3164         if (psind == 1) {
3165                 /* Assert the required virtual and physical alignment. */
3166                 KASSERT((va & L2_OFFSET) == 0, ("pmap_enter: va unaligned"));
3167                 KASSERT(m->psind > 0, ("pmap_enter: m->psind < psind"));
3168                 rv = pmap_enter_l2(pmap, va, (new_l3 & ~L3_PAGE) | L2_BLOCK,
3169                     flags, m, &lock);
3170                 goto out;
3171         }
3172
3173         pde = pmap_pde(pmap, va, &lvl);
3174         if (pde != NULL && lvl == 1) {
3175                 l2 = pmap_l1_to_l2(pde, va);
3176                 if ((pmap_load(l2) & ATTR_DESCR_MASK) == L2_BLOCK &&
3177                     (l3 = pmap_demote_l2_locked(pmap, l2, va, &lock)) != NULL) {
3178                         l3 = &l3[pmap_l3_index(va)];
3179                         if (va < VM_MAXUSER_ADDRESS) {
3180                                 mpte = PHYS_TO_VM_PAGE(
3181                                     pmap_load(l2) & ~ATTR_MASK);
3182                                 mpte->wire_count++;
3183                         }
3184                         goto havel3;
3185                 }
3186         }
3187
3188         if (va < VM_MAXUSER_ADDRESS) {
3189                 nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0;
3190                 mpte = pmap_alloc_l3(pmap, va, nosleep ? NULL : &lock);
3191                 if (mpte == NULL && nosleep) {
3192                         CTR0(KTR_PMAP, "pmap_enter: mpte == NULL");
3193                         if (lock != NULL)
3194                                 rw_wunlock(lock);
3195                         PMAP_UNLOCK(pmap);
3196                         return (KERN_RESOURCE_SHORTAGE);
3197                 }
3198                 pde = pmap_pde(pmap, va, &lvl);
3199                 KASSERT(pde != NULL,
3200                     ("pmap_enter: Invalid page entry, va: 0x%lx", va));
3201                 KASSERT(lvl == 2,
3202                     ("pmap_enter: Invalid level %d", lvl));
3203         } else {
3204                 /*
3205                  * If we get a level 2 pde it must point to a level 3 entry
3206                  * otherwise we will need to create the intermediate tables
3207                  */
3208                 if (lvl < 2) {
3209                         switch (lvl) {
3210                         default:
3211                         case -1:
3212                                 /* Get the l0 pde to update */
3213                                 pde = pmap_l0(pmap, va);
3214                                 KASSERT(pde != NULL, ("..."));
3215
3216                                 l1_m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL |
3217                                     VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
3218                                     VM_ALLOC_ZERO);
3219                                 if (l1_m == NULL)
3220                                         panic("pmap_enter: l1 pte_m == NULL");
3221                                 if ((l1_m->flags & PG_ZERO) == 0)
3222                                         pmap_zero_page(l1_m);
3223
3224                                 l1_pa = VM_PAGE_TO_PHYS(l1_m);
3225                                 pmap_load_store(pde, l1_pa | L0_TABLE);
3226                                 /* FALLTHROUGH */
3227                         case 0:
3228                                 /* Get the l1 pde to update */
3229                                 pde = pmap_l1_to_l2(pde, va);
3230                                 KASSERT(pde != NULL, ("..."));
3231
3232                                 l2_m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL |
3233                                     VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
3234                                     VM_ALLOC_ZERO);
3235                                 if (l2_m == NULL)
3236                                         panic("pmap_enter: l2 pte_m == NULL");
3237                                 if ((l2_m->flags & PG_ZERO) == 0)
3238                                         pmap_zero_page(l2_m);
3239
3240                                 l2_pa = VM_PAGE_TO_PHYS(l2_m);
3241                                 pmap_load_store(pde, l2_pa | L1_TABLE);
3242                                 /* FALLTHROUGH */
3243                         case 1:
3244                                 /* Get the l2 pde to update */
3245                                 pde = pmap_l1_to_l2(pde, va);
3246                                 KASSERT(pde != NULL, ("..."));
3247
3248                                 l3_m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL |
3249                                     VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
3250                                     VM_ALLOC_ZERO);
3251                                 if (l3_m == NULL)
3252                                         panic("pmap_enter: l3 pte_m == NULL");
3253                                 if ((l3_m->flags & PG_ZERO) == 0)
3254                                         pmap_zero_page(l3_m);
3255
3256                                 l3_pa = VM_PAGE_TO_PHYS(l3_m);
3257                                 pmap_load_store(pde, l3_pa | L2_TABLE);
3258                                 break;
3259                         }
3260                 }
3261         }
3262         l3 = pmap_l2_to_l3(pde, va);
3263
3264 havel3:
3265         orig_l3 = pmap_load(l3);
3266         opa = orig_l3 & ~ATTR_MASK;
3267         pv = NULL;
3268
3269         /*
3270          * Is the specified virtual address already mapped?
3271          */
3272         if (pmap_l3_valid(orig_l3)) {
3273                 /*
3274                  * Wiring change, just update stats. We don't worry about
3275                  * wiring PT pages as they remain resident as long as there
3276                  * are valid mappings in them. Hence, if a user page is wired,
3277                  * the PT page will be also.
3278                  */
3279                 if ((flags & PMAP_ENTER_WIRED) != 0 &&
3280                     (orig_l3 & ATTR_SW_WIRED) == 0)
3281                         pmap->pm_stats.wired_count++;
3282                 else if ((flags & PMAP_ENTER_WIRED) == 0 &&
3283                     (orig_l3 & ATTR_SW_WIRED) != 0)
3284                         pmap->pm_stats.wired_count--;
3285
3286                 /*
3287                  * Remove the extra PT page reference.
3288                  */
3289                 if (mpte != NULL) {
3290                         mpte->wire_count--;
3291                         KASSERT(mpte->wire_count > 0,
3292                             ("pmap_enter: missing reference to page table page,"
3293                              " va: 0x%lx", va));
3294                 }
3295
3296                 /*
3297                  * Has the physical page changed?
3298                  */
3299                 if (opa == pa) {
3300                         /*
3301                          * No, might be a protection or wiring change.
3302                          */
3303                         if ((orig_l3 & ATTR_SW_MANAGED) != 0) {
3304                                 if ((new_l3 & ATTR_AP(ATTR_AP_RW)) ==
3305                                     ATTR_AP(ATTR_AP_RW)) {
3306                                         vm_page_aflag_set(m, PGA_WRITEABLE);
3307                                 }
3308                         }
3309                         goto validate;
3310                 }
3311
3312                 /*
3313                  * The physical page has changed.
3314                  */
3315                 (void)pmap_load_clear(l3);
3316                 KASSERT((orig_l3 & ~ATTR_MASK) == opa,
3317                     ("pmap_enter: unexpected pa update for %#lx", va));
3318                 if ((orig_l3 & ATTR_SW_MANAGED) != 0) {
3319                         om = PHYS_TO_VM_PAGE(opa);
3320
3321                         /*
3322                          * The pmap lock is sufficient to synchronize with
3323                          * concurrent calls to pmap_page_test_mappings() and
3324                          * pmap_ts_referenced().
3325                          */
3326                         if (pmap_page_dirty(orig_l3))
3327                                 vm_page_dirty(om);
3328                         if ((orig_l3 & ATTR_AF) != 0)
3329                                 vm_page_aflag_set(om, PGA_REFERENCED);
3330                         CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, opa);
3331                         pv = pmap_pvh_remove(&om->md, pmap, va);
3332                         if ((m->oflags & VPO_UNMANAGED) != 0)
3333                                 free_pv_entry(pmap, pv);
3334                         if ((om->aflags & PGA_WRITEABLE) != 0 &&
3335                             TAILQ_EMPTY(&om->md.pv_list) &&
3336                             ((om->flags & PG_FICTITIOUS) != 0 ||
3337                             TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list)))
3338                                 vm_page_aflag_clear(om, PGA_WRITEABLE);
3339                 }
3340                 pmap_invalidate_page(pmap, va);
3341                 orig_l3 = 0;
3342         } else {
3343                 /*
3344                  * Increment the counters.
3345                  */
3346                 if ((new_l3 & ATTR_SW_WIRED) != 0)
3347                         pmap->pm_stats.wired_count++;
3348                 pmap_resident_count_inc(pmap, 1);
3349         }
3350         /*
3351          * Enter on the PV list if part of our managed memory.
3352          */
3353         if ((m->oflags & VPO_UNMANAGED) == 0) {
3354                 if (pv == NULL) {
3355                         pv = get_pv_entry(pmap, &lock);
3356                         pv->pv_va = va;
3357                 }
3358                 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa);
3359                 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
3360                 m->md.pv_gen++;
3361                 if ((new_l3 & ATTR_AP_RW_BIT) == ATTR_AP(ATTR_AP_RW))
3362                         vm_page_aflag_set(m, PGA_WRITEABLE);
3363         }
3364
3365 validate:
3366         /*
3367          * Sync icache if exec permission and attribute VM_MEMATTR_WRITE_BACK
3368          * is set. Do it now, before the mapping is stored and made
3369          * valid for hardware table walk. If done later, then other can
3370          * access this page before caches are properly synced.
3371          * Don't do it for kernel memory which is mapped with exec
3372          * permission even if the memory isn't going to hold executable
3373          * code. The only time when icache sync is needed is after
3374          * kernel module is loaded and the relocation info is processed.
3375          * And it's done in elf_cpu_load_file().
3376         */
3377         if ((prot & VM_PROT_EXECUTE) &&  pmap != kernel_pmap &&
3378             m->md.pv_memattr == VM_MEMATTR_WRITE_BACK &&
3379             (opa != pa || (orig_l3 & ATTR_XN)))
3380                 cpu_icache_sync_range(PHYS_TO_DMAP(pa), PAGE_SIZE);
3381
3382         /*
3383          * Update the L3 entry
3384          */
3385         if (pmap_l3_valid(orig_l3)) {
3386                 KASSERT(opa == pa, ("pmap_enter: invalid update"));
3387                 if ((orig_l3 & ~ATTR_AF) != (new_l3 & ~ATTR_AF)) {
3388                         /* same PA, different attributes */
3389                         pmap_load_store(l3, new_l3);
3390                         pmap_invalidate_page(pmap, va);
3391                         if (pmap_page_dirty(orig_l3) &&
3392                             (orig_l3 & ATTR_SW_MANAGED) != 0)
3393                                 vm_page_dirty(m);
3394                 } else {
3395                         /*
3396                          * orig_l3 == new_l3
3397                          * This can happens if multiple threads simultaneously
3398                          * access not yet mapped page. This bad for performance
3399                          * since this can cause full demotion-NOP-promotion
3400                          * cycle.
3401                          * Another possible reasons are:
3402                          * - VM and pmap memory layout are diverged
3403                          * - tlb flush is missing somewhere and CPU doesn't see
3404                          *   actual mapping.
3405                          */
3406                         CTR4(KTR_PMAP, "%s: already mapped page - "
3407                             "pmap %p va 0x%#lx pte 0x%lx",
3408                             __func__, pmap, va, new_l3);
3409                 }
3410         } else {
3411                 /* New mapping */
3412                 pmap_load_store(l3, new_l3);
3413                 dsb(ishst);
3414         }
3415
3416 #if VM_NRESERVLEVEL > 0
3417         if (pmap != pmap_kernel() &&
3418             (mpte == NULL || mpte->wire_count == NL3PG) &&
3419             pmap_ps_enabled(pmap) &&
3420             (m->flags & PG_FICTITIOUS) == 0 &&
3421             vm_reserv_level_iffullpop(m) == 0) {
3422                 pmap_promote_l2(pmap, pde, va, &lock);
3423         }
3424 #endif
3425
3426         rv = KERN_SUCCESS;
3427 out:
3428         if (lock != NULL)
3429                 rw_wunlock(lock);
3430         PMAP_UNLOCK(pmap);
3431         return (rv);
3432 }
3433
3434 /*
3435  * Tries to create a read- and/or execute-only 2MB page mapping.  Returns true
3436  * if successful.  Returns false if (1) a page table page cannot be allocated
3437  * without sleeping, (2) a mapping already exists at the specified virtual
3438  * address, or (3) a PV entry cannot be allocated without reclaiming another
3439  * PV entry.
3440  */
3441 static bool
3442 pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
3443     struct rwlock **lockp)
3444 {
3445         pd_entry_t new_l2;
3446
3447         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3448
3449         new_l2 = (pd_entry_t)(VM_PAGE_TO_PHYS(m) | ATTR_DEFAULT |
3450             ATTR_IDX(m->md.pv_memattr) | ATTR_AP(ATTR_AP_RO) | L2_BLOCK);
3451         if ((m->oflags & VPO_UNMANAGED) == 0)
3452                 new_l2 |= ATTR_SW_MANAGED;
3453         if ((prot & VM_PROT_EXECUTE) == 0 || m->md.pv_memattr == DEVICE_MEMORY)
3454                 new_l2 |= ATTR_XN;
3455         if (va < VM_MAXUSER_ADDRESS)
3456                 new_l2 |= ATTR_AP(ATTR_AP_USER) | ATTR_PXN;
3457         return (pmap_enter_l2(pmap, va, new_l2, PMAP_ENTER_NOSLEEP |
3458             PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, NULL, lockp) ==
3459             KERN_SUCCESS);
3460 }
3461
3462 /*
3463  * Tries to create the specified 2MB page mapping.  Returns KERN_SUCCESS if
3464  * the mapping was created, and either KERN_FAILURE or KERN_RESOURCE_SHORTAGE
3465  * otherwise.  Returns KERN_FAILURE if PMAP_ENTER_NOREPLACE was specified and
3466  * a mapping already exists at the specified virtual address.  Returns
3467  * KERN_RESOURCE_SHORTAGE if PMAP_ENTER_NOSLEEP was specified and a page table
3468  * page allocation failed.  Returns KERN_RESOURCE_SHORTAGE if
3469  * PMAP_ENTER_NORECLAIM was specified and a PV entry allocation failed.
3470  *
3471  * The parameter "m" is only used when creating a managed, writeable mapping.
3472  */
3473 static int
3474 pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t new_l2, u_int flags,
3475     vm_page_t m, struct rwlock **lockp)
3476 {
3477         struct spglist free;
3478         pd_entry_t *l2, old_l2;
3479         vm_page_t l2pg, mt;
3480
3481         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3482
3483         if ((l2pg = pmap_alloc_l2(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ?
3484             NULL : lockp)) == NULL) {
3485                 CTR2(KTR_PMAP, "pmap_enter_l2: failure for va %#lx in pmap %p",
3486                     va, pmap);
3487                 return (KERN_RESOURCE_SHORTAGE);
3488         }
3489
3490         l2 = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(l2pg));
3491         l2 = &l2[pmap_l2_index(va)];
3492         if ((old_l2 = pmap_load(l2)) != 0) {
3493                 KASSERT(l2pg->wire_count > 1,
3494                     ("pmap_enter_l2: l2pg's wire count is too low"));
3495                 if ((flags & PMAP_ENTER_NOREPLACE) != 0) {
3496                         l2pg->wire_count--;
3497                         CTR2(KTR_PMAP,
3498                             "pmap_enter_l2: failure for va %#lx in pmap %p",
3499                             va, pmap);
3500                         return (KERN_FAILURE);
3501                 }
3502                 SLIST_INIT(&free);
3503                 if ((old_l2 & ATTR_DESCR_MASK) == L2_BLOCK)
3504                         (void)pmap_remove_l2(pmap, l2, va,
3505                             pmap_load(pmap_l1(pmap, va)), &free, lockp);
3506                 else
3507                         pmap_remove_l3_range(pmap, old_l2, va, va + L2_SIZE,
3508                             &free, lockp);
3509                 vm_page_free_pages_toq(&free, true);
3510                 if (va >= VM_MAXUSER_ADDRESS) {
3511                         /*
3512                          * Both pmap_remove_l2() and pmap_remove_l3() will
3513                          * leave the kernel page table page zero filled.
3514                          */
3515                         mt = PHYS_TO_VM_PAGE(pmap_load(l2) & ~ATTR_MASK);
3516                         if (pmap_insert_pt_page(pmap, mt, false))
3517                                 panic("pmap_enter_l2: trie insert failed");
3518                 } else
3519                         KASSERT(pmap_load(l2) == 0,
3520                             ("pmap_enter_l2: non-zero L2 entry %p", l2));
3521         }
3522
3523         if ((new_l2 & ATTR_SW_MANAGED) != 0) {
3524                 /*
3525                  * Abort this mapping if its PV entry could not be created.
3526                  */
3527                 if (!pmap_pv_insert_l2(pmap, va, new_l2, flags, lockp)) {
3528                         SLIST_INIT(&free);
3529                         if (pmap_unwire_l3(pmap, va, l2pg, &free)) {
3530                                 /*
3531                                  * Although "va" is not mapped, paging-structure
3532                                  * caches could nonetheless have entries that
3533                                  * refer to the freed page table pages.
3534                                  * Invalidate those entries.
3535                                  */
3536                                 pmap_invalidate_page(pmap, va);
3537                                 vm_page_free_pages_toq(&free, true);
3538                         }
3539                         CTR2(KTR_PMAP,
3540                             "pmap_enter_l2: failure for va %#lx in pmap %p",
3541                             va, pmap);
3542                         return (KERN_RESOURCE_SHORTAGE);
3543                 }
3544                 if ((new_l2 & ATTR_AP_RW_BIT) == ATTR_AP(ATTR_AP_RW))
3545                         for (mt = m; mt < &m[L2_SIZE / PAGE_SIZE]; mt++)
3546                                 vm_page_aflag_set(mt, PGA_WRITEABLE);
3547         }
3548
3549         /*
3550          * Increment counters.
3551          */
3552         if ((new_l2 & ATTR_SW_WIRED) != 0)
3553                 pmap->pm_stats.wired_count += L2_SIZE / PAGE_SIZE;
3554         pmap->pm_stats.resident_count += L2_SIZE / PAGE_SIZE;
3555
3556         /*
3557          * Map the superpage.
3558          */
3559         (void)pmap_load_store(l2, new_l2);
3560         dsb(ishst);
3561
3562         atomic_add_long(&pmap_l2_mappings, 1);
3563         CTR2(KTR_PMAP, "pmap_enter_l2: success for va %#lx in pmap %p",
3564             va, pmap);
3565
3566         return (KERN_SUCCESS);
3567 }
3568
3569 /*
3570  * Maps a sequence of resident pages belonging to the same object.
3571  * The sequence begins with the given page m_start.  This page is
3572  * mapped at the given virtual address start.  Each subsequent page is
3573  * mapped at a virtual address that is offset from start by the same
3574  * amount as the page is offset from m_start within the object.  The
3575  * last page in the sequence is the page with the largest offset from
3576  * m_start that can be mapped at a virtual address less than the given
3577  * virtual address end.  Not every virtual page between start and end
3578  * is mapped; only those for which a resident page exists with the
3579  * corresponding offset from m_start are mapped.
3580  */
3581 void
3582 pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,
3583     vm_page_t m_start, vm_prot_t prot)
3584 {
3585         struct rwlock *lock;
3586         vm_offset_t va;
3587         vm_page_t m, mpte;
3588         vm_pindex_t diff, psize;
3589
3590         VM_OBJECT_ASSERT_LOCKED(m_start->object);
3591
3592         psize = atop(end - start);
3593         mpte = NULL;
3594         m = m_start;
3595         lock = NULL;
3596         PMAP_LOCK(pmap);
3597         while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
3598                 va = start + ptoa(diff);
3599                 if ((va & L2_OFFSET) == 0 && va + L2_SIZE <= end &&
3600                     m->psind == 1 && pmap_ps_enabled(pmap) &&
3601                     pmap_enter_2mpage(pmap, va, m, prot, &lock))
3602                         m = &m[L2_SIZE / PAGE_SIZE - 1];
3603                 else
3604                         mpte = pmap_enter_quick_locked(pmap, va, m, prot, mpte,
3605                             &lock);
3606                 m = TAILQ_NEXT(m, listq);
3607         }
3608         if (lock != NULL)
3609                 rw_wunlock(lock);
3610         PMAP_UNLOCK(pmap);
3611 }
3612
3613 /*
3614  * this code makes some *MAJOR* assumptions:
3615  * 1. Current pmap & pmap exists.
3616  * 2. Not wired.
3617  * 3. Read access.
3618  * 4. No page table pages.
3619  * but is *MUCH* faster than pmap_enter...
3620  */
3621
3622 void
3623 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
3624 {
3625         struct rwlock *lock;
3626
3627         lock = NULL;
3628         PMAP_LOCK(pmap);
3629         (void)pmap_enter_quick_locked(pmap, va, m, prot, NULL, &lock);
3630         if (lock != NULL)
3631                 rw_wunlock(lock);
3632         PMAP_UNLOCK(pmap);
3633 }
3634
3635 static vm_page_t
3636 pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
3637     vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp)
3638 {
3639         struct spglist free;
3640         pd_entry_t *pde;
3641         pt_entry_t *l2, *l3, l3_val;
3642         vm_paddr_t pa;
3643         int lvl;
3644
3645         KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva ||
3646             (m->oflags & VPO_UNMANAGED) != 0,
3647             ("pmap_enter_quick_locked: managed mapping within the clean submap"));
3648         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3649
3650         CTR2(KTR_PMAP, "pmap_enter_quick_locked: %p %lx", pmap, va);
3651         /*
3652          * In the case that a page table page is not
3653          * resident, we are creating it here.
3654          */
3655         if (va < VM_MAXUSER_ADDRESS) {
3656                 vm_pindex_t l2pindex;
3657
3658                 /*
3659                  * Calculate pagetable page index
3660                  */
3661                 l2pindex = pmap_l2_pindex(va);
3662                 if (mpte && (mpte->pindex == l2pindex)) {
3663                         mpte->wire_count++;
3664                 } else {
3665                         /*
3666                          * Get the l2 entry
3667                          */
3668                         pde = pmap_pde(pmap, va, &lvl);
3669
3670                         /*
3671                          * If the page table page is mapped, we just increment
3672                          * the hold count, and activate it.  Otherwise, we
3673                          * attempt to allocate a page table page.  If this
3674                          * attempt fails, we don't retry.  Instead, we give up.
3675                          */
3676                         if (lvl == 1) {
3677                                 l2 = pmap_l1_to_l2(pde, va);
3678                                 if ((pmap_load(l2) & ATTR_DESCR_MASK) ==
3679                                     L2_BLOCK)
3680                                         return (NULL);
3681                         }
3682                         if (lvl == 2 && pmap_load(pde) != 0) {
3683                                 mpte =
3684                                     PHYS_TO_VM_PAGE(pmap_load(pde) & ~ATTR_MASK);
3685                                 mpte->wire_count++;
3686                         } else {
3687                                 /*
3688                                  * Pass NULL instead of the PV list lock
3689                                  * pointer, because we don't intend to sleep.
3690                                  */
3691                                 mpte = _pmap_alloc_l3(pmap, l2pindex, NULL);
3692                                 if (mpte == NULL)
3693                                         return (mpte);
3694                         }
3695                 }
3696                 l3 = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte));
3697                 l3 = &l3[pmap_l3_index(va)];
3698         } else {
3699                 mpte = NULL;
3700                 pde = pmap_pde(kernel_pmap, va, &lvl);
3701                 KASSERT(pde != NULL,
3702                     ("pmap_enter_quick_locked: Invalid page entry, va: 0x%lx",
3703                      va));
3704                 KASSERT(lvl == 2,
3705                     ("pmap_enter_quick_locked: Invalid level %d", lvl));
3706                 l3 = pmap_l2_to_l3(pde, va);
3707         }
3708
3709         /*
3710          * Abort if a mapping already exists.
3711          */
3712         if (pmap_load(l3) != 0) {
3713                 if (mpte != NULL) {
3714                         mpte->wire_count--;
3715                         mpte = NULL;
3716                 }
3717                 return (mpte);
3718         }
3719
3720         /*
3721          * Enter on the PV list if part of our managed memory.
3722          */
3723         if ((m->oflags & VPO_UNMANAGED) == 0 &&
3724             !pmap_try_insert_pv_entry(pmap, va, m, lockp)) {
3725                 if (mpte != NULL) {
3726                         SLIST_INIT(&free);
3727                         if (pmap_unwire_l3(pmap, va, mpte, &free)) {
3728                                 pmap_invalidate_page(pmap, va);
3729                                 vm_page_free_pages_toq(&free, true);
3730                         }
3731                         mpte = NULL;
3732                 }
3733                 return (mpte);
3734         }
3735
3736         /*
3737          * Increment counters
3738          */
3739         pmap_resident_count_inc(pmap, 1);
3740
3741         pa = VM_PAGE_TO_PHYS(m);
3742         l3_val = pa | ATTR_DEFAULT | ATTR_IDX(m->md.pv_memattr) |
3743             ATTR_AP(ATTR_AP_RO) | L3_PAGE;
3744         if ((prot & VM_PROT_EXECUTE) == 0 || m->md.pv_memattr == DEVICE_MEMORY)
3745                 l3_val |= ATTR_XN;
3746         else if (va < VM_MAXUSER_ADDRESS)
3747                 l3_val |= ATTR_PXN;
3748
3749         /*
3750          * Now validate mapping with RO protection
3751          */
3752         if ((m->oflags & VPO_UNMANAGED) == 0)
3753                 l3_val |= ATTR_SW_MANAGED;
3754
3755         /* Sync icache before the mapping is stored to PTE */
3756         if ((prot & VM_PROT_EXECUTE) && pmap != kernel_pmap &&
3757             m->md.pv_memattr == VM_MEMATTR_WRITE_BACK)
3758                 cpu_icache_sync_range(PHYS_TO_DMAP(pa), PAGE_SIZE);
3759
3760         pmap_load_store(l3, l3_val);
3761
3762         /*
3763          * XXX In principle, because this L3 entry was invalid, we should not
3764          * need to perform a TLB invalidation here.  However, in practice,
3765          * when simply performing a "dsb ishst" here, processes are being
3766          * terminated due to bus errors and segmentation violations.
3767          */
3768         pmap_invalidate_page(pmap, va);
3769
3770         return (mpte);
3771 }
3772
3773 /*
3774  * This code maps large physical mmap regions into the
3775  * processor address space.  Note that some shortcuts
3776  * are taken, but the code works.
3777  */
3778 void
3779 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,
3780     vm_pindex_t pindex, vm_size_t size)
3781 {
3782
3783         VM_OBJECT_ASSERT_WLOCKED(object);
3784         KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG,
3785             ("pmap_object_init_pt: non-device object"));
3786 }
3787
3788 /*
3789  *      Clear the wired attribute from the mappings for the specified range of
3790  *      addresses in the given pmap.  Every valid mapping within that range
3791  *      must have the wired attribute set.  In contrast, invalid mappings
3792  *      cannot have the wired attribute set, so they are ignored.
3793  *
3794  *      The wired attribute of the page table entry is not a hardware feature,
3795  *      so there is no need to invalidate any TLB entries.
3796  */
3797 void
3798 pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
3799 {
3800         vm_offset_t va_next;
3801         pd_entry_t *l0, *l1, *l2;
3802         pt_entry_t *l3;
3803
3804         PMAP_LOCK(pmap);
3805         for (; sva < eva; sva = va_next) {
3806                 l0 = pmap_l0(pmap, sva);
3807                 if (pmap_load(l0) == 0) {
3808                         va_next = (sva + L0_SIZE) & ~L0_OFFSET;
3809                         if (va_next < sva)
3810                                 va_next = eva;
3811                         continue;
3812                 }
3813
3814                 l1 = pmap_l0_to_l1(l0, sva);
3815                 if (pmap_load(l1) == 0) {
3816                         va_next = (sva + L1_SIZE) & ~L1_OFFSET;
3817                         if (va_next < sva)
3818                                 va_next = eva;
3819                         continue;
3820                 }
3821
3822                 va_next = (sva + L2_SIZE) & ~L2_OFFSET;
3823                 if (va_next < sva)
3824                         va_next = eva;
3825
3826                 l2 = pmap_l1_to_l2(l1, sva);
3827                 if (pmap_load(l2) == 0)
3828                         continue;
3829
3830                 if ((pmap_load(l2) & ATTR_DESCR_MASK) == L2_BLOCK) {
3831                         if ((pmap_load(l2) & ATTR_SW_WIRED) == 0)
3832                                 panic("pmap_unwire: l2 %#jx is missing "
3833                                     "ATTR_SW_WIRED", (uintmax_t)pmap_load(l2));
3834
3835                         /*
3836                          * Are we unwiring the entire large page?  If not,
3837                          * demote the mapping and fall through.
3838                          */
3839                         if (sva + L2_SIZE == va_next && eva >= va_next) {
3840                                 atomic_clear_64(l2, ATTR_SW_WIRED);
3841                                 pmap->pm_stats.wired_count -= L2_SIZE /
3842                                     PAGE_SIZE;
3843                                 continue;
3844                         } else if (pmap_demote_l2(pmap, l2, sva) == NULL)
3845                                 panic("pmap_unwire: demotion failed");
3846                 }
3847                 KASSERT((pmap_load(l2) & ATTR_DESCR_MASK) == L2_TABLE,
3848                     ("pmap_unwire: Invalid l2 entry after demotion"));
3849
3850                 if (va_next > eva)
3851                         va_next = eva;
3852                 for (l3 = pmap_l2_to_l3(l2, sva); sva != va_next; l3++,
3853                     sva += L3_SIZE) {
3854                         if (pmap_load(l3) == 0)
3855                                 continue;
3856                         if ((pmap_load(l3) & ATTR_SW_WIRED) == 0)
3857                                 panic("pmap_unwire: l3 %#jx is missing "
3858                                     "ATTR_SW_WIRED", (uintmax_t)pmap_load(l3));
3859
3860                         /*
3861                          * ATTR_SW_WIRED must be cleared atomically.  Although
3862                          * the pmap lock synchronizes access to ATTR_SW_WIRED,
3863                          * the System MMU may write to the entry concurrently.
3864                          */
3865                         atomic_clear_64(l3, ATTR_SW_WIRED);
3866                         pmap->pm_stats.wired_count--;
3867                 }
3868         }
3869         PMAP_UNLOCK(pmap);
3870 }
3871
3872 /*
3873  *      Copy the range specified by src_addr/len
3874  *      from the source map to the range dst_addr/len
3875  *      in the destination map.
3876  *
3877  *      This routine is only advisory and need not do anything.
3878  *
3879  *      Because the executable mappings created by this routine are copied,
3880  *      it should not have to flush the instruction cache.
3881  */
3882 void
3883 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
3884     vm_offset_t src_addr)
3885 {
3886         struct rwlock *lock;
3887         struct spglist free;
3888         pd_entry_t *l0, *l1, *l2, srcptepaddr;
3889         pt_entry_t *dst_pte, ptetemp, *src_pte;
3890         vm_offset_t addr, end_addr, va_next;
3891         vm_page_t dst_l2pg, dstmpte, srcmpte;
3892
3893         if (dst_addr != src_addr)
3894                 return;
3895         end_addr = src_addr + len;
3896         lock = NULL;
3897         if (dst_pmap < src_pmap) {
3898                 PMAP_LOCK(dst_pmap);
3899                 PMAP_LOCK(src_pmap);
3900         } else {
3901                 PMAP_LOCK(src_pmap);
3902                 PMAP_LOCK(dst_pmap);
3903         }
3904         for (addr = src_addr; addr < end_addr; addr = va_next) {
3905                 l0 = pmap_l0(src_pmap, addr);
3906                 if (pmap_load(l0) == 0) {
3907                         va_next = (addr + L0_SIZE) & ~L0_OFFSET;
3908                         if (va_next < addr)
3909                                 va_next = end_addr;
3910                         continue;
3911                 }
3912                 l1 = pmap_l0_to_l1(l0, addr);
3913                 if (pmap_load(l1) == 0) {
3914                         va_next = (addr + L1_SIZE) & ~L1_OFFSET;
3915                         if (va_next < addr)
3916                                 va_next = end_addr;
3917                         continue;
3918                 }
3919                 va_next = (addr + L2_SIZE) & ~L2_OFFSET;
3920                 if (va_next < addr)
3921                         va_next = end_addr;
3922                 l2 = pmap_l1_to_l2(l1, addr);
3923                 srcptepaddr = pmap_load(l2);
3924                 if (srcptepaddr == 0)
3925                         continue;
3926                 if ((srcptepaddr & ATTR_DESCR_MASK) == L2_BLOCK) {
3927                         if ((addr & L2_OFFSET) != 0 ||
3928                             addr + L2_SIZE > end_addr)
3929                                 continue;
3930                         dst_l2pg = pmap_alloc_l2(dst_pmap, addr, NULL);
3931                         if (dst_l2pg == NULL)
3932                                 break;
3933                         l2 = (pd_entry_t *)
3934                             PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dst_l2pg));
3935                         l2 = &l2[pmap_l2_index(addr)];
3936                         if (pmap_load(l2) == 0 &&
3937                             ((srcptepaddr & ATTR_SW_MANAGED) == 0 ||
3938                             pmap_pv_insert_l2(dst_pmap, addr, srcptepaddr,
3939                             PMAP_ENTER_NORECLAIM, &lock))) {
3940                                 (void)pmap_load_store(l2, srcptepaddr &
3941                                     ~ATTR_SW_WIRED);
3942                                 pmap_resident_count_inc(dst_pmap, L2_SIZE /
3943                                     PAGE_SIZE);
3944                                 atomic_add_long(&pmap_l2_mappings, 1);
3945                         } else
3946                                 dst_l2pg->wire_count--;
3947                         continue;
3948                 }
3949                 KASSERT((srcptepaddr & ATTR_DESCR_MASK) == L2_TABLE,
3950                     ("pmap_copy: invalid L2 entry"));
3951                 srcptepaddr &= ~ATTR_MASK;
3952                 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr);
3953                 KASSERT(srcmpte->wire_count > 0,
3954                     ("pmap_copy: source page table page is unused"));
3955                 if (va_next > end_addr)
3956                         va_next = end_addr;
3957                 src_pte = (pt_entry_t *)PHYS_TO_DMAP(srcptepaddr);
3958                 src_pte = &src_pte[pmap_l3_index(addr)];
3959                 dstmpte = NULL;
3960                 for (; addr < va_next; addr += PAGE_SIZE, src_pte++) {
3961                         ptetemp = pmap_load(src_pte);
3962
3963                         /*
3964                          * We only virtual copy managed pages.
3965                          */
3966                         if ((ptetemp & ATTR_SW_MANAGED) == 0)
3967                                 continue;
3968
3969                         if (dstmpte != NULL) {
3970                                 KASSERT(dstmpte->pindex == pmap_l2_pindex(addr),
3971                                     ("dstmpte pindex/addr mismatch"));
3972                                 dstmpte->wire_count++;
3973                         } else if ((dstmpte = pmap_alloc_l3(dst_pmap, addr,
3974                             NULL)) == NULL)
3975                                 goto out;
3976                         dst_pte = (pt_entry_t *)
3977                             PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpte));
3978                         dst_pte = &dst_pte[pmap_l3_index(addr)];
3979                         if (pmap_load(dst_pte) == 0 &&
3980                             pmap_try_insert_pv_entry(dst_pmap, addr,
3981                             PHYS_TO_VM_PAGE(ptetemp & ~ATTR_MASK), &lock)) {
3982                                 /*
3983                                  * Clear the wired, modified, and accessed
3984                                  * (referenced) bits during the copy.
3985                                  *
3986                                  * XXX not yet
3987                                  */
3988                                 (void)pmap_load_store(dst_pte, ptetemp &
3989                                     ~ATTR_SW_WIRED);
3990                                 pmap_resident_count_inc(dst_pmap, 1);
3991                         } else {
3992                                 SLIST_INIT(&free);
3993                                 if (pmap_unwire_l3(dst_pmap, addr, dstmpte,
3994                                     &free)) {
3995                                         /*
3996                                          * Although "addr" is not mapped,
3997                                          * paging-structure caches could
3998                                          * nonetheless have entries that refer
3999                                          * to the freed page table pages.
4000                                          * Invalidate those entries.
4001                                          *
4002                                          * XXX redundant invalidation
4003                                          */
4004                                         pmap_invalidate_page(dst_pmap, addr);
4005                                         vm_page_free_pages_toq(&free, true);
4006                                 }
4007                                 goto out;
4008                         }
4009                         /* Have we copied all of the valid mappings? */
4010                         if (dstmpte->wire_count >= srcmpte->wire_count)
4011                                 break;
4012                 }
4013         }
4014 out:
4015         /*
4016          * XXX This barrier may not be needed because the destination pmap is
4017          * not active.
4018          */
4019         dsb(ishst);
4020
4021         if (lock != NULL)
4022                 rw_wunlock(lock);
4023         PMAP_UNLOCK(src_pmap);
4024         PMAP_UNLOCK(dst_pmap);
4025 }
4026
4027 /*
4028  *      pmap_zero_page zeros the specified hardware page by mapping
4029  *      the page into KVM and using bzero to clear its contents.
4030  */
4031 void
4032 pmap_zero_page(vm_page_t m)
4033 {
4034         vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
4035
4036         pagezero((void *)va);
4037 }
4038
4039 /*
4040  *      pmap_zero_page_area zeros the specified hardware page by mapping
4041  *      the page into KVM and using bzero to clear its contents.
4042  *
4043  *      off and size may not cover an area beyond a single hardware page.
4044  */
4045 void
4046 pmap_zero_page_area(vm_page_t m, int off, int size)
4047 {
4048         vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
4049
4050         if (off == 0 && size == PAGE_SIZE)
4051                 pagezero((void *)va);
4052         else
4053                 bzero((char *)va + off, size);
4054 }
4055
4056 /*
4057  *      pmap_copy_page copies the specified (machine independent)
4058  *      page by mapping the page into virtual memory and using
4059  *      bcopy to copy the page, one machine dependent page at a
4060  *      time.
4061  */
4062 void
4063 pmap_copy_page(vm_page_t msrc, vm_page_t mdst)
4064 {
4065         vm_offset_t src = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(msrc));
4066         vm_offset_t dst = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mdst));
4067
4068         pagecopy((void *)src, (void *)dst);
4069 }
4070
4071 int unmapped_buf_allowed = 1;
4072
4073 void
4074 pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[],
4075     vm_offset_t b_offset, int xfersize)
4076 {
4077         void *a_cp, *b_cp;
4078         vm_page_t m_a, m_b;
4079         vm_paddr_t p_a, p_b;
4080         vm_offset_t a_pg_offset, b_pg_offset;
4081         int cnt;
4082
4083         while (xfersize > 0) {
4084                 a_pg_offset = a_offset & PAGE_MASK;
4085                 m_a = ma[a_offset >> PAGE_SHIFT];
4086                 p_a = m_a->phys_addr;
4087                 b_pg_offset = b_offset & PAGE_MASK;
4088                 m_b = mb[b_offset >> PAGE_SHIFT];
4089                 p_b = m_b->phys_addr;
4090                 cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
4091                 cnt = min(cnt, PAGE_SIZE - b_pg_offset);
4092                 if (__predict_false(!PHYS_IN_DMAP(p_a))) {
4093                         panic("!DMAP a %lx", p_a);
4094                 } else {
4095                         a_cp = (char *)PHYS_TO_DMAP(p_a) + a_pg_offset;
4096                 }
4097                 if (__predict_false(!PHYS_IN_DMAP(p_b))) {
4098                         panic("!DMAP b %lx", p_b);
4099                 } else {
4100                         b_cp = (char *)PHYS_TO_DMAP(p_b) + b_pg_offset;
4101                 }
4102                 bcopy(a_cp, b_cp, cnt);
4103                 a_offset += cnt;
4104                 b_offset += cnt;
4105                 xfersize -= cnt;
4106         }
4107 }
4108
4109 vm_offset_t
4110 pmap_quick_enter_page(vm_page_t m)
4111 {
4112
4113         return (PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)));
4114 }
4115
4116 void
4117 pmap_quick_remove_page(vm_offset_t addr)
4118 {
4119 }
4120
4121 /*
4122  * Returns true if the pmap's pv is one of the first
4123  * 16 pvs linked to from this page.  This count may
4124  * be changed upwards or downwards in the future; it
4125  * is only necessary that true be returned for a small
4126  * subset of pmaps for proper page aging.
4127  */
4128 boolean_t
4129 pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
4130 {
4131         struct md_page *pvh;
4132         struct rwlock *lock;
4133         pv_entry_t pv;
4134         int loops = 0;
4135         boolean_t rv;
4136
4137         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
4138             ("pmap_page_exists_quick: page %p is not managed", m));
4139         rv = FALSE;
4140         lock = VM_PAGE_TO_PV_LIST_LOCK(m);
4141         rw_rlock(lock);
4142         TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
4143                 if (PV_PMAP(pv) == pmap) {
4144                         rv = TRUE;
4145                         break;
4146                 }
4147                 loops++;
4148                 if (loops >= 16)
4149                         break;
4150         }
4151         if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) {
4152                 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
4153                 TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
4154                         if (PV_PMAP(pv) == pmap) {
4155                                 rv = TRUE;
4156                                 break;
4157                         }
4158                         loops++;
4159                         if (loops >= 16)
4160                                 break;
4161                 }
4162         }
4163         rw_runlock(lock);
4164         return (rv);
4165 }
4166
4167 /*
4168  *      pmap_page_wired_mappings:
4169  *
4170  *      Return the number of managed mappings to the given physical page
4171  *      that are wired.
4172  */
4173 int
4174 pmap_page_wired_mappings(vm_page_t m)
4175 {
4176         struct rwlock *lock;
4177         struct md_page *pvh;
4178         pmap_t pmap;
4179         pt_entry_t *pte;
4180         pv_entry_t pv;
4181         int count, lvl, md_gen, pvh_gen;
4182
4183         if ((m->oflags & VPO_UNMANAGED) != 0)
4184                 return (0);
4185         lock = VM_PAGE_TO_PV_LIST_LOCK(m);
4186         rw_rlock(lock);
4187 restart:
4188         count = 0;
4189         TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
4190                 pmap = PV_PMAP(pv);
4191                 if (!PMAP_TRYLOCK(pmap)) {
4192                         md_gen = m->md.pv_gen;
4193                         rw_runlock(lock);
4194                         PMAP_LOCK(pmap);
4195                         rw_rlock(lock);
4196                         if (md_gen != m->md.pv_gen) {
4197                                 PMAP_UNLOCK(pmap);
4198                                 goto restart;
4199                         }
4200                 }
4201                 pte = pmap_pte(pmap, pv->pv_va, &lvl);
4202                 if (pte != NULL && (pmap_load(pte) & ATTR_SW_WIRED) != 0)
4203                         count++;
4204                 PMAP_UNLOCK(pmap);
4205         }
4206         if ((m->flags & PG_FICTITIOUS) == 0) {
4207                 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
4208                 TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
4209                         pmap = PV_PMAP(pv);
4210                         if (!PMAP_TRYLOCK(pmap)) {
4211                                 md_gen = m->md.pv_gen;
4212                                 pvh_gen = pvh->pv_gen;
4213                                 rw_runlock(lock);
4214                                 PMAP_LOCK(pmap);
4215                                 rw_rlock(lock);
4216                                 if (md_gen != m->md.pv_gen ||
4217                                     pvh_gen != pvh->pv_gen) {
4218                                         PMAP_UNLOCK(pmap);
4219                                         goto restart;
4220                                 }
4221                         }
4222                         pte = pmap_pte(pmap, pv->pv_va, &lvl);
4223                         if (pte != NULL &&
4224                             (pmap_load(pte) & ATTR_SW_WIRED) != 0)
4225                                 count++;
4226                         PMAP_UNLOCK(pmap);
4227                 }
4228         }
4229         rw_runlock(lock);
4230         return (count);
4231 }
4232
4233 /*
4234  * Destroy all managed, non-wired mappings in the given user-space
4235  * pmap.  This pmap cannot be active on any processor besides the
4236  * caller.
4237  *
4238  * This function cannot be applied to the kernel pmap.  Moreover, it
4239  * is not intended for general use.  It is only to be used during
4240  * process termination.  Consequently, it can be implemented in ways
4241  * that make it faster than pmap_remove().  First, it can more quickly
4242  * destroy mappings by iterating over the pmap's collection of PV
4243  * entries, rather than searching the page table.  Second, it doesn't
4244  * have to test and clear the page table entries atomically, because
4245  * no processor is currently accessing the user address space.  In
4246  * particular, a page table entry's dirty bit won't change state once
4247  * this function starts.
4248  */
4249 void
4250 pmap_remove_pages(pmap_t pmap)
4251 {
4252         pd_entry_t *pde;
4253         pt_entry_t *pte, tpte;
4254         struct spglist free;
4255         vm_page_t m, ml3, mt;
4256         pv_entry_t pv;
4257         struct md_page *pvh;
4258         struct pv_chunk *pc, *npc;
4259         struct rwlock *lock;
4260         int64_t bit;
4261         uint64_t inuse, bitmask;
4262         int allfree, field, freed, idx, lvl;
4263         vm_paddr_t pa;
4264
4265         lock = NULL;
4266
4267         SLIST_INIT(&free);
4268         PMAP_LOCK(pmap);
4269         TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {
4270                 allfree = 1;
4271                 freed = 0;
4272                 for (field = 0; field < _NPCM; field++) {
4273                         inuse = ~pc->pc_map[field] & pc_freemask[field];
4274                         while (inuse != 0) {
4275                                 bit = ffsl(inuse) - 1;
4276                                 bitmask = 1UL << bit;
4277                                 idx = field * 64 + bit;
4278                                 pv = &pc->pc_pventry[idx];
4279                                 inuse &= ~bitmask;
4280
4281                                 pde = pmap_pde(pmap, pv->pv_va, &lvl);
4282                                 KASSERT(pde != NULL,
4283                                     ("Attempting to remove an unmapped page"));
4284
4285                                 switch(lvl) {
4286                                 case 1:
4287                                         pte = pmap_l1_to_l2(pde, pv->pv_va);
4288                                         tpte = pmap_load(pte);
4289                                         KASSERT((tpte & ATTR_DESCR_MASK) ==
4290                                             L2_BLOCK,
4291                                             ("Attempting to remove an invalid "
4292                                             "block: %lx", tpte));
4293                                         tpte = pmap_load(pte);
4294                                         break;
4295                                 case 2:
4296                                         pte = pmap_l2_to_l3(pde, pv->pv_va);
4297                                         tpte = pmap_load(pte);
4298                                         KASSERT((tpte & ATTR_DESCR_MASK) ==
4299                                             L3_PAGE,
4300                                             ("Attempting to remove an invalid "
4301                                              "page: %lx", tpte));
4302                                         break;
4303                                 default:
4304                                         panic(
4305                                             "Invalid page directory level: %d",
4306                                             lvl);
4307                                 }
4308
4309 /*
4310  * We cannot remove wired pages from a process' mapping at this time
4311  */
4312                                 if (tpte & ATTR_SW_WIRED) {
4313                                         allfree = 0;
4314                                         continue;
4315                                 }
4316
4317                                 pa = tpte & ~ATTR_MASK;
4318
4319                                 m = PHYS_TO_VM_PAGE(pa);
4320                                 KASSERT(m->phys_addr == pa,
4321                                     ("vm_page_t %p phys_addr mismatch %016jx %016jx",
4322                                     m, (uintmax_t)m->phys_addr,
4323                                     (uintmax_t)tpte));
4324
4325                                 KASSERT((m->flags & PG_FICTITIOUS) != 0 ||
4326                                     m < &vm_page_array[vm_page_array_size],
4327                                     ("pmap_remove_pages: bad pte %#jx",
4328                                     (uintmax_t)tpte));
4329
4330                                 pmap_load_clear(pte);
4331
4332                                 /*
4333                                  * Update the vm_page_t clean/reference bits.
4334                                  */
4335                                 if ((tpte & ATTR_AP_RW_BIT) ==
4336                                     ATTR_AP(ATTR_AP_RW)) {
4337                                         switch (lvl) {
4338                                         case 1:
4339                                                 for (mt = m; mt < &m[L2_SIZE / PAGE_SIZE]; mt++)
4340                                                         vm_page_dirty(mt);
4341                                                 break;
4342                                         case 2:
4343                                                 vm_page_dirty(m);
4344                                                 break;
4345                                         }
4346                                 }
4347
4348                                 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m);
4349
4350                                 /* Mark free */
4351                                 pc->pc_map[field] |= bitmask;
4352                                 switch (lvl) {
4353                                 case 1:
4354                                         pmap_resident_count_dec(pmap,
4355                                             L2_SIZE / PAGE_SIZE);
4356                                         pvh = pa_to_pvh(tpte & ~ATTR_MASK);
4357                                         TAILQ_REMOVE(&pvh->pv_list, pv,pv_next);
4358                                         pvh->pv_gen++;
4359                                         if (TAILQ_EMPTY(&pvh->pv_list)) {
4360                                                 for (mt = m; mt < &m[L2_SIZE / PAGE_SIZE]; mt++)
4361                                                         if ((mt->aflags & PGA_WRITEABLE) != 0 &&
4362                                                             TAILQ_EMPTY(&mt->md.pv_list))
4363                                                                 vm_page_aflag_clear(mt, PGA_WRITEABLE);
4364                                         }
4365                                         ml3 = pmap_remove_pt_page(pmap,
4366                                             pv->pv_va);
4367                                         if (ml3 != NULL) {
4368                                                 KASSERT(ml3->valid == VM_PAGE_BITS_ALL,
4369                                                     ("pmap_remove_pages: l3 page not promoted"));
4370                                                 pmap_resident_count_dec(pmap,1);
4371                                                 KASSERT(ml3->wire_count == NL3PG,
4372                                                     ("pmap_remove_pages: l3 page wire count error"));
4373                                                 ml3->wire_count = 0;
4374                                                 pmap_add_delayed_free_list(ml3,
4375                                                     &free, FALSE);
4376                                         }
4377                                         break;
4378                                 case 2:
4379                                         pmap_resident_count_dec(pmap, 1);
4380                                         TAILQ_REMOVE(&m->md.pv_list, pv,
4381                                             pv_next);
4382                                         m->md.pv_gen++;
4383                                         if ((m->aflags & PGA_WRITEABLE) != 0 &&
4384                                             TAILQ_EMPTY(&m->md.pv_list) &&
4385                                             (m->flags & PG_FICTITIOUS) == 0) {
4386                                                 pvh = pa_to_pvh(
4387                                                     VM_PAGE_TO_PHYS(m));
4388                                                 if (TAILQ_EMPTY(&pvh->pv_list))
4389                                                         vm_page_aflag_clear(m,
4390                                                             PGA_WRITEABLE);
4391                                         }
4392                                         break;
4393                                 }
4394                                 pmap_unuse_pt(pmap, pv->pv_va, pmap_load(pde),
4395                                     &free);
4396                                 freed++;
4397                         }
4398                 }
4399                 PV_STAT(atomic_add_long(&pv_entry_frees, freed));
4400                 PV_STAT(atomic_add_int(&pv_entry_spare, freed));
4401                 PV_STAT(atomic_subtract_long(&pv_entry_count, freed));
4402                 if (allfree) {
4403                         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
4404                         free_pv_chunk(pc);
4405                 }
4406         }
4407         pmap_invalidate_all(pmap);
4408         if (lock != NULL)
4409                 rw_wunlock(lock);
4410         PMAP_UNLOCK(pmap);
4411         vm_page_free_pages_toq(&free, true);
4412 }
4413
4414 /*
4415  * This is used to check if a page has been accessed or modified. As we
4416  * don't have a bit to see if it has been modified we have to assume it
4417  * has been if the page is read/write.
4418  */
4419 static boolean_t
4420 pmap_page_test_mappings(vm_page_t m, boolean_t accessed, boolean_t modified)
4421 {
4422         struct rwlock *lock;
4423         pv_entry_t pv;
4424         struct md_page *pvh;
4425         pt_entry_t *pte, mask, value;
4426         pmap_t pmap;
4427         int lvl, md_gen, pvh_gen;
4428         boolean_t rv;
4429
4430         rv = FALSE;
4431         lock = VM_PAGE_TO_PV_LIST_LOCK(m);
4432         rw_rlock(lock);
4433 restart:
4434         TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
4435                 pmap = PV_PMAP(pv);
4436                 if (!PMAP_TRYLOCK(pmap)) {
4437                         md_gen = m->md.pv_gen;
4438                         rw_runlock(lock);
4439                         PMAP_LOCK(pmap);
4440                         rw_rlock(lock);
4441                         if (md_gen != m->md.pv_gen) {
4442                                 PMAP_UNLOCK(pmap);
4443                                 goto restart;
4444                         }
4445                 }
4446                 pte = pmap_pte(pmap, pv->pv_va, &lvl);
4447                 KASSERT(lvl == 3,
4448                     ("pmap_page_test_mappings: Invalid level %d", lvl));
4449                 mask = 0;
4450                 value = 0;
4451                 if (modified) {
4452                         mask |= ATTR_AP_RW_BIT;
4453                         value |= ATTR_AP(ATTR_AP_RW);
4454                 }
4455                 if (accessed) {
4456                         mask |= ATTR_AF | ATTR_DESCR_MASK;
4457                         value |= ATTR_AF | L3_PAGE;
4458                 }
4459                 rv = (pmap_load(pte) & mask) == value;
4460                 PMAP_UNLOCK(pmap);
4461                 if (rv)
4462                         goto out;
4463         }
4464         if ((m->flags & PG_FICTITIOUS) == 0) {
4465                 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
4466                 TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
4467                         pmap = PV_PMAP(pv);
4468                         if (!PMAP_TRYLOCK(pmap)) {
4469                                 md_gen = m->md.pv_gen;
4470                                 pvh_gen = pvh->pv_gen;
4471                                 rw_runlock(lock);
4472                                 PMAP_LOCK(pmap);
4473                                 rw_rlock(lock);
4474                                 if (md_gen != m->md.pv_gen ||
4475                                     pvh_gen != pvh->pv_gen) {
4476                                         PMAP_UNLOCK(pmap);
4477                                         goto restart;
4478                                 }
4479                         }
4480                         pte = pmap_pte(pmap, pv->pv_va, &lvl);
4481                         KASSERT(lvl == 2,
4482                             ("pmap_page_test_mappings: Invalid level %d", lvl));
4483                         mask = 0;
4484                         value = 0;
4485                         if (modified) {
4486                                 mask |= ATTR_AP_RW_BIT;
4487                                 value |= ATTR_AP(ATTR_AP_RW);
4488                         }
4489                         if (accessed) {
4490                                 mask |= ATTR_AF | ATTR_DESCR_MASK;
4491                                 value |= ATTR_AF | L2_BLOCK;
4492                         }
4493                         rv = (pmap_load(pte) & mask) == value;
4494                         PMAP_UNLOCK(pmap);
4495                         if (rv)
4496                                 goto out;
4497                 }
4498         }
4499 out:
4500         rw_runlock(lock);
4501         return (rv);
4502 }
4503
4504 /*
4505  *      pmap_is_modified:
4506  *
4507  *      Return whether or not the specified physical page was modified
4508  *      in any physical maps.
4509  */
4510 boolean_t
4511 pmap_is_modified(vm_page_t m)
4512 {
4513
4514         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
4515             ("pmap_is_modified: page %p is not managed", m));
4516
4517         /*
4518          * If the page is not exclusive busied, then PGA_WRITEABLE cannot be
4519          * concurrently set while the object is locked.  Thus, if PGA_WRITEABLE
4520          * is clear, no PTEs can have PG_M set.
4521          */
4522         VM_OBJECT_ASSERT_WLOCKED(m->object);
4523         if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0)
4524                 return (FALSE);
4525         return (pmap_page_test_mappings(m, FALSE, TRUE));
4526 }
4527
4528 /*
4529  *      pmap_is_prefaultable:
4530  *
4531  *      Return whether or not the specified virtual address is eligible
4532  *      for prefault.
4533  */
4534 boolean_t
4535 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)
4536 {
4537         pt_entry_t *pte;
4538         boolean_t rv;
4539         int lvl;
4540
4541         rv = FALSE;
4542         PMAP_LOCK(pmap);
4543         pte = pmap_pte(pmap, addr, &lvl);
4544         if (pte != NULL && pmap_load(pte) != 0) {
4545                 rv = TRUE;
4546         }
4547         PMAP_UNLOCK(pmap);
4548         return (rv);
4549 }
4550
4551 /*
4552  *      pmap_is_referenced:
4553  *
4554  *      Return whether or not the specified physical page was referenced
4555  *      in any physical maps.
4556  */
4557 boolean_t
4558 pmap_is_referenced(vm_page_t m)
4559 {
4560
4561         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
4562             ("pmap_is_referenced: page %p is not managed", m));
4563         return (pmap_page_test_mappings(m, TRUE, FALSE));
4564 }
4565
4566 /*
4567  * Clear the write and modified bits in each of the given page's mappings.
4568  */
4569 void
4570 pmap_remove_write(vm_page_t m)
4571 {
4572         struct md_page *pvh;
4573         pmap_t pmap;
4574         struct rwlock *lock;
4575         pv_entry_t next_pv, pv;
4576         pt_entry_t oldpte, *pte;
4577         vm_offset_t va;
4578         int lvl, md_gen, pvh_gen;
4579
4580         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
4581             ("pmap_remove_write: page %p is not managed", m));
4582
4583         /*
4584          * If the page is not exclusive busied, then PGA_WRITEABLE cannot be
4585          * set by another thread while the object is locked.  Thus,
4586          * if PGA_WRITEABLE is clear, no page table entries need updating.
4587          */
4588         VM_OBJECT_ASSERT_WLOCKED(m->object);
4589         if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0)
4590                 return;
4591         lock = VM_PAGE_TO_PV_LIST_LOCK(m);
4592         pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
4593             pa_to_pvh(VM_PAGE_TO_PHYS(m));
4594 retry_pv_loop:
4595         rw_wlock(lock);
4596         TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) {
4597                 pmap = PV_PMAP(pv);
4598                 if (!PMAP_TRYLOCK(pmap)) {
4599                         pvh_gen = pvh->pv_gen;
4600                         rw_wunlock(lock);
4601                         PMAP_LOCK(pmap);
4602                         rw_wlock(lock);
4603                         if (pvh_gen != pvh->pv_gen) {
4604                                 PMAP_UNLOCK(pmap);
4605                                 rw_wunlock(lock);
4606                                 goto retry_pv_loop;
4607                         }
4608                 }
4609                 va = pv->pv_va;
4610                 pte = pmap_pte(pmap, pv->pv_va, &lvl);
4611                 if ((pmap_load(pte) & ATTR_AP_RW_BIT) == ATTR_AP(ATTR_AP_RW))
4612                         (void)pmap_demote_l2_locked(pmap, pte, va, &lock);
4613                 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m),
4614                     ("inconsistent pv lock %p %p for page %p",
4615                     lock, VM_PAGE_TO_PV_LIST_LOCK(m), m));
4616                 PMAP_UNLOCK(pmap);
4617         }
4618         TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
4619                 pmap = PV_PMAP(pv);
4620                 if (!PMAP_TRYLOCK(pmap)) {
4621                         pvh_gen = pvh->pv_gen;
4622                         md_gen = m->md.pv_gen;
4623                         rw_wunlock(lock);
4624                         PMAP_LOCK(pmap);
4625                         rw_wlock(lock);
4626                         if (pvh_gen != pvh->pv_gen ||
4627                             md_gen != m->md.pv_gen) {
4628                                 PMAP_UNLOCK(pmap);
4629                                 rw_wunlock(lock);
4630                                 goto retry_pv_loop;
4631                         }
4632                 }
4633                 pte = pmap_pte(pmap, pv->pv_va, &lvl);
4634 retry:
4635                 oldpte = pmap_load(pte);
4636                 if ((oldpte & ATTR_AP_RW_BIT) == ATTR_AP(ATTR_AP_RW)) {
4637                         if (!atomic_cmpset_long(pte, oldpte,
4638                             oldpte | ATTR_AP(ATTR_AP_RO)))
4639                                 goto retry;
4640                         if ((oldpte & ATTR_AF) != 0)
4641                                 vm_page_dirty(m);
4642                         pmap_invalidate_page(pmap, pv->pv_va);
4643                 }
4644                 PMAP_UNLOCK(pmap);
4645         }
4646         rw_wunlock(lock);
4647         vm_page_aflag_clear(m, PGA_WRITEABLE);
4648 }
4649
4650 static __inline boolean_t
4651 safe_to_clear_referenced(pmap_t pmap, pt_entry_t pte)
4652 {
4653
4654         return (FALSE);
4655 }
4656
4657 /*
4658  *      pmap_ts_referenced:
4659  *
4660  *      Return a count of reference bits for a page, clearing those bits.
4661  *      It is not necessary for every reference bit to be cleared, but it
4662  *      is necessary that 0 only be returned when there are truly no
4663  *      reference bits set.
4664  *
4665  *      As an optimization, update the page's dirty field if a modified bit is
4666  *      found while counting reference bits.  This opportunistic update can be
4667  *      performed at low cost and can eliminate the need for some future calls
4668  *      to pmap_is_modified().  However, since this function stops after
4669  *      finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some
4670  *      dirty pages.  Those dirty pages will only be detected by a future call
4671  *      to pmap_is_modified().
4672  */
4673 int
4674 pmap_ts_referenced(vm_page_t m)
4675 {
4676         struct md_page *pvh;
4677         pv_entry_t pv, pvf;
4678         pmap_t pmap;
4679         struct rwlock *lock;
4680         pd_entry_t *pde, tpde;
4681         pt_entry_t *pte, tpte;
4682         pt_entry_t *l3;
4683         vm_offset_t va;
4684         vm_paddr_t pa;
4685         int cleared, md_gen, not_cleared, lvl, pvh_gen;
4686         struct spglist free;
4687         bool demoted;
4688
4689         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
4690             ("pmap_ts_referenced: page %p is not managed", m));
4691         SLIST_INIT(&free);
4692         cleared = 0;
4693         pa = VM_PAGE_TO_PHYS(m);
4694         lock = PHYS_TO_PV_LIST_LOCK(pa);
4695         pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : pa_to_pvh(pa);
4696         rw_wlock(lock);
4697 retry:
4698         not_cleared = 0;
4699         if ((pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL)
4700                 goto small_mappings;
4701         pv = pvf;
4702         do {
4703                 if (pvf == NULL)
4704                         pvf = pv;
4705                 pmap = PV_PMAP(pv);
4706                 if (!PMAP_TRYLOCK(pmap)) {
4707                         pvh_gen = pvh->pv_gen;
4708                         rw_wunlock(lock);
4709                         PMAP_LOCK(pmap);
4710                         rw_wlock(lock);
4711                         if (pvh_gen != pvh->pv_gen) {
4712                                 PMAP_UNLOCK(pmap);
4713                                 goto retry;
4714                         }
4715                 }
4716                 va = pv->pv_va;
4717                 pde = pmap_pde(pmap, pv->pv_va, &lvl);
4718                 KASSERT(pde != NULL, ("pmap_ts_referenced: no l1 table found"));
4719                 KASSERT(lvl == 1,
4720                     ("pmap_ts_referenced: invalid pde level %d", lvl));
4721                 tpde = pmap_load(pde);
4722                 KASSERT((tpde & ATTR_DESCR_MASK) == L1_TABLE,
4723                     ("pmap_ts_referenced: found an invalid l1 table"));
4724                 pte = pmap_l1_to_l2(pde, pv->pv_va);
4725                 tpte = pmap_load(pte);
4726                 if (pmap_page_dirty(tpte)) {
4727                         /*
4728                          * Although "tpte" is mapping a 2MB page, because
4729                          * this function is called at a 4KB page granularity,
4730                          * we only update the 4KB page under test.
4731                          */
4732                         vm_page_dirty(m);
4733                 }
4734                 if ((tpte & ATTR_AF) != 0) {
4735                         /*
4736                          * Since this reference bit is shared by 512 4KB
4737                          * pages, it should not be cleared every time it is
4738                          * tested.  Apply a simple "hash" function on the
4739                          * physical page number, the virtual superpage number,
4740                          * and the pmap address to select one 4KB page out of
4741                          * the 512 on which testing the reference bit will
4742                          * result in clearing that reference bit.  This
4743                          * function is designed to avoid the selection of the
4744                          * same 4KB page for every 2MB page mapping.
4745                          *
4746                          * On demotion, a mapping that hasn't been referenced
4747                          * is simply destroyed.  To avoid the possibility of a
4748                          * subsequent page fault on a demoted wired mapping,
4749                          * always leave its reference bit set.  Moreover,
4750                          * since the superpage is wired, the current state of
4751                          * its reference bit won't affect page replacement.
4752                          */
4753                         if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> L2_SHIFT) ^
4754                             (uintptr_t)pmap) & (Ln_ENTRIES - 1)) == 0 &&
4755                             (tpte & ATTR_SW_WIRED) == 0) {
4756                                 if (safe_to_clear_referenced(pmap, tpte)) {
4757                                         /*
4758                                          * TODO: We don't handle the access
4759                                          * flag at all. We need to be able
4760                                          * to set it in  the exception handler.
4761                                          */
4762                                         panic("ARM64TODO: "
4763                                             "safe_to_clear_referenced\n");
4764                                 } else if (pmap_demote_l2_locked(pmap, pte,
4765                                     pv->pv_va, &lock) != NULL) {
4766                                         demoted = true;
4767                                         va += VM_PAGE_TO_PHYS(m) -
4768                                             (tpte & ~ATTR_MASK);
4769                                         l3 = pmap_l2_to_l3(pte, va);
4770                                         pmap_remove_l3(pmap, l3, va,
4771                                             pmap_load(pte), NULL, &lock);
4772                                 } else
4773                                         demoted = true;
4774
4775                                 if (demoted) {
4776                                         /*
4777                                          * The superpage mapping was removed
4778                                          * entirely and therefore 'pv' is no
4779                                          * longer valid.
4780                                          */
4781                                         if (pvf == pv)
4782                                                 pvf = NULL;
4783                                         pv = NULL;
4784                                 }
4785                                 cleared++;
4786                                 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m),
4787                                     ("inconsistent pv lock %p %p for page %p",
4788                                     lock, VM_PAGE_TO_PV_LIST_LOCK(m), m));
4789                         } else
4790                                 not_cleared++;
4791                 }
4792                 PMAP_UNLOCK(pmap);
4793                 /* Rotate the PV list if it has more than one entry. */
4794                 if (pv != NULL && TAILQ_NEXT(pv, pv_next) != NULL) {
4795                         TAILQ_REMOVE(&pvh->pv_list, pv, pv_next);
4796                         TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next);
4797                         pvh->pv_gen++;
4798                 }
4799                 if (cleared + not_cleared >= PMAP_TS_REFERENCED_MAX)
4800                         goto out;
4801         } while ((pv = TAILQ_FIRST(&pvh->pv_list)) != pvf);
4802 small_mappings:
4803         if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL)
4804                 goto out;
4805         pv = pvf;
4806         do {
4807                 if (pvf == NULL)
4808                         pvf = pv;
4809                 pmap = PV_PMAP(pv);
4810                 if (!PMAP_TRYLOCK(pmap)) {
4811                         pvh_gen = pvh->pv_gen;
4812                         md_gen = m->md.pv_gen;
4813                         rw_wunlock(lock);
4814                         PMAP_LOCK(pmap);
4815                         rw_wlock(lock);
4816                         if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
4817                                 PMAP_UNLOCK(pmap);
4818                                 goto retry;
4819                         }
4820                 }
4821                 pde = pmap_pde(pmap, pv->pv_va, &lvl);
4822                 KASSERT(pde != NULL, ("pmap_ts_referenced: no l2 table found"));
4823                 KASSERT(lvl == 2,
4824                     ("pmap_ts_referenced: invalid pde level %d", lvl));
4825                 tpde = pmap_load(pde);
4826                 KASSERT((tpde & ATTR_DESCR_MASK) == L2_TABLE,
4827                     ("pmap_ts_referenced: found an invalid l2 table"));
4828                 pte = pmap_l2_to_l3(pde, pv->pv_va);
4829                 tpte = pmap_load(pte);
4830                 if (pmap_page_dirty(tpte))
4831                         vm_page_dirty(m);
4832                 if ((tpte & ATTR_AF) != 0) {
4833                         if (safe_to_clear_referenced(pmap, tpte)) {
4834                                 /*
4835                                  * TODO: We don't handle the access flag
4836                                  * at all. We need to be able to set it in
4837                                  * the exception handler.
4838                                  */
4839                                 panic("ARM64TODO: safe_to_clear_referenced\n");
4840                         } else if ((tpte & ATTR_SW_WIRED) == 0) {
4841                                 /*
4842                                  * Wired pages cannot be paged out so
4843                                  * doing accessed bit emulation for
4844                                  * them is wasted effort. We do the
4845                                  * hard work for unwired pages only.
4846                                  */
4847                                 pmap_remove_l3(pmap, pte, pv->pv_va, tpde,
4848                                     &free, &lock);
4849                                 cleared++;
4850                                 if (pvf == pv)
4851                                         pvf = NULL;
4852                                 pv = NULL;
4853                                 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m),
4854                                     ("inconsistent pv lock %p %p for page %p",
4855                                     lock, VM_PAGE_TO_PV_LIST_LOCK(m), m));
4856                         } else
4857                                 not_cleared++;
4858                 }
4859                 PMAP_UNLOCK(pmap);
4860                 /* Rotate the PV list if it has more than one entry. */
4861                 if (pv != NULL && TAILQ_NEXT(pv, pv_next) != NULL) {
4862                         TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
4863                         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
4864                         m->md.pv_gen++;
4865                 }
4866         } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared +
4867             not_cleared < PMAP_TS_REFERENCED_MAX);
4868 out:
4869         rw_wunlock(lock);
4870         vm_page_free_pages_toq(&free, true);
4871         return (cleared + not_cleared);
4872 }
4873
4874 /*
4875  *      Apply the given advice to the specified range of addresses within the
4876  *      given pmap.  Depending on the advice, clear the referenced and/or
4877  *      modified flags in each mapping and set the mapped page's dirty field.
4878  */
4879 void
4880 pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
4881 {
4882 }
4883
4884 /*
4885  *      Clear the modify bits on the specified physical page.
4886  */
4887 void
4888 pmap_clear_modify(vm_page_t m)
4889 {
4890
4891         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
4892             ("pmap_clear_modify: page %p is not managed", m));
4893         VM_OBJECT_ASSERT_WLOCKED(m->object);
4894         KASSERT(!vm_page_xbusied(m),
4895             ("pmap_clear_modify: page %p is exclusive busied", m));
4896
4897         /*
4898          * If the page is not PGA_WRITEABLE, then no PTEs can have PG_M set.
4899          * If the object containing the page is locked and the page is not
4900          * exclusive busied, then PGA_WRITEABLE cannot be concurrently set.
4901          */
4902         if ((m->aflags & PGA_WRITEABLE) == 0)
4903                 return;
4904
4905         /* ARM64TODO: We lack support for tracking if a page is modified */
4906 }
4907
4908 void *
4909 pmap_mapbios(vm_paddr_t pa, vm_size_t size)
4910 {
4911         struct pmap_preinit_mapping *ppim;
4912         vm_offset_t va, offset;
4913         pd_entry_t *pde;
4914         pt_entry_t *l2;
4915         int i, lvl, l2_blocks, free_l2_count, start_idx;
4916
4917         if (!vm_initialized) {
4918                 /*
4919                  * No L3 ptables so map entire L2 blocks where start VA is:
4920                  *      preinit_map_va + start_idx * L2_SIZE
4921                  * There may be duplicate mappings (multiple VA -> same PA) but
4922                  * ARM64 dcache is always PIPT so that's acceptable.
4923                  */
4924                  if (size == 0)
4925                          return (NULL);
4926
4927                  /* Calculate how many L2 blocks are needed for the mapping */
4928                 l2_blocks = (roundup2(pa + size, L2_SIZE) -
4929                     rounddown2(pa, L2_SIZE)) >> L2_SHIFT;
4930
4931                 offset = pa & L2_OFFSET;
4932
4933                 if (preinit_map_va == 0)
4934                         return (NULL);
4935
4936                 /* Map 2MiB L2 blocks from reserved VA space */
4937
4938                 free_l2_count = 0;
4939                 start_idx = -1;
4940                 /* Find enough free contiguous VA space */
4941                 for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) {
4942                         ppim = pmap_preinit_mapping + i;
4943                         if (free_l2_count > 0 && ppim->pa != 0) {
4944                                 /* Not enough space here */
4945                                 free_l2_count = 0;
4946                                 start_idx = -1;
4947                                 continue;
4948                         }
4949
4950                         if (ppim->pa == 0) {
4951                                 /* Free L2 block */
4952                                 if (start_idx == -1)
4953                                         start_idx = i;
4954                                 free_l2_count++;
4955                                 if (free_l2_count == l2_blocks)
4956                                         break;
4957                         }
4958                 }
4959                 if (free_l2_count != l2_blocks)
4960                         panic("%s: too many preinit mappings", __func__);
4961
4962                 va = preinit_map_va + (start_idx * L2_SIZE);
4963                 for (i = start_idx; i < start_idx + l2_blocks; i++) {
4964                         /* Mark entries as allocated */
4965                         ppim = pmap_preinit_mapping + i;
4966                         ppim->pa = pa;
4967                         ppim->va = va + offset;
4968                         ppim->size = size;
4969                 }
4970
4971                 /* Map L2 blocks */
4972                 pa = rounddown2(pa, L2_SIZE);
4973                 for (i = 0; i < l2_blocks; i++) {
4974                         pde = pmap_pde(kernel_pmap, va, &lvl);
4975                         KASSERT(pde != NULL,
4976                             ("pmap_mapbios: Invalid page entry, va: 0x%lx",
4977                             va));
4978                         KASSERT(lvl == 1,
4979                             ("pmap_mapbios: Invalid level %d", lvl));
4980
4981                         /* Insert L2_BLOCK */
4982                         l2 = pmap_l1_to_l2(pde, va);
4983                         pmap_load_store(l2,
4984                             pa | ATTR_DEFAULT | ATTR_XN |
4985                             ATTR_IDX(CACHED_MEMORY) | L2_BLOCK);
4986
4987                         va += L2_SIZE;
4988                         pa += L2_SIZE;
4989                 }
4990                 pmap_invalidate_all(kernel_pmap);
4991
4992                 va = preinit_map_va + (start_idx * L2_SIZE);
4993
4994         } else {
4995                 /* kva_alloc may be used to map the pages */
4996                 offset = pa & PAGE_MASK;
4997                 size = round_page(offset + size);
4998
4999                 va = kva_alloc(size);
5000                 if (va == 0)
5001                         panic("%s: Couldn't allocate KVA", __func__);
5002
5003                 pde = pmap_pde(kernel_pmap, va, &lvl);
5004                 KASSERT(lvl == 2, ("pmap_mapbios: Invalid level %d", lvl));
5005
5006                 /* L3 table is linked */
5007                 va = trunc_page(va);
5008                 pa = trunc_page(pa);
5009                 pmap_kenter(va, size, pa, CACHED_MEMORY);
5010         }
5011
5012         return ((void *)(va + offset));
5013 }
5014
5015 void
5016 pmap_unmapbios(vm_offset_t va, vm_size_t size)
5017 {
5018         struct pmap_preinit_mapping *ppim;
5019         vm_offset_t offset, tmpsize, va_trunc;
5020         pd_entry_t *pde;
5021         pt_entry_t *l2;
5022         int i, lvl, l2_blocks, block;
5023         bool preinit_map;
5024
5025         l2_blocks =
5026            (roundup2(va + size, L2_SIZE) - rounddown2(va, L2_SIZE)) >> L2_SHIFT;
5027         KASSERT(l2_blocks > 0, ("pmap_unmapbios: invalid size %lx", size));
5028
5029         /* Remove preinit mapping */
5030         preinit_map = false;
5031         block = 0;
5032         for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) {
5033                 ppim = pmap_preinit_mapping + i;
5034                 if (ppim->va == va) {
5035                         KASSERT(ppim->size == size,
5036                             ("pmap_unmapbios: size mismatch"));
5037                         ppim->va = 0;
5038                         ppim->pa = 0;
5039                         ppim->size = 0;
5040                         preinit_map = true;
5041                         offset = block * L2_SIZE;
5042                         va_trunc = rounddown2(va, L2_SIZE) + offset;
5043
5044                         /* Remove L2_BLOCK */
5045                         pde = pmap_pde(kernel_pmap, va_trunc, &lvl);
5046                         KASSERT(pde != NULL,
5047                             ("pmap_unmapbios: Invalid page entry, va: 0x%lx",
5048                             va_trunc));
5049                         l2 = pmap_l1_to_l2(pde, va_trunc);
5050                         pmap_load_clear(l2);
5051
5052                         if (block == (l2_blocks - 1))
5053                                 break;
5054                         block++;
5055                 }
5056         }
5057         if (preinit_map) {
5058                 pmap_invalidate_all(kernel_pmap);
5059                 return;
5060         }
5061
5062         /* Unmap the pages reserved with kva_alloc. */
5063         if (vm_initialized) {
5064                 offset = va & PAGE_MASK;
5065                 size = round_page(offset + size);
5066                 va = trunc_page(va);
5067
5068                 pde = pmap_pde(kernel_pmap, va, &lvl);
5069                 KASSERT(pde != NULL,
5070                     ("pmap_unmapbios: Invalid page entry, va: 0x%lx", va));
5071                 KASSERT(lvl == 2, ("pmap_unmapbios: Invalid level %d", lvl));
5072
5073                 /* Unmap and invalidate the pages */
5074                 for (tmpsize = 0; tmpsize < size; tmpsize += PAGE_SIZE)
5075                         pmap_kremove(va + tmpsize);
5076
5077                 kva_free(va, size);
5078         }
5079 }
5080
5081 /*
5082  * Sets the memory attribute for the specified page.
5083  */
5084 void
5085 pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma)
5086 {
5087
5088         m->md.pv_memattr = ma;
5089
5090         /*
5091          * If "m" is a normal page, update its direct mapping.  This update
5092          * can be relied upon to perform any cache operations that are
5093          * required for data coherence.
5094          */
5095         if ((m->flags & PG_FICTITIOUS) == 0 &&
5096             pmap_change_attr(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)), PAGE_SIZE,
5097             m->md.pv_memattr) != 0)
5098                 panic("memory attribute change on the direct map failed");
5099 }
5100
5101 /*
5102  * Changes the specified virtual address range's memory type to that given by
5103  * the parameter "mode".  The specified virtual address range must be
5104  * completely contained within either the direct map or the kernel map.  If
5105  * the virtual address range is contained within the kernel map, then the
5106  * memory type for each of the corresponding ranges of the direct map is also
5107  * changed.  (The corresponding ranges of the direct map are those ranges that
5108  * map the same physical pages as the specified virtual address range.)  These
5109  * changes to the direct map are necessary because Intel describes the
5110  * behavior of their processors as "undefined" if two or more mappings to the
5111  * same physical page have different memory types.
5112  *
5113  * Returns zero if the change completed successfully, and either EINVAL or
5114  * ENOMEM if the change failed.  Specifically, EINVAL is returned if some part
5115  * of the virtual address range was not mapped, and ENOMEM is returned if
5116  * there was insufficient memory available to complete the change.  In the
5117  * latter case, the memory type may have been changed on some part of the
5118  * virtual address range or the direct map.
5119  */
5120 static int
5121 pmap_change_attr(vm_offset_t va, vm_size_t size, int mode)
5122 {
5123         int error;
5124
5125         PMAP_LOCK(kernel_pmap);
5126         error = pmap_change_attr_locked(va, size, mode);
5127         PMAP_UNLOCK(kernel_pmap);
5128         return (error);
5129 }
5130
5131 static int
5132 pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode)
5133 {
5134         vm_offset_t base, offset, tmpva;
5135         pt_entry_t l3, *pte, *newpte;
5136         int lvl;
5137
5138         PMAP_LOCK_ASSERT(kernel_pmap, MA_OWNED);
5139         base = trunc_page(va);
5140         offset = va & PAGE_MASK;
5141         size = round_page(offset + size);
5142
5143         if (!VIRT_IN_DMAP(base))
5144                 return (EINVAL);
5145
5146         for (tmpva = base; tmpva < base + size; ) {
5147                 pte = pmap_pte(kernel_pmap, tmpva, &lvl);
5148                 if (pte == NULL)
5149                         return (EINVAL);
5150
5151                 if ((pmap_load(pte) & ATTR_IDX_MASK) == ATTR_IDX(mode)) {
5152                         /*
5153                          * We already have the correct attribute,
5154                          * ignore this entry.
5155                          */
5156                         switch (lvl) {
5157                         default:
5158                                 panic("Invalid DMAP table level: %d\n", lvl);
5159                         case 1:
5160                                 tmpva = (tmpva & ~L1_OFFSET) + L1_SIZE;
5161                                 break;
5162                         case 2:
5163                                 tmpva = (tmpva & ~L2_OFFSET) + L2_SIZE;
5164                                 break;
5165                         case 3:
5166                                 tmpva += PAGE_SIZE;
5167                                 break;
5168                         }
5169                 } else {
5170                         /*
5171                          * Split the entry to an level 3 table, then
5172                          * set the new attribute.
5173                          */
5174                         switch (lvl) {
5175                         default:
5176                                 panic("Invalid DMAP table level: %d\n", lvl);
5177                         case 1:
5178                                 newpte = pmap_demote_l1(kernel_pmap, pte,
5179                                     tmpva & ~L1_OFFSET);
5180                                 if (newpte == NULL)
5181                                         return (EINVAL);
5182                                 pte = pmap_l1_to_l2(pte, tmpva);
5183                         case 2:
5184                                 newpte = pmap_demote_l2(kernel_pmap, pte,
5185                                     tmpva);
5186                                 if (newpte == NULL)
5187                                         return (EINVAL);
5188                                 pte = pmap_l2_to_l3(pte, tmpva);
5189                         case 3:
5190                                 /* Update the entry */
5191                                 l3 = pmap_load(pte);
5192                                 l3 &= ~ATTR_IDX_MASK;
5193                                 l3 |= ATTR_IDX(mode);
5194                                 if (mode == DEVICE_MEMORY)
5195                                         l3 |= ATTR_XN;
5196
5197                                 pmap_update_entry(kernel_pmap, pte, l3, tmpva,
5198                                     PAGE_SIZE);
5199
5200                                 /*
5201                                  * If moving to a non-cacheable entry flush
5202                                  * the cache.
5203                                  */
5204                                 if (mode == VM_MEMATTR_UNCACHEABLE)
5205                                         cpu_dcache_wbinv_range(tmpva, L3_SIZE);
5206
5207                                 break;
5208                         }
5209                         tmpva += PAGE_SIZE;
5210                 }
5211         }
5212
5213         return (0);
5214 }
5215
5216 /*
5217  * Create an L2 table to map all addresses within an L1 mapping.
5218  */
5219 static pt_entry_t *
5220 pmap_demote_l1(pmap_t pmap, pt_entry_t *l1, vm_offset_t va)
5221 {
5222         pt_entry_t *l2, newl2, oldl1;
5223         vm_offset_t tmpl1;
5224         vm_paddr_t l2phys, phys;
5225         vm_page_t ml2;
5226         int i;
5227
5228         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5229         oldl1 = pmap_load(l1);
5230         KASSERT((oldl1 & ATTR_DESCR_MASK) == L1_BLOCK,
5231             ("pmap_demote_l1: Demoting a non-block entry"));
5232         KASSERT((va & L1_OFFSET) == 0,
5233             ("pmap_demote_l1: Invalid virtual address %#lx", va));
5234         KASSERT((oldl1 & ATTR_SW_MANAGED) == 0,
5235             ("pmap_demote_l1: Level 1 table shouldn't be managed"));
5236
5237         tmpl1 = 0;
5238         if (va <= (vm_offset_t)l1 && va + L1_SIZE > (vm_offset_t)l1) {
5239                 tmpl1 = kva_alloc(PAGE_SIZE);
5240                 if (tmpl1 == 0)
5241                         return (NULL);
5242         }
5243
5244         if ((ml2 = vm_page_alloc(NULL, 0, VM_ALLOC_INTERRUPT |
5245             VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) {
5246                 CTR2(KTR_PMAP, "pmap_demote_l1: failure for va %#lx"
5247                     " in pmap %p", va, pmap);
5248                 return (NULL);
5249         }
5250
5251         l2phys = VM_PAGE_TO_PHYS(ml2);
5252         l2 = (pt_entry_t *)PHYS_TO_DMAP(l2phys);
5253
5254         /* Address the range points at */
5255         phys = oldl1 & ~ATTR_MASK;
5256         /* The attributed from the old l1 table to be copied */
5257         newl2 = oldl1 & ATTR_MASK;
5258
5259         /* Create the new entries */
5260         for (i = 0; i < Ln_ENTRIES; i++) {
5261                 l2[i] = newl2 | phys;
5262                 phys += L2_SIZE;
5263         }
5264         KASSERT(l2[0] == ((oldl1 & ~ATTR_DESCR_MASK) | L2_BLOCK),
5265             ("Invalid l2 page (%lx != %lx)", l2[0],
5266             (oldl1 & ~ATTR_DESCR_MASK) | L2_BLOCK));
5267
5268         if (tmpl1 != 0) {
5269                 pmap_kenter(tmpl1, PAGE_SIZE,
5270                     DMAP_TO_PHYS((vm_offset_t)l1) & ~L3_OFFSET, CACHED_MEMORY);
5271                 l1 = (pt_entry_t *)(tmpl1 + ((vm_offset_t)l1 & PAGE_MASK));
5272         }
5273
5274         pmap_update_entry(pmap, l1, l2phys | L1_TABLE, va, PAGE_SIZE);
5275
5276         if (tmpl1 != 0) {
5277                 pmap_kremove(tmpl1);
5278                 kva_free(tmpl1, PAGE_SIZE);
5279         }
5280
5281         return (l2);
5282 }
5283
5284 static void
5285 pmap_demote_l2_abort(pmap_t pmap, vm_offset_t va, pt_entry_t *l2,
5286     struct rwlock **lockp)
5287 {
5288         struct spglist free;
5289
5290         SLIST_INIT(&free);
5291         (void)pmap_remove_l2(pmap, l2, va, pmap_load(pmap_l1(pmap, va)), &free,
5292             lockp);
5293         vm_page_free_pages_toq(&free, true);
5294 }
5295
5296 /*
5297  * Create an L3 table to map all addresses within an L2 mapping.
5298  */
5299 static pt_entry_t *
5300 pmap_demote_l2_locked(pmap_t pmap, pt_entry_t *l2, vm_offset_t va,
5301     struct rwlock **lockp)
5302 {
5303         pt_entry_t *l3, newl3, oldl2;
5304         vm_offset_t tmpl2;
5305         vm_paddr_t l3phys, phys;
5306         vm_page_t ml3;
5307         int i;
5308
5309         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5310         l3 = NULL;
5311         oldl2 = pmap_load(l2);
5312         KASSERT((oldl2 & ATTR_DESCR_MASK) == L2_BLOCK,
5313             ("pmap_demote_l2: Demoting a non-block entry"));
5314         va &= ~L2_OFFSET;
5315
5316         tmpl2 = 0;
5317         if (va <= (vm_offset_t)l2 && va + L2_SIZE > (vm_offset_t)l2) {
5318                 tmpl2 = kva_alloc(PAGE_SIZE);
5319                 if (tmpl2 == 0)
5320                         return (NULL);
5321         }
5322
5323         /*
5324          * Invalidate the 2MB page mapping and return "failure" if the
5325          * mapping was never accessed.
5326          */
5327         if ((oldl2 & ATTR_AF) == 0) {
5328                 KASSERT((oldl2 & ATTR_SW_WIRED) == 0,
5329                     ("pmap_demote_l2: a wired mapping is missing ATTR_AF"));
5330                 pmap_demote_l2_abort(pmap, va, l2, lockp);
5331                 CTR2(KTR_PMAP, "pmap_demote_l2: failure for va %#lx in pmap %p",
5332                     va, pmap);
5333                 goto fail;
5334         }
5335
5336         if ((ml3 = pmap_remove_pt_page(pmap, va)) == NULL) {
5337                 KASSERT((oldl2 & ATTR_SW_WIRED) == 0,
5338                     ("pmap_demote_l2: page table page for a wired mapping"
5339                     " is missing"));
5340
5341                 /*
5342                  * If the page table page is missing and the mapping
5343                  * is for a kernel address, the mapping must belong to
5344                  * the direct map.  Page table pages are preallocated
5345                  * for every other part of the kernel address space,
5346                  * so the direct map region is the only part of the
5347                  * kernel address space that must be handled here.
5348                  */
5349                 KASSERT(va < VM_MAXUSER_ADDRESS || VIRT_IN_DMAP(va),
5350                     ("pmap_demote_l2: No saved mpte for va %#lx", va));
5351
5352                 /*
5353                  * If the 2MB page mapping belongs to the direct map
5354                  * region of the kernel's address space, then the page
5355                  * allocation request specifies the highest possible
5356                  * priority (VM_ALLOC_INTERRUPT).  Otherwise, the
5357                  * priority is normal.
5358                  */
5359                 ml3 = vm_page_alloc(NULL, pmap_l2_pindex(va),
5360                     (VIRT_IN_DMAP(va) ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) |
5361                     VM_ALLOC_NOOBJ | VM_ALLOC_WIRED);
5362
5363                 /*
5364                  * If the allocation of the new page table page fails,
5365                  * invalidate the 2MB page mapping and return "failure".
5366                  */
5367                 if (ml3 == NULL) {
5368                         pmap_demote_l2_abort(pmap, va, l2, lockp);
5369                         CTR2(KTR_PMAP, "pmap_demote_l2: failure for va %#lx"
5370                             " in pmap %p", va, pmap);
5371                         goto fail;
5372                 }
5373
5374                 if (va < VM_MAXUSER_ADDRESS) {
5375                         ml3->wire_count = NL3PG;
5376                         pmap_resident_count_inc(pmap, 1);
5377                 }
5378         }
5379
5380         l3phys = VM_PAGE_TO_PHYS(ml3);
5381         l3 = (pt_entry_t *)PHYS_TO_DMAP(l3phys);
5382
5383         /* Address the range points at */
5384         phys = oldl2 & ~ATTR_MASK;
5385         /* The attributed from the old l2 table to be copied */
5386         newl3 = (oldl2 & (ATTR_MASK & ~ATTR_DESCR_MASK)) | L3_PAGE;
5387
5388         /*
5389          * If the page table page is not leftover from an earlier promotion,
5390          * initialize it.
5391          */
5392         if (ml3->valid == 0) {
5393                 for (i = 0; i < Ln_ENTRIES; i++) {
5394                         l3[i] = newl3 | phys;
5395                         phys += L3_SIZE;
5396                 }
5397         }
5398         KASSERT(l3[0] == ((oldl2 & ~ATTR_DESCR_MASK) | L3_PAGE),
5399             ("Invalid l3 page (%lx != %lx)", l3[0],
5400             (oldl2 & ~ATTR_DESCR_MASK) | L3_PAGE));
5401
5402         /*
5403          * Map the temporary page so we don't lose access to the l2 table.
5404          */
5405         if (tmpl2 != 0) {
5406                 pmap_kenter(tmpl2, PAGE_SIZE,
5407                     DMAP_TO_PHYS((vm_offset_t)l2) & ~L3_OFFSET, CACHED_MEMORY);
5408                 l2 = (pt_entry_t *)(tmpl2 + ((vm_offset_t)l2 & PAGE_MASK));
5409         }
5410
5411         /*
5412          * The spare PV entries must be reserved prior to demoting the
5413          * mapping, that is, prior to changing the PDE.  Otherwise, the state
5414          * of the L2 and the PV lists will be inconsistent, which can result
5415          * in reclaim_pv_chunk() attempting to remove a PV entry from the
5416          * wrong PV list and pmap_pv_demote_l2() failing to find the expected
5417          * PV entry for the 2MB page mapping that is being demoted.
5418          */
5419         if ((oldl2 & ATTR_SW_MANAGED) != 0)
5420                 reserve_pv_entries(pmap, Ln_ENTRIES - 1, lockp);
5421
5422         /*
5423          * Pass PAGE_SIZE so that a single TLB invalidation is performed on
5424          * the 2MB page mapping.
5425          */
5426         pmap_update_entry(pmap, l2, l3phys | L2_TABLE, va, PAGE_SIZE);
5427
5428         /*
5429          * Demote the PV entry.
5430          */
5431         if ((oldl2 & ATTR_SW_MANAGED) != 0)
5432                 pmap_pv_demote_l2(pmap, va, oldl2 & ~ATTR_MASK, lockp);
5433
5434         atomic_add_long(&pmap_l2_demotions, 1);
5435         CTR3(KTR_PMAP, "pmap_demote_l2: success for va %#lx"
5436             " in pmap %p %lx", va, pmap, l3[0]);
5437
5438 fail:
5439         if (tmpl2 != 0) {
5440                 pmap_kremove(tmpl2);
5441                 kva_free(tmpl2, PAGE_SIZE);
5442         }
5443
5444         return (l3);
5445
5446 }
5447
5448 static pt_entry_t *
5449 pmap_demote_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t va)
5450 {
5451         struct rwlock *lock;
5452         pt_entry_t *l3;
5453
5454         lock = NULL;
5455         l3 = pmap_demote_l2_locked(pmap, l2, va, &lock);
5456         if (lock != NULL)
5457                 rw_wunlock(lock);
5458         return (l3);
5459 }
5460
5461 /*
5462  * perform the pmap work for mincore
5463  */
5464 int
5465 pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa)
5466 {
5467         pt_entry_t *pte, tpte;
5468         vm_paddr_t mask, pa;
5469         int lvl, val;
5470         bool managed;
5471
5472         PMAP_LOCK(pmap);
5473 retry:
5474         val = 0;
5475         pte = pmap_pte(pmap, addr, &lvl);
5476         if (pte != NULL) {
5477                 tpte = pmap_load(pte);
5478
5479                 switch (lvl) {
5480                 case 3:
5481                         mask = L3_OFFSET;
5482                         break;
5483                 case 2:
5484                         mask = L2_OFFSET;
5485                         break;
5486                 case 1:
5487                         mask = L1_OFFSET;
5488                         break;
5489                 default:
5490                         panic("pmap_mincore: invalid level %d", lvl);
5491                 }
5492
5493                 val = MINCORE_INCORE;
5494                 if (lvl != 3)
5495                         val |= MINCORE_SUPER;
5496                 if (pmap_page_dirty(tpte))
5497                         val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER;
5498                 if ((tpte & ATTR_AF) == ATTR_AF)
5499                         val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER;
5500
5501                 managed = (tpte & ATTR_SW_MANAGED) == ATTR_SW_MANAGED;
5502                 pa = (tpte & ~ATTR_MASK) | (addr & mask);
5503         } else
5504                 managed = false;
5505
5506         if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) !=
5507             (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && managed) {
5508                 /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */
5509                 if (vm_page_pa_tryrelock(pmap, pa, locked_pa))
5510                         goto retry;
5511         } else
5512                 PA_UNLOCK_COND(*locked_pa);
5513         PMAP_UNLOCK(pmap);
5514
5515         return (val);
5516 }
5517
5518 void
5519 pmap_activate(struct thread *td)
5520 {
5521         pmap_t  pmap;
5522
5523         critical_enter();
5524         pmap = vmspace_pmap(td->td_proc->p_vmspace);
5525         td->td_proc->p_md.md_l0addr = vtophys(pmap->pm_l0);
5526         __asm __volatile("msr ttbr0_el1, %0" : :
5527             "r"(td->td_proc->p_md.md_l0addr));
5528         pmap_invalidate_all(pmap);
5529         critical_exit();
5530 }
5531
5532 struct pcb *
5533 pmap_switch(struct thread *old, struct thread *new)
5534 {
5535         pcpu_bp_harden bp_harden;
5536         struct pcb *pcb;
5537
5538         /* Store the new curthread */
5539         PCPU_SET(curthread, new);
5540
5541         /* And the new pcb */
5542         pcb = new->td_pcb;
5543         PCPU_SET(curpcb, pcb);
5544
5545         /*
5546          * TODO: We may need to flush the cache here if switching
5547          * to a user process.
5548          */
5549
5550         if (old == NULL ||
5551             old->td_proc->p_md.md_l0addr != new->td_proc->p_md.md_l0addr) {
5552                 __asm __volatile(
5553                     /* Switch to the new pmap */
5554                     "msr        ttbr0_el1, %0   \n"
5555                     "isb                        \n"
5556
5557                     /* Invalidate the TLB */
5558                     "dsb        ishst           \n"
5559                     "tlbi       vmalle1is       \n"
5560                     "dsb        ish             \n"
5561                     "isb                        \n"
5562                     : : "r"(new->td_proc->p_md.md_l0addr));
5563
5564                 /*
5565                  * Stop userspace from training the branch predictor against
5566                  * other processes. This will call into a CPU specific
5567                  * function that clears the branch predictor state.
5568                  */
5569                 bp_harden = PCPU_GET(bp_harden);
5570                 if (bp_harden != NULL)
5571                         bp_harden();
5572         }
5573
5574         return (pcb);
5575 }
5576
5577 void
5578 pmap_sync_icache(pmap_t pmap, vm_offset_t va, vm_size_t sz)
5579 {
5580
5581         if (va >= VM_MIN_KERNEL_ADDRESS) {
5582                 cpu_icache_sync_range(va, sz);
5583         } else {
5584                 u_int len, offset;
5585                 vm_paddr_t pa;
5586
5587                 /* Find the length of data in this page to flush */
5588                 offset = va & PAGE_MASK;
5589                 len = imin(PAGE_SIZE - offset, sz);
5590
5591                 while (sz != 0) {
5592                         /* Extract the physical address & find it in the DMAP */
5593                         pa = pmap_extract(pmap, va);
5594                         if (pa != 0)
5595                                 cpu_icache_sync_range(PHYS_TO_DMAP(pa), len);
5596
5597                         /* Move to the next page */
5598                         sz -= len;
5599                         va += len;
5600                         /* Set the length for the next iteration */
5601                         len = imin(PAGE_SIZE, sz);
5602                 }
5603         }
5604 }
5605
5606 int
5607 pmap_fault(pmap_t pmap, uint64_t esr, uint64_t far)
5608 {
5609 #ifdef SMP
5610         register_t intr;
5611         uint64_t par;
5612
5613         switch (ESR_ELx_EXCEPTION(esr)) {
5614         case EXCP_INSN_ABORT_L:
5615         case EXCP_INSN_ABORT:
5616         case EXCP_DATA_ABORT_L:
5617         case EXCP_DATA_ABORT:
5618                 break;
5619         default:
5620                 return (KERN_FAILURE);
5621         }
5622
5623         /* Data and insn aborts use same encoding for FCS field. */
5624         switch (esr & ISS_DATA_DFSC_MASK) {
5625         case ISS_DATA_DFSC_TF_L0:
5626         case ISS_DATA_DFSC_TF_L1:
5627         case ISS_DATA_DFSC_TF_L2:
5628         case ISS_DATA_DFSC_TF_L3:
5629                 PMAP_LOCK(pmap);
5630                 /* Ask the MMU to check the address */
5631                 intr = intr_disable();
5632                 if (pmap == kernel_pmap)
5633                         par = arm64_address_translate_s1e1r(far);
5634                 else
5635                         par = arm64_address_translate_s1e0r(far);
5636                 intr_restore(intr);
5637                 PMAP_UNLOCK(pmap);
5638
5639                 /*
5640                  * If the translation was successful the address was invalid
5641                  * due to a break-before-make sequence. We can unlock and
5642                  * return success to the trap handler.
5643                  */
5644                 if (PAR_SUCCESS(par))
5645                         return (KERN_SUCCESS);
5646                 break;
5647         default:
5648                 break;
5649         }
5650 #endif
5651
5652         return (KERN_FAILURE);
5653 }
5654
5655 /*
5656  *      Increase the starting virtual address of the given mapping if a
5657  *      different alignment might result in more superpage mappings.
5658  */
5659 void
5660 pmap_align_superpage(vm_object_t object, vm_ooffset_t offset,
5661     vm_offset_t *addr, vm_size_t size)
5662 {
5663         vm_offset_t superpage_offset;
5664
5665         if (size < L2_SIZE)
5666                 return;
5667         if (object != NULL && (object->flags & OBJ_COLORED) != 0)
5668                 offset += ptoa(object->pg_color);
5669         superpage_offset = offset & L2_OFFSET;
5670         if (size - ((L2_SIZE - superpage_offset) & L2_OFFSET) < L2_SIZE ||
5671             (*addr & L2_OFFSET) == superpage_offset)
5672                 return;
5673         if ((*addr & L2_OFFSET) < superpage_offset)
5674                 *addr = (*addr & ~L2_OFFSET) + superpage_offset;
5675         else
5676                 *addr = ((*addr + L2_OFFSET) & ~L2_OFFSET) + superpage_offset;
5677 }
5678
5679 /**
5680  * Get the kernel virtual address of a set of physical pages. If there are
5681  * physical addresses not covered by the DMAP perform a transient mapping
5682  * that will be removed when calling pmap_unmap_io_transient.
5683  *
5684  * \param page        The pages the caller wishes to obtain the virtual
5685  *                    address on the kernel memory map.
5686  * \param vaddr       On return contains the kernel virtual memory address
5687  *                    of the pages passed in the page parameter.
5688  * \param count       Number of pages passed in.
5689  * \param can_fault   TRUE if the thread using the mapped pages can take
5690  *                    page faults, FALSE otherwise.
5691  *
5692  * \returns TRUE if the caller must call pmap_unmap_io_transient when
5693  *          finished or FALSE otherwise.
5694  *
5695  */
5696 boolean_t
5697 pmap_map_io_transient(vm_page_t page[], vm_offset_t vaddr[], int count,
5698     boolean_t can_fault)
5699 {
5700         vm_paddr_t paddr;
5701         boolean_t needs_mapping;
5702         int error, i;
5703
5704         /*
5705          * Allocate any KVA space that we need, this is done in a separate
5706          * loop to prevent calling vmem_alloc while pinned.
5707          */
5708         needs_mapping = FALSE;
5709         for (i = 0; i < count; i++) {
5710                 paddr = VM_PAGE_TO_PHYS(page[i]);
5711                 if (__predict_false(!PHYS_IN_DMAP(paddr))) {
5712                         error = vmem_alloc(kernel_arena, PAGE_SIZE,
5713                             M_BESTFIT | M_WAITOK, &vaddr[i]);
5714                         KASSERT(error == 0, ("vmem_alloc failed: %d", error));
5715                         needs_mapping = TRUE;
5716                 } else {
5717                         vaddr[i] = PHYS_TO_DMAP(paddr);
5718                 }
5719         }
5720
5721         /* Exit early if everything is covered by the DMAP */
5722         if (!needs_mapping)
5723                 return (FALSE);
5724
5725         if (!can_fault)
5726                 sched_pin();
5727         for (i = 0; i < count; i++) {
5728                 paddr = VM_PAGE_TO_PHYS(page[i]);
5729                 if (!PHYS_IN_DMAP(paddr)) {
5730                         panic(
5731                            "pmap_map_io_transient: TODO: Map out of DMAP data");
5732                 }
5733         }
5734
5735         return (needs_mapping);
5736 }
5737
5738 void
5739 pmap_unmap_io_transient(vm_page_t page[], vm_offset_t vaddr[], int count,
5740     boolean_t can_fault)
5741 {
5742         vm_paddr_t paddr;
5743         int i;
5744
5745         if (!can_fault)
5746                 sched_unpin();
5747         for (i = 0; i < count; i++) {
5748                 paddr = VM_PAGE_TO_PHYS(page[i]);
5749                 if (!PHYS_IN_DMAP(paddr)) {
5750                         panic("ARM64TODO: pmap_unmap_io_transient: Unmap data");
5751                 }
5752         }
5753 }
5754
5755 boolean_t
5756 pmap_is_valid_memattr(pmap_t pmap __unused, vm_memattr_t mode)
5757 {
5758
5759         return (mode >= VM_MEMATTR_DEVICE && mode <= VM_MEMATTR_WRITE_THROUGH);
5760 }