2 * Copyright (c) 1991 Regents of the University of California.
4 * Copyright (c) 1994 John S. Dyson
6 * Copyright (c) 1994 David Greenman
8 * Copyright (c) 1998,2000 Doug Rabson
11 * This code is derived from software contributed to Berkeley by
12 * the Systems Programming Group of the University of Utah Computer
13 * Science Department and William Jolitz of UUNET Technologies Inc.
15 * Redistribution and use in source and binary forms, with or without
16 * modification, are permitted provided that the following conditions
18 * 1. Redistributions of source code must retain the above copyright
19 * notice, this list of conditions and the following disclaimer.
20 * 2. Redistributions in binary form must reproduce the above copyright
21 * notice, this list of conditions and the following disclaimer in the
22 * documentation and/or other materials provided with the distribution.
23 * 3. All advertising materials mentioning features or use of this software
24 * must display the following acknowledgement:
25 * This product includes software developed by the University of
26 * California, Berkeley and its contributors.
27 * 4. Neither the name of the University nor the names of its contributors
28 * may be used to endorse or promote products derived from this software
29 * without specific prior written permission.
31 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
32 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
33 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
34 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
35 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
36 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
37 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
38 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
39 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
40 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
43 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91
44 * from: i386 Id: pmap.c,v 1.193 1998/04/19 15:22:48 bde Exp
45 * with some ideas from NetBSD's alpha pmap
48 #include <sys/cdefs.h>
49 __FBSDID("$FreeBSD$");
53 #include <sys/param.h>
54 #include <sys/kernel.h>
57 #include <sys/mutex.h>
59 #include <sys/rwlock.h>
61 #include <sys/sysctl.h>
62 #include <sys/systm.h>
65 #include <vm/vm_param.h>
66 #include <vm/vm_page.h>
67 #include <vm/vm_map.h>
68 #include <vm/vm_object.h>
69 #include <vm/vm_pageout.h>
72 #include <machine/bootinfo.h>
73 #include <machine/efi.h>
74 #include <machine/md_var.h>
75 #include <machine/pal.h>
78 * Manages physical address maps.
80 * Since the information managed by this module is
81 * also stored by the logical address mapping module,
82 * this module may throw away valid virtual-to-physical
83 * mappings at almost any time. However, invalidations
84 * of virtual-to-physical mappings must be done as
87 * In order to cope with hardware architectures which
88 * make virtual-to-physical map invalidates expensive,
89 * this module may delay invalidate or reduced protection
90 * operations until such time as they are actually
91 * necessary. This module is given full information as
92 * to which processors are currently using which maps,
93 * and to when physical maps must be made correct.
97 * Following the Linux model, region IDs are allocated in groups of
98 * eight so that a single region ID can be used for as many RRs as we
99 * want by encoding the RR number into the low bits of the ID.
101 * We reserve region ID 0 for the kernel and allocate the remaining
102 * IDs for user pmaps.
104 * Region 0-3: User virtually mapped
105 * Region 4: PBVM and special mappings
106 * Region 5: Kernel virtual memory
107 * Region 6: Direct-mapped uncacheable
108 * Region 7: Direct-mapped cacheable
111 /* XXX move to a header. */
112 extern uint64_t ia64_gateway_page[];
114 #if !defined(DIAGNOSTIC)
115 #define PMAP_INLINE __inline
121 #define PV_STAT(x) do { x ; } while (0)
123 #define PV_STAT(x) do { } while (0)
126 #define pmap_accessed(lpte) ((lpte)->pte & PTE_ACCESSED)
127 #define pmap_dirty(lpte) ((lpte)->pte & PTE_DIRTY)
128 #define pmap_exec(lpte) ((lpte)->pte & PTE_AR_RX)
129 #define pmap_managed(lpte) ((lpte)->pte & PTE_MANAGED)
130 #define pmap_ppn(lpte) ((lpte)->pte & PTE_PPN_MASK)
131 #define pmap_present(lpte) ((lpte)->pte & PTE_PRESENT)
132 #define pmap_prot(lpte) (((lpte)->pte & PTE_PROT_MASK) >> 56)
133 #define pmap_wired(lpte) ((lpte)->pte & PTE_WIRED)
135 #define pmap_clear_accessed(lpte) (lpte)->pte &= ~PTE_ACCESSED
136 #define pmap_clear_dirty(lpte) (lpte)->pte &= ~PTE_DIRTY
137 #define pmap_clear_present(lpte) (lpte)->pte &= ~PTE_PRESENT
138 #define pmap_clear_wired(lpte) (lpte)->pte &= ~PTE_WIRED
140 #define pmap_set_wired(lpte) (lpte)->pte |= PTE_WIRED
143 * Individual PV entries are stored in per-pmap chunks. This saves
144 * space by eliminating the need to record the pmap within every PV
147 #if PAGE_SIZE == 8192
151 #elif PAGE_SIZE == 16384
158 TAILQ_ENTRY(pv_chunk) pc_list;
159 u_long pc_map[_NPCM]; /* bitmap; 1 = free */
160 TAILQ_ENTRY(pv_chunk) pc_lru;
161 u_long pc_spare[_NPCS];
162 struct pv_entry pc_pventry[_NPCPV];
166 * The VHPT bucket head structure.
175 * Statically allocated kernel pmap
177 struct pmap kernel_pmap_store;
179 vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */
180 vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */
183 * Kernel virtual memory management.
186 extern struct ia64_lpte ***ia64_kptdir;
188 #define KPTE_DIR0_INDEX(va) \
189 (((va) >> (3*PAGE_SHIFT-8)) & ((1<<(PAGE_SHIFT-3))-1))
190 #define KPTE_DIR1_INDEX(va) \
191 (((va) >> (2*PAGE_SHIFT-5)) & ((1<<(PAGE_SHIFT-3))-1))
192 #define KPTE_PTE_INDEX(va) \
193 (((va) >> PAGE_SHIFT) & ((1<<(PAGE_SHIFT-5))-1))
194 #define NKPTEPG (PAGE_SIZE / sizeof(struct ia64_lpte))
196 vm_offset_t kernel_vm_end;
198 /* Values for ptc.e. XXX values for SKI. */
199 static uint64_t pmap_ptc_e_base = 0x100000000;
200 static uint64_t pmap_ptc_e_count1 = 3;
201 static uint64_t pmap_ptc_e_count2 = 2;
202 static uint64_t pmap_ptc_e_stride1 = 0x2000;
203 static uint64_t pmap_ptc_e_stride2 = 0x100000000;
205 struct mtx pmap_ptc_mutex;
208 * Data for the RID allocator
210 static int pmap_ridcount;
211 static int pmap_rididx;
212 static int pmap_ridmapsz;
213 static int pmap_ridmax;
214 static uint64_t *pmap_ridmap;
215 struct mtx pmap_ridmutex;
217 static struct rwlock_padalign pvh_global_lock;
220 * Data for the pv entry allocation mechanism
222 static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks);
223 static int pv_entry_count;
226 * Data for allocating PTEs for user processes.
228 static uma_zone_t ptezone;
231 * Virtual Hash Page Table (VHPT) data.
233 /* SYSCTL_DECL(_machdep); */
234 static SYSCTL_NODE(_machdep, OID_AUTO, vhpt, CTLFLAG_RD, 0, "");
236 struct ia64_bucket *pmap_vhpt_bucket;
238 int pmap_vhpt_nbuckets;
239 SYSCTL_INT(_machdep_vhpt, OID_AUTO, nbuckets, CTLFLAG_RD,
240 &pmap_vhpt_nbuckets, 0, "");
242 int pmap_vhpt_log2size = 0;
243 TUNABLE_INT("machdep.vhpt.log2size", &pmap_vhpt_log2size);
244 SYSCTL_INT(_machdep_vhpt, OID_AUTO, log2size, CTLFLAG_RD,
245 &pmap_vhpt_log2size, 0, "");
247 static int pmap_vhpt_inserts;
248 SYSCTL_INT(_machdep_vhpt, OID_AUTO, inserts, CTLFLAG_RD,
249 &pmap_vhpt_inserts, 0, "");
251 static int pmap_vhpt_population(SYSCTL_HANDLER_ARGS);
252 SYSCTL_PROC(_machdep_vhpt, OID_AUTO, population, CTLTYPE_INT | CTLFLAG_RD,
253 NULL, 0, pmap_vhpt_population, "I", "");
255 static struct ia64_lpte *pmap_find_vhpt(vm_offset_t va);
257 static void free_pv_chunk(struct pv_chunk *pc);
258 static void free_pv_entry(pmap_t pmap, pv_entry_t pv);
259 static pv_entry_t get_pv_entry(pmap_t pmap, boolean_t try);
260 static vm_page_t pmap_pv_reclaim(pmap_t locked_pmap);
262 static void pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
263 vm_page_t m, vm_prot_t prot);
264 static void pmap_free_pte(struct ia64_lpte *pte, vm_offset_t va);
265 static int pmap_remove_pte(pmap_t pmap, struct ia64_lpte *pte,
266 vm_offset_t va, pv_entry_t pv, int freepte);
267 static int pmap_remove_vhpt(vm_offset_t va);
268 static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,
272 pmap_initialize_vhpt(vm_offset_t vhpt)
274 struct ia64_lpte *pte;
277 pte = (struct ia64_lpte *)vhpt;
278 for (i = 0; i < pmap_vhpt_nbuckets; i++) {
281 pte[i].tag = 1UL << 63; /* Invalid tag */
282 pte[i].chain = (uintptr_t)(pmap_vhpt_bucket + i);
288 pmap_alloc_vhpt(void)
294 size = 1UL << pmap_vhpt_log2size;
295 m = vm_page_alloc_contig(NULL, 0, VM_ALLOC_SYSTEM | VM_ALLOC_NOOBJ |
296 VM_ALLOC_WIRED, atop(size), 0UL, ~0UL, size, 0UL,
299 vhpt = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m));
300 pmap_initialize_vhpt(vhpt);
308 * Bootstrap the system enough to run with virtual memory.
313 struct ia64_pal_result res;
319 * Query the PAL Code to find the loop parameters for the
322 res = ia64_call_pal_static(PAL_PTCE_INFO, 0, 0, 0);
323 if (res.pal_status != 0)
324 panic("Can't configure ptc.e parameters");
325 pmap_ptc_e_base = res.pal_result[0];
326 pmap_ptc_e_count1 = res.pal_result[1] >> 32;
327 pmap_ptc_e_count2 = res.pal_result[1] & ((1L<<32) - 1);
328 pmap_ptc_e_stride1 = res.pal_result[2] >> 32;
329 pmap_ptc_e_stride2 = res.pal_result[2] & ((1L<<32) - 1);
331 printf("ptc.e base=0x%lx, count1=%ld, count2=%ld, "
332 "stride1=0x%lx, stride2=0x%lx\n",
339 mtx_init(&pmap_ptc_mutex, "PTC.G mutex", NULL, MTX_SPIN);
342 * Setup RIDs. RIDs 0..7 are reserved for the kernel.
344 * We currently need at least 19 bits in the RID because PID_MAX
345 * can only be encoded in 17 bits and we need RIDs for 4 regions
346 * per process. With PID_MAX equalling 99999 this means that we
347 * need to be able to encode 399996 (=4*PID_MAX).
348 * The Itanium processor only has 18 bits and the architected
349 * minimum is exactly that. So, we cannot use a PID based scheme
350 * in those cases. Enter pmap_ridmap...
351 * We should avoid the map when running on a processor that has
352 * implemented enough bits. This means that we should pass the
353 * process/thread ID to pmap. This we currently don't do, so we
354 * use the map anyway. However, we don't want to allocate a map
355 * that is large enough to cover the range dictated by the number
356 * of bits in the RID, because that may result in a RID map of
357 * 2MB in size for a 24-bit RID. A 64KB map is enough.
358 * The bottomline: we create a 32KB map when the processor only
359 * implements 18 bits (or when we can't figure it out). Otherwise
360 * we create a 64KB map.
362 res = ia64_call_pal_static(PAL_VM_SUMMARY, 0, 0, 0);
363 if (res.pal_status != 0) {
365 printf("Can't read VM Summary - assuming 18 Region ID bits\n");
366 ridbits = 18; /* guaranteed minimum */
368 ridbits = (res.pal_result[1] >> 8) & 0xff;
370 printf("Processor supports %d Region ID bits\n",
376 pmap_ridmax = (1 << ridbits);
377 pmap_ridmapsz = pmap_ridmax / 64;
378 pmap_ridmap = ia64_physmem_alloc(pmap_ridmax / 8, PAGE_SIZE);
379 pmap_ridmap[0] |= 0xff;
382 mtx_init(&pmap_ridmutex, "RID allocator lock", NULL, MTX_DEF);
385 * Allocate some memory for initial kernel 'page tables'.
387 ia64_kptdir = ia64_physmem_alloc(PAGE_SIZE, PAGE_SIZE);
389 kernel_vm_end = VM_INIT_KERNEL_ADDRESS;
392 * Determine a valid (mappable) VHPT size.
394 TUNABLE_INT_FETCH("machdep.vhpt.log2size", &pmap_vhpt_log2size);
395 if (pmap_vhpt_log2size == 0)
396 pmap_vhpt_log2size = 20;
397 else if (pmap_vhpt_log2size < 16)
398 pmap_vhpt_log2size = 16;
399 else if (pmap_vhpt_log2size > 28)
400 pmap_vhpt_log2size = 28;
401 if (pmap_vhpt_log2size & 1)
402 pmap_vhpt_log2size--;
404 size = 1UL << pmap_vhpt_log2size;
405 base = (uintptr_t)ia64_physmem_alloc(size, size);
407 panic("Unable to allocate VHPT");
409 PCPU_SET(md.vhpt, base);
411 printf("VHPT: address=%#lx, size=%#lx\n", base, size);
413 pmap_vhpt_nbuckets = size / sizeof(struct ia64_lpte);
414 pmap_vhpt_bucket = ia64_physmem_alloc(pmap_vhpt_nbuckets *
415 sizeof(struct ia64_bucket), PAGE_SIZE);
416 for (i = 0; i < pmap_vhpt_nbuckets; i++) {
417 /* Stolen memory is zeroed. */
418 mtx_init(&pmap_vhpt_bucket[i].mutex, "VHPT bucket lock", NULL,
419 MTX_NOWITNESS | MTX_SPIN);
422 pmap_initialize_vhpt(base);
424 ia64_set_pta(base + (1 << 8) + (pmap_vhpt_log2size << 2) + 1);
427 virtual_avail = VM_INIT_KERNEL_ADDRESS;
428 virtual_end = VM_MAX_KERNEL_ADDRESS;
431 * Initialize the kernel pmap (which is statically allocated).
433 PMAP_LOCK_INIT(kernel_pmap);
434 for (i = 0; i < IA64_VM_MINKERN_REGION; i++)
435 kernel_pmap->pm_rid[i] = 0;
436 TAILQ_INIT(&kernel_pmap->pm_pvchunk);
437 PCPU_SET(md.current_pmap, kernel_pmap);
440 * Initialize the global pv list lock.
442 rw_init(&pvh_global_lock, "pmap pv global");
444 /* Region 5 is mapped via the VHPT. */
445 ia64_set_rr(IA64_RR_BASE(5), (5 << 8) | (PAGE_SHIFT << 2) | 1);
448 * Clear out any random TLB entries left over from booting.
450 pmap_invalidate_all();
456 pmap_vhpt_population(SYSCTL_HANDLER_ARGS)
461 for (i = 0; i < pmap_vhpt_nbuckets; i++)
462 count += pmap_vhpt_bucket[i].length;
464 error = SYSCTL_OUT(req, &count, sizeof(count));
469 pmap_page_to_va(vm_page_t m)
474 pa = VM_PAGE_TO_PHYS(m);
475 va = (m->md.memattr == VM_MEMATTR_UNCACHEABLE) ? IA64_PHYS_TO_RR6(pa) :
476 IA64_PHYS_TO_RR7(pa);
481 * Initialize a vm_page's machine-dependent fields.
484 pmap_page_init(vm_page_t m)
487 TAILQ_INIT(&m->md.pv_list);
488 m->md.memattr = VM_MEMATTR_DEFAULT;
492 * Initialize the pmap module.
493 * Called by vm_init, to initialize any structures that the pmap
494 * system needs to map virtual memory.
500 ptezone = uma_zcreate("PT ENTRY", sizeof (struct ia64_lpte),
501 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM|UMA_ZONE_NOFREE);
505 /***************************************************
506 * Manipulate TLBs for a pmap
507 ***************************************************/
510 pmap_invalidate_page(vm_offset_t va)
512 struct ia64_lpte *pte;
519 vhpt_ofs = ia64_thash(va) - PCPU_GET(md.vhpt);
521 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
522 pte = (struct ia64_lpte *)(pc->pc_md.vhpt + vhpt_ofs);
523 atomic_cmpset_64(&pte->tag, tag, 1UL << 63);
526 mtx_lock_spin(&pmap_ptc_mutex);
528 ia64_ptc_ga(va, PAGE_SHIFT << 2);
532 mtx_unlock_spin(&pmap_ptc_mutex);
540 pmap_invalidate_all(void)
545 addr = pmap_ptc_e_base;
546 for (i = 0; i < pmap_ptc_e_count1; i++) {
547 for (j = 0; j < pmap_ptc_e_count2; j++) {
549 addr += pmap_ptc_e_stride2;
551 addr += pmap_ptc_e_stride1;
557 pmap_allocate_rid(void)
562 mtx_lock(&pmap_ridmutex);
563 if (pmap_ridcount == pmap_ridmax)
564 panic("pmap_allocate_rid: All Region IDs used");
566 /* Find an index with a free bit. */
567 while ((bits = pmap_ridmap[pmap_rididx]) == ~0UL) {
569 if (pmap_rididx == pmap_ridmapsz)
572 rid = pmap_rididx * 64;
574 /* Find a free bit. */
581 pmap_ridmap[pmap_rididx] |= bit;
583 mtx_unlock(&pmap_ridmutex);
589 pmap_free_rid(uint32_t rid)
595 bit = ~(1UL << (rid & 63));
597 mtx_lock(&pmap_ridmutex);
598 pmap_ridmap[idx] &= bit;
600 mtx_unlock(&pmap_ridmutex);
603 /***************************************************
604 * Page table page management routines.....
605 ***************************************************/
608 pmap_pinit0(struct pmap *pmap)
611 PMAP_LOCK_INIT(pmap);
612 /* kernel_pmap is the same as any other pmap. */
617 * Initialize a preallocated and zeroed pmap structure,
618 * such as one in a vmspace structure.
621 pmap_pinit(struct pmap *pmap)
625 for (i = 0; i < IA64_VM_MINKERN_REGION; i++)
626 pmap->pm_rid[i] = pmap_allocate_rid();
627 TAILQ_INIT(&pmap->pm_pvchunk);
628 bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
632 /***************************************************
633 * Pmap allocation/deallocation routines.
634 ***************************************************/
637 * Release any resources held by the given physical map.
638 * Called when a pmap initialized by pmap_pinit is being released.
639 * Should only be called if the map contains no valid mappings.
642 pmap_release(pmap_t pmap)
646 for (i = 0; i < IA64_VM_MINKERN_REGION; i++)
648 pmap_free_rid(pmap->pm_rid[i]);
652 * grow the number of kernel page table entries, if needed
655 pmap_growkernel(vm_offset_t addr)
657 struct ia64_lpte **dir1;
658 struct ia64_lpte *leaf;
661 while (kernel_vm_end <= addr) {
662 if (nkpt == PAGE_SIZE/8 + PAGE_SIZE*PAGE_SIZE/64)
663 panic("%s: out of kernel address space", __func__);
665 dir1 = ia64_kptdir[KPTE_DIR0_INDEX(kernel_vm_end)];
667 nkpg = vm_page_alloc(NULL, nkpt++,
668 VM_ALLOC_NOOBJ|VM_ALLOC_INTERRUPT|VM_ALLOC_WIRED);
670 panic("%s: cannot add dir. page", __func__);
672 dir1 = (struct ia64_lpte **)pmap_page_to_va(nkpg);
673 bzero(dir1, PAGE_SIZE);
674 ia64_kptdir[KPTE_DIR0_INDEX(kernel_vm_end)] = dir1;
677 nkpg = vm_page_alloc(NULL, nkpt++,
678 VM_ALLOC_NOOBJ|VM_ALLOC_INTERRUPT|VM_ALLOC_WIRED);
680 panic("%s: cannot add PTE page", __func__);
682 leaf = (struct ia64_lpte *)pmap_page_to_va(nkpg);
683 bzero(leaf, PAGE_SIZE);
684 dir1[KPTE_DIR1_INDEX(kernel_vm_end)] = leaf;
686 kernel_vm_end += PAGE_SIZE * NKPTEPG;
690 /***************************************************
691 * page management routines.
692 ***************************************************/
694 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE);
696 static __inline struct pv_chunk *
697 pv_to_chunk(pv_entry_t pv)
700 return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK));
703 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
705 #define PC_FREE_FULL 0xfffffffffffffffful
706 #define PC_FREE_PARTIAL \
707 ((1UL << (_NPCPV - sizeof(u_long) * 8 * (_NPCM - 1))) - 1)
709 #if PAGE_SIZE == 8192
710 static const u_long pc_freemask[_NPCM] = {
711 PC_FREE_FULL, PC_FREE_FULL, PC_FREE_FULL,
712 PC_FREE_FULL, PC_FREE_FULL, PC_FREE_PARTIAL
714 #elif PAGE_SIZE == 16384
715 static const u_long pc_freemask[_NPCM] = {
716 PC_FREE_FULL, PC_FREE_FULL, PC_FREE_FULL,
717 PC_FREE_FULL, PC_FREE_FULL, PC_FREE_FULL,
718 PC_FREE_FULL, PC_FREE_FULL, PC_FREE_FULL,
719 PC_FREE_FULL, PC_FREE_PARTIAL
723 static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");
725 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0,
726 "Current number of pv entries");
729 static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail;
731 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0,
732 "Current number of pv entry chunks");
733 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0,
734 "Current number of pv entry chunks allocated");
735 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0,
736 "Current number of pv entry chunks frees");
737 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, &pc_chunk_tryfail, 0,
738 "Number of times tried to get a chunk page but failed.");
740 static long pv_entry_frees, pv_entry_allocs;
741 static int pv_entry_spare;
743 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, &pv_entry_frees, 0,
744 "Current number of pv entry frees");
745 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, &pv_entry_allocs, 0,
746 "Current number of pv entry allocs");
747 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0,
748 "Current number of spare pv entries");
752 * We are in a serious low memory condition. Resort to
753 * drastic measures to free some pages so we can allocate
754 * another pv entry chunk.
757 pmap_pv_reclaim(pmap_t locked_pmap)
761 struct ia64_lpte *pte;
767 int bit, field, freed, idx;
769 PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED);
772 TAILQ_INIT(&newtail);
773 while ((pc = TAILQ_FIRST(&pv_chunks)) != NULL) {
774 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
775 if (pmap != pc->pc_pmap) {
777 if (pmap != locked_pmap) {
778 pmap_switch(locked_pmap);
783 /* Avoid deadlock and lock recursion. */
784 if (pmap > locked_pmap)
786 else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap)) {
788 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
795 * Destroy every non-wired, 8 KB page mapping in the chunk.
798 for (field = 0; field < _NPCM; field++) {
799 for (inuse = ~pc->pc_map[field] & pc_freemask[field];
800 inuse != 0; inuse &= ~(1UL << bit)) {
801 bit = ffsl(inuse) - 1;
802 idx = field * sizeof(inuse) * NBBY + bit;
803 pv = &pc->pc_pventry[idx];
805 pte = pmap_find_vhpt(va);
806 KASSERT(pte != NULL, ("pte"));
809 pmap_remove_vhpt(va);
810 pmap_invalidate_page(va);
811 m = PHYS_TO_VM_PAGE(pmap_ppn(pte));
812 if (pmap_accessed(pte))
813 vm_page_aflag_set(m, PGA_REFERENCED);
816 pmap_free_pte(pte, va);
817 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
818 if (TAILQ_EMPTY(&m->md.pv_list))
819 vm_page_aflag_clear(m, PGA_WRITEABLE);
820 pc->pc_map[field] |= 1UL << bit;
825 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
828 /* Every freed mapping is for a 8 KB page. */
829 pmap->pm_stats.resident_count -= freed;
830 PV_STAT(pv_entry_frees += freed);
831 PV_STAT(pv_entry_spare += freed);
832 pv_entry_count -= freed;
833 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
834 for (field = 0; field < _NPCM; field++)
835 if (pc->pc_map[field] != pc_freemask[field]) {
836 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,
838 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
841 * One freed pv entry in locked_pmap is
844 if (pmap == locked_pmap)
848 if (field == _NPCM) {
849 PV_STAT(pv_entry_spare -= _NPCPV);
850 PV_STAT(pc_chunk_count--);
851 PV_STAT(pc_chunk_frees++);
852 /* Entire chunk is free; return it. */
853 m_pc = PHYS_TO_VM_PAGE(IA64_RR_MASK((vm_offset_t)pc));
858 TAILQ_CONCAT(&pv_chunks, &newtail, pc_lru);
860 if (pmap != locked_pmap) {
861 pmap_switch(locked_pmap);
869 * free the pv_entry back to the free list
872 free_pv_entry(pmap_t pmap, pv_entry_t pv)
877 rw_assert(&pvh_global_lock, RA_WLOCKED);
878 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
879 PV_STAT(pv_entry_frees++);
880 PV_STAT(pv_entry_spare++);
882 pc = pv_to_chunk(pv);
883 idx = pv - &pc->pc_pventry[0];
884 field = idx / (sizeof(u_long) * NBBY);
885 bit = idx % (sizeof(u_long) * NBBY);
886 pc->pc_map[field] |= 1ul << bit;
887 for (idx = 0; idx < _NPCM; idx++)
888 if (pc->pc_map[idx] != pc_freemask[idx]) {
890 * 98% of the time, pc is already at the head of the
891 * list. If it isn't already, move it to the head.
893 if (__predict_false(TAILQ_FIRST(&pmap->pm_pvchunk) !=
895 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
896 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,
901 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
906 free_pv_chunk(struct pv_chunk *pc)
910 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
911 PV_STAT(pv_entry_spare -= _NPCPV);
912 PV_STAT(pc_chunk_count--);
913 PV_STAT(pc_chunk_frees++);
914 /* entire chunk is free, return it */
915 m = PHYS_TO_VM_PAGE(IA64_RR_MASK((vm_offset_t)pc));
916 vm_page_unwire(m, 0);
921 * get a new pv_entry, allocating a block from the system
925 get_pv_entry(pmap_t pmap, boolean_t try)
932 rw_assert(&pvh_global_lock, RA_WLOCKED);
933 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
934 PV_STAT(pv_entry_allocs++);
937 pc = TAILQ_FIRST(&pmap->pm_pvchunk);
939 for (field = 0; field < _NPCM; field++) {
940 if (pc->pc_map[field]) {
941 bit = ffsl(pc->pc_map[field]) - 1;
946 idx = field * sizeof(pc->pc_map[field]) * NBBY + bit;
947 pv = &pc->pc_pventry[idx];
948 pc->pc_map[field] &= ~(1ul << bit);
949 /* If this was the last item, move it to tail */
950 for (field = 0; field < _NPCM; field++)
951 if (pc->pc_map[field] != 0) {
952 PV_STAT(pv_entry_spare--);
953 return (pv); /* not full, return */
955 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
956 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
957 PV_STAT(pv_entry_spare--);
961 /* No free items, allocate another chunk */
962 m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ |
967 PV_STAT(pc_chunk_tryfail++);
970 m = pmap_pv_reclaim(pmap);
974 PV_STAT(pc_chunk_count++);
975 PV_STAT(pc_chunk_allocs++);
976 pc = (struct pv_chunk *)IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m));
978 pc->pc_map[0] = pc_freemask[0] & ~1ul; /* preallocated bit 0 */
979 for (field = 1; field < _NPCM; field++)
980 pc->pc_map[field] = pc_freemask[field];
981 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
982 pv = &pc->pc_pventry[0];
983 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
984 PV_STAT(pv_entry_spare += _NPCPV - 1);
989 * Conditionally create a pv entry.
992 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
996 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
997 rw_assert(&pvh_global_lock, RA_WLOCKED);
998 if ((pv = get_pv_entry(pmap, TRUE)) != NULL) {
1000 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
1007 * Add an ia64_lpte to the VHPT.
1010 pmap_enter_vhpt(struct ia64_lpte *pte, vm_offset_t va)
1012 struct ia64_bucket *bckt;
1013 struct ia64_lpte *vhpte;
1016 /* Can fault, so get it out of the way. */
1017 pte_pa = ia64_tpa((vm_offset_t)pte);
1019 vhpte = (struct ia64_lpte *)ia64_thash(va);
1020 bckt = (struct ia64_bucket *)vhpte->chain;
1022 mtx_lock_spin(&bckt->mutex);
1023 pte->chain = bckt->chain;
1025 bckt->chain = pte_pa;
1027 pmap_vhpt_inserts++;
1029 mtx_unlock_spin(&bckt->mutex);
1033 * Remove the ia64_lpte matching va from the VHPT. Return zero if it
1034 * worked or an appropriate error code otherwise.
1037 pmap_remove_vhpt(vm_offset_t va)
1039 struct ia64_bucket *bckt;
1040 struct ia64_lpte *pte;
1041 struct ia64_lpte *lpte;
1042 struct ia64_lpte *vhpte;
1043 uint64_t chain, tag;
1045 tag = ia64_ttag(va);
1046 vhpte = (struct ia64_lpte *)ia64_thash(va);
1047 bckt = (struct ia64_bucket *)vhpte->chain;
1050 mtx_lock_spin(&bckt->mutex);
1051 chain = bckt->chain;
1052 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
1053 while (chain != 0 && pte->tag != tag) {
1056 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
1059 mtx_unlock_spin(&bckt->mutex);
1063 /* Snip this pv_entry out of the collision chain. */
1065 bckt->chain = pte->chain;
1067 lpte->chain = pte->chain;
1071 mtx_unlock_spin(&bckt->mutex);
1076 * Find the ia64_lpte for the given va, if any.
1078 static struct ia64_lpte *
1079 pmap_find_vhpt(vm_offset_t va)
1081 struct ia64_bucket *bckt;
1082 struct ia64_lpte *pte;
1083 uint64_t chain, tag;
1085 tag = ia64_ttag(va);
1086 pte = (struct ia64_lpte *)ia64_thash(va);
1087 bckt = (struct ia64_bucket *)pte->chain;
1089 mtx_lock_spin(&bckt->mutex);
1090 chain = bckt->chain;
1091 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
1092 while (chain != 0 && pte->tag != tag) {
1094 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
1096 mtx_unlock_spin(&bckt->mutex);
1097 return ((chain != 0) ? pte : NULL);
1101 * Remove an entry from the list of managed mappings.
1104 pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va, pv_entry_t pv)
1107 rw_assert(&pvh_global_lock, RA_WLOCKED);
1109 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
1110 if (pmap == PV_PMAP(pv) && va == pv->pv_va)
1116 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
1117 if (TAILQ_FIRST(&m->md.pv_list) == NULL)
1118 vm_page_aflag_clear(m, PGA_WRITEABLE);
1120 free_pv_entry(pmap, pv);
1128 * Create a pv entry for page at pa for
1132 pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
1136 rw_assert(&pvh_global_lock, RA_WLOCKED);
1137 pv = get_pv_entry(pmap, FALSE);
1139 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
1143 * Routine: pmap_extract
1145 * Extract the physical page address associated
1146 * with the given map/virtual_address pair.
1149 pmap_extract(pmap_t pmap, vm_offset_t va)
1151 struct ia64_lpte *pte;
1157 oldpmap = pmap_switch(pmap);
1158 pte = pmap_find_vhpt(va);
1159 if (pte != NULL && pmap_present(pte))
1161 pmap_switch(oldpmap);
1167 * Routine: pmap_extract_and_hold
1169 * Atomically extract and hold the physical page
1170 * with the given pmap and virtual address pair
1171 * if that mapping permits the given protection.
1174 pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
1176 struct ia64_lpte *pte;
1184 oldpmap = pmap_switch(pmap);
1186 pte = pmap_find_vhpt(va);
1187 if (pte != NULL && pmap_present(pte) &&
1188 (pmap_prot(pte) & prot) == prot) {
1189 m = PHYS_TO_VM_PAGE(pmap_ppn(pte));
1190 if (vm_page_pa_tryrelock(pmap, pmap_ppn(pte), &pa))
1195 pmap_switch(oldpmap);
1200 /***************************************************
1201 * Low level mapping routines.....
1202 ***************************************************/
1205 * Find the kernel lpte for mapping the given virtual address, which
1206 * must be in the part of region 5 which we can cover with our kernel
1209 static struct ia64_lpte *
1210 pmap_find_kpte(vm_offset_t va)
1212 struct ia64_lpte **dir1;
1213 struct ia64_lpte *leaf;
1215 KASSERT((va >> 61) == 5,
1216 ("kernel mapping 0x%lx not in region 5", va));
1217 KASSERT(va < kernel_vm_end,
1218 ("kernel mapping 0x%lx out of range", va));
1220 dir1 = ia64_kptdir[KPTE_DIR0_INDEX(va)];
1221 leaf = dir1[KPTE_DIR1_INDEX(va)];
1222 return (&leaf[KPTE_PTE_INDEX(va)]);
1226 * Find a pte suitable for mapping a user-space address. If one exists
1227 * in the VHPT, that one will be returned, otherwise a new pte is
1230 static struct ia64_lpte *
1231 pmap_find_pte(vm_offset_t va)
1233 struct ia64_lpte *pte;
1235 if (va >= VM_MAXUSER_ADDRESS)
1236 return pmap_find_kpte(va);
1238 pte = pmap_find_vhpt(va);
1240 pte = uma_zalloc(ptezone, M_NOWAIT | M_ZERO);
1241 pte->tag = 1UL << 63;
1247 * Free a pte which is now unused. This simply returns it to the zone
1248 * allocator if it is a user mapping. For kernel mappings, clear the
1249 * valid bit to make it clear that the mapping is not currently used.
1252 pmap_free_pte(struct ia64_lpte *pte, vm_offset_t va)
1254 if (va < VM_MAXUSER_ADDRESS)
1255 uma_zfree(ptezone, pte);
1257 pmap_clear_present(pte);
1260 static PMAP_INLINE void
1261 pmap_pte_prot(pmap_t pm, struct ia64_lpte *pte, vm_prot_t prot)
1263 static long prot2ar[4] = {
1264 PTE_AR_R, /* VM_PROT_NONE */
1265 PTE_AR_RW, /* VM_PROT_WRITE */
1266 PTE_AR_RX|PTE_ED, /* VM_PROT_EXECUTE */
1267 PTE_AR_RWX|PTE_ED /* VM_PROT_WRITE|VM_PROT_EXECUTE */
1270 pte->pte &= ~(PTE_PROT_MASK | PTE_PL_MASK | PTE_AR_MASK | PTE_ED);
1271 pte->pte |= (uint64_t)(prot & VM_PROT_ALL) << 56;
1272 pte->pte |= (prot == VM_PROT_NONE || pm == kernel_pmap)
1273 ? PTE_PL_KERN : PTE_PL_USER;
1274 pte->pte |= prot2ar[(prot & VM_PROT_ALL) >> 1];
1277 static PMAP_INLINE void
1278 pmap_pte_attr(struct ia64_lpte *pte, vm_memattr_t ma)
1281 pte->pte &= ~PTE_MA_MASK;
1282 pte->pte |= (ma & PTE_MA_MASK);
1286 * Set a pte to contain a valid mapping and enter it in the VHPT. If
1287 * the pte was orginally valid, then its assumed to already be in the
1289 * This functions does not set the protection bits. It's expected
1290 * that those have been set correctly prior to calling this function.
1293 pmap_set_pte(struct ia64_lpte *pte, vm_offset_t va, vm_offset_t pa,
1294 boolean_t wired, boolean_t managed)
1297 pte->pte &= PTE_PROT_MASK | PTE_MA_MASK | PTE_PL_MASK |
1298 PTE_AR_MASK | PTE_ED;
1299 pte->pte |= PTE_PRESENT;
1300 pte->pte |= (managed) ? PTE_MANAGED : (PTE_DIRTY | PTE_ACCESSED);
1301 pte->pte |= (wired) ? PTE_WIRED : 0;
1302 pte->pte |= pa & PTE_PPN_MASK;
1304 pte->itir = PAGE_SHIFT << 2;
1308 pte->tag = ia64_ttag(va);
1312 * Remove the (possibly managed) mapping represented by pte from the
1316 pmap_remove_pte(pmap_t pmap, struct ia64_lpte *pte, vm_offset_t va,
1317 pv_entry_t pv, int freepte)
1323 * First remove from the VHPT.
1325 error = pmap_remove_vhpt(va);
1326 KASSERT(error == 0, ("%s: pmap_remove_vhpt returned %d",
1329 pmap_invalidate_page(va);
1331 if (pmap_wired(pte))
1332 pmap->pm_stats.wired_count -= 1;
1334 pmap->pm_stats.resident_count -= 1;
1335 if (pmap_managed(pte)) {
1336 m = PHYS_TO_VM_PAGE(pmap_ppn(pte));
1337 if (pmap_dirty(pte))
1339 if (pmap_accessed(pte))
1340 vm_page_aflag_set(m, PGA_REFERENCED);
1342 error = pmap_remove_entry(pmap, m, va, pv);
1345 pmap_free_pte(pte, va);
1351 * Extract the physical page address associated with a kernel
1355 pmap_kextract(vm_offset_t va)
1357 struct ia64_lpte *pte;
1358 uint64_t *pbvm_pgtbl;
1362 KASSERT(va >= VM_MAXUSER_ADDRESS, ("Must be kernel VA"));
1364 /* Regions 6 and 7 are direct mapped. */
1365 if (va >= IA64_RR_BASE(6)) {
1366 pa = IA64_RR_MASK(va);
1370 /* Region 5 is our KVA. Bail out if the VA is beyond our limits. */
1371 if (va >= kernel_vm_end)
1373 if (va >= VM_INIT_KERNEL_ADDRESS) {
1374 pte = pmap_find_kpte(va);
1375 pa = pmap_present(pte) ? pmap_ppn(pte) | (va & PAGE_MASK) : 0;
1379 /* The PBVM page table. */
1380 if (va >= IA64_PBVM_PGTBL + bootinfo->bi_pbvm_pgtblsz)
1382 if (va >= IA64_PBVM_PGTBL) {
1383 pa = (va - IA64_PBVM_PGTBL) + bootinfo->bi_pbvm_pgtbl;
1387 /* The PBVM itself. */
1388 if (va >= IA64_PBVM_BASE) {
1389 pbvm_pgtbl = (void *)IA64_PBVM_PGTBL;
1390 idx = (va - IA64_PBVM_BASE) >> IA64_PBVM_PAGE_SHIFT;
1391 if (idx >= (bootinfo->bi_pbvm_pgtblsz >> 3))
1393 if ((pbvm_pgtbl[idx] & PTE_PRESENT) == 0)
1395 pa = (pbvm_pgtbl[idx] & PTE_PPN_MASK) +
1396 (va & IA64_PBVM_PAGE_MASK);
1401 printf("XXX: %s: va=%#lx is invalid\n", __func__, va);
1410 * Add a list of wired pages to the kva this routine is only used for
1411 * temporary kernel mappings that do not need to have page modification
1412 * or references recorded. Note that old mappings are simply written
1413 * over. The page is effectively wired, but it's customary to not have
1414 * the PTE reflect that, nor update statistics.
1417 pmap_qenter(vm_offset_t va, vm_page_t *m, int count)
1419 struct ia64_lpte *pte;
1422 for (i = 0; i < count; i++) {
1423 pte = pmap_find_kpte(va);
1424 if (pmap_present(pte))
1425 pmap_invalidate_page(va);
1427 pmap_enter_vhpt(pte, va);
1428 pmap_pte_prot(kernel_pmap, pte, VM_PROT_ALL);
1429 pmap_pte_attr(pte, m[i]->md.memattr);
1430 pmap_set_pte(pte, va, VM_PAGE_TO_PHYS(m[i]), FALSE, FALSE);
1436 * this routine jerks page mappings from the
1437 * kernel -- it is meant only for temporary mappings.
1440 pmap_qremove(vm_offset_t va, int count)
1442 struct ia64_lpte *pte;
1445 for (i = 0; i < count; i++) {
1446 pte = pmap_find_kpte(va);
1447 if (pmap_present(pte)) {
1448 pmap_remove_vhpt(va);
1449 pmap_invalidate_page(va);
1450 pmap_clear_present(pte);
1457 * Add a wired page to the kva. As for pmap_qenter(), it's customary
1458 * to not have the PTE reflect that, nor update statistics.
1461 pmap_kenter(vm_offset_t va, vm_offset_t pa)
1463 struct ia64_lpte *pte;
1465 pte = pmap_find_kpte(va);
1466 if (pmap_present(pte))
1467 pmap_invalidate_page(va);
1469 pmap_enter_vhpt(pte, va);
1470 pmap_pte_prot(kernel_pmap, pte, VM_PROT_ALL);
1471 pmap_pte_attr(pte, VM_MEMATTR_DEFAULT);
1472 pmap_set_pte(pte, va, pa, FALSE, FALSE);
1476 * Remove a page from the kva
1479 pmap_kremove(vm_offset_t va)
1481 struct ia64_lpte *pte;
1483 pte = pmap_find_kpte(va);
1484 if (pmap_present(pte)) {
1485 pmap_remove_vhpt(va);
1486 pmap_invalidate_page(va);
1487 pmap_clear_present(pte);
1492 * Used to map a range of physical addresses into kernel
1493 * virtual address space.
1495 * The value passed in '*virt' is a suggested virtual address for
1496 * the mapping. Architectures which can support a direct-mapped
1497 * physical to virtual region can return the appropriate address
1498 * within that region, leaving '*virt' unchanged. Other
1499 * architectures should map the pages starting at '*virt' and
1500 * update '*virt' with the first usable address after the mapped
1504 pmap_map(vm_offset_t *virt, vm_offset_t start, vm_offset_t end, int prot)
1506 return IA64_PHYS_TO_RR7(start);
1510 * Remove the given range of addresses from the specified map.
1512 * It is assumed that the start and end are properly
1513 * rounded to the page size.
1515 * Sparsely used ranges are inefficiently removed. The VHPT is
1516 * probed for every page within the range. XXX
1519 pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
1523 struct ia64_lpte *pte;
1526 * Perform an unsynchronized read. This is, however, safe.
1528 if (pmap->pm_stats.resident_count == 0)
1531 rw_wlock(&pvh_global_lock);
1533 oldpmap = pmap_switch(pmap);
1534 for (va = sva; va < eva; va += PAGE_SIZE) {
1535 pte = pmap_find_vhpt(va);
1537 pmap_remove_pte(pmap, pte, va, 0, 1);
1539 rw_wunlock(&pvh_global_lock);
1540 pmap_switch(oldpmap);
1545 * Routine: pmap_remove_all
1547 * Removes this physical page from
1548 * all physical maps in which it resides.
1549 * Reflects back modify bits to the pager.
1552 * Original versions of this routine were very
1553 * inefficient because they iteratively called
1554 * pmap_remove (slow...)
1558 pmap_remove_all(vm_page_t m)
1563 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
1564 ("pmap_remove_all: page %p is not managed", m));
1565 rw_wlock(&pvh_global_lock);
1566 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
1567 struct ia64_lpte *pte;
1568 pmap_t pmap = PV_PMAP(pv);
1569 vm_offset_t va = pv->pv_va;
1572 oldpmap = pmap_switch(pmap);
1573 pte = pmap_find_vhpt(va);
1574 KASSERT(pte != NULL, ("pte"));
1575 if (pmap_ppn(pte) != VM_PAGE_TO_PHYS(m))
1576 panic("pmap_remove_all: pv_table for %lx is inconsistent", VM_PAGE_TO_PHYS(m));
1577 pmap_remove_pte(pmap, pte, va, pv, 1);
1578 pmap_switch(oldpmap);
1581 vm_page_aflag_clear(m, PGA_WRITEABLE);
1582 rw_wunlock(&pvh_global_lock);
1586 * Set the physical protection on the
1587 * specified range of this map as requested.
1590 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
1593 struct ia64_lpte *pte;
1595 if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
1596 pmap_remove(pmap, sva, eva);
1600 if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) ==
1601 (VM_PROT_WRITE|VM_PROT_EXECUTE))
1604 if ((sva & PAGE_MASK) || (eva & PAGE_MASK))
1605 panic("pmap_protect: unaligned addresses");
1608 oldpmap = pmap_switch(pmap);
1609 for ( ; sva < eva; sva += PAGE_SIZE) {
1610 /* If page is invalid, skip this page */
1611 pte = pmap_find_vhpt(sva);
1615 /* If there's no change, skip it too */
1616 if (pmap_prot(pte) == prot)
1619 if ((prot & VM_PROT_WRITE) == 0 &&
1620 pmap_managed(pte) && pmap_dirty(pte)) {
1621 vm_paddr_t pa = pmap_ppn(pte);
1622 vm_page_t m = PHYS_TO_VM_PAGE(pa);
1625 pmap_clear_dirty(pte);
1628 if (prot & VM_PROT_EXECUTE)
1629 ia64_sync_icache(sva, PAGE_SIZE);
1631 pmap_pte_prot(pmap, pte, prot);
1632 pmap_invalidate_page(sva);
1634 pmap_switch(oldpmap);
1639 * Insert the given physical page (p) at
1640 * the specified virtual address (v) in the
1641 * target physical map with the protection requested.
1643 * If specified, the page will be wired down, meaning
1644 * that the related pte can not be reclaimed.
1646 * NB: This is the only routine which MAY NOT lazy-evaluate
1647 * or lose information. That is, this routine must actually
1648 * insert this page into the given map NOW.
1651 pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t m,
1652 vm_prot_t prot, boolean_t wired)
1657 struct ia64_lpte origpte;
1658 struct ia64_lpte *pte;
1659 boolean_t icache_inval, managed;
1661 rw_wlock(&pvh_global_lock);
1663 oldpmap = pmap_switch(pmap);
1666 KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig"));
1667 KASSERT((m->oflags & VPO_UNMANAGED) != 0 || vm_page_xbusied(m),
1668 ("pmap_enter: page %p is not busy", m));
1671 * Find (or create) a pte for the given mapping.
1673 while ((pte = pmap_find_pte(va)) == NULL) {
1674 pmap_switch(oldpmap);
1676 rw_wunlock(&pvh_global_lock);
1678 rw_wlock(&pvh_global_lock);
1680 oldpmap = pmap_switch(pmap);
1683 if (!pmap_present(pte)) {
1685 pmap_enter_vhpt(pte, va);
1687 opa = pmap_ppn(pte);
1689 pa = VM_PAGE_TO_PHYS(m);
1691 icache_inval = (prot & VM_PROT_EXECUTE) ? TRUE : FALSE;
1694 * Mapping has not changed, must be protection or wiring change.
1698 * Wiring change, just update stats. We don't worry about
1699 * wiring PT pages as they remain resident as long as there
1700 * are valid mappings in them. Hence, if a user page is wired,
1701 * the PT page will be also.
1703 if (wired && !pmap_wired(&origpte))
1704 pmap->pm_stats.wired_count++;
1705 else if (!wired && pmap_wired(&origpte))
1706 pmap->pm_stats.wired_count--;
1708 managed = (pmap_managed(&origpte)) ? TRUE : FALSE;
1711 * We might be turning off write access to the page,
1712 * so we go ahead and sense modify status. Otherwise,
1713 * we can avoid I-cache invalidation if the page
1714 * already allowed execution.
1716 if (managed && pmap_dirty(&origpte))
1718 else if (pmap_exec(&origpte))
1719 icache_inval = FALSE;
1721 pmap_invalidate_page(va);
1726 * Mapping has changed, invalidate old range and fall
1727 * through to handle validating new mapping.
1730 pmap_remove_pte(pmap, pte, va, 0, 0);
1731 pmap_enter_vhpt(pte, va);
1735 * Enter on the PV list if part of our managed memory.
1737 if ((m->oflags & VPO_UNMANAGED) == 0) {
1738 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva,
1739 ("pmap_enter: managed mapping within the clean submap"));
1740 pmap_insert_entry(pmap, va, m);
1745 * Increment counters
1747 pmap->pm_stats.resident_count++;
1749 pmap->pm_stats.wired_count++;
1754 * Now validate mapping with desired protection/wiring. This
1755 * adds the pte to the VHPT if necessary.
1757 pmap_pte_prot(pmap, pte, prot);
1758 pmap_pte_attr(pte, m->md.memattr);
1759 pmap_set_pte(pte, va, pa, wired, managed);
1761 /* Invalidate the I-cache when needed. */
1763 ia64_sync_icache(va, PAGE_SIZE);
1765 if ((prot & VM_PROT_WRITE) != 0 && managed)
1766 vm_page_aflag_set(m, PGA_WRITEABLE);
1767 rw_wunlock(&pvh_global_lock);
1768 pmap_switch(oldpmap);
1773 * Maps a sequence of resident pages belonging to the same object.
1774 * The sequence begins with the given page m_start. This page is
1775 * mapped at the given virtual address start. Each subsequent page is
1776 * mapped at a virtual address that is offset from start by the same
1777 * amount as the page is offset from m_start within the object. The
1778 * last page in the sequence is the page with the largest offset from
1779 * m_start that can be mapped at a virtual address less than the given
1780 * virtual address end. Not every virtual page between start and end
1781 * is mapped; only those for which a resident page exists with the
1782 * corresponding offset from m_start are mapped.
1785 pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,
1786 vm_page_t m_start, vm_prot_t prot)
1790 vm_pindex_t diff, psize;
1792 VM_OBJECT_ASSERT_LOCKED(m_start->object);
1794 psize = atop(end - start);
1796 rw_wlock(&pvh_global_lock);
1798 oldpmap = pmap_switch(pmap);
1799 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
1800 pmap_enter_quick_locked(pmap, start + ptoa(diff), m, prot);
1801 m = TAILQ_NEXT(m, listq);
1803 rw_wunlock(&pvh_global_lock);
1804 pmap_switch(oldpmap);
1809 * this code makes some *MAJOR* assumptions:
1810 * 1. Current pmap & pmap exists.
1813 * 4. No page table pages.
1814 * but is *MUCH* faster than pmap_enter...
1818 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
1822 rw_wlock(&pvh_global_lock);
1824 oldpmap = pmap_switch(pmap);
1825 pmap_enter_quick_locked(pmap, va, m, prot);
1826 rw_wunlock(&pvh_global_lock);
1827 pmap_switch(oldpmap);
1832 pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
1835 struct ia64_lpte *pte;
1838 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva ||
1839 (m->oflags & VPO_UNMANAGED) != 0,
1840 ("pmap_enter_quick_locked: managed mapping within the clean submap"));
1841 rw_assert(&pvh_global_lock, RA_WLOCKED);
1842 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1844 if ((pte = pmap_find_pte(va)) == NULL)
1847 if (!pmap_present(pte)) {
1848 /* Enter on the PV list if the page is managed. */
1849 if ((m->oflags & VPO_UNMANAGED) == 0) {
1850 if (!pmap_try_insert_pv_entry(pmap, va, m)) {
1851 pmap_free_pte(pte, va);
1858 /* Increment counters. */
1859 pmap->pm_stats.resident_count++;
1861 /* Initialise with R/O protection and enter into VHPT. */
1862 pmap_enter_vhpt(pte, va);
1863 pmap_pte_prot(pmap, pte,
1864 prot & (VM_PROT_READ | VM_PROT_EXECUTE));
1865 pmap_pte_attr(pte, m->md.memattr);
1866 pmap_set_pte(pte, va, VM_PAGE_TO_PHYS(m), FALSE, managed);
1868 if (prot & VM_PROT_EXECUTE)
1869 ia64_sync_icache(va, PAGE_SIZE);
1874 * pmap_object_init_pt preloads the ptes for a given object
1875 * into the specified pmap. This eliminates the blast of soft
1876 * faults on process startup and immediately after an mmap.
1879 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr,
1880 vm_object_t object, vm_pindex_t pindex,
1884 VM_OBJECT_ASSERT_WLOCKED(object);
1885 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG,
1886 ("pmap_object_init_pt: non-device object"));
1890 * Routine: pmap_change_wiring
1891 * Function: Change the wiring attribute for a map/virtual-address
1893 * In/out conditions:
1894 * The mapping must already exist in the pmap.
1897 pmap_change_wiring(pmap, va, wired)
1898 register pmap_t pmap;
1903 struct ia64_lpte *pte;
1906 oldpmap = pmap_switch(pmap);
1908 pte = pmap_find_vhpt(va);
1909 KASSERT(pte != NULL, ("pte"));
1910 if (wired && !pmap_wired(pte)) {
1911 pmap->pm_stats.wired_count++;
1912 pmap_set_wired(pte);
1913 } else if (!wired && pmap_wired(pte)) {
1914 pmap->pm_stats.wired_count--;
1915 pmap_clear_wired(pte);
1918 pmap_switch(oldpmap);
1925 * Copy the range specified by src_addr/len
1926 * from the source map to the range dst_addr/len
1927 * in the destination map.
1929 * This routine is only advisory and need not do anything.
1933 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
1934 vm_offset_t src_addr)
1940 * pmap_zero_page zeros the specified hardware page by
1941 * mapping it into virtual memory and using bzero to clear
1946 pmap_zero_page(vm_page_t m)
1950 p = (void *)pmap_page_to_va(m);
1951 bzero(p, PAGE_SIZE);
1956 * pmap_zero_page_area zeros the specified hardware page by
1957 * mapping it into virtual memory and using bzero to clear
1960 * off and size must reside within a single page.
1964 pmap_zero_page_area(vm_page_t m, int off, int size)
1968 p = (void *)pmap_page_to_va(m);
1969 bzero(p + off, size);
1974 * pmap_zero_page_idle zeros the specified hardware page by
1975 * mapping it into virtual memory and using bzero to clear
1976 * its contents. This is for the vm_idlezero process.
1980 pmap_zero_page_idle(vm_page_t m)
1984 p = (void *)pmap_page_to_va(m);
1985 bzero(p, PAGE_SIZE);
1990 * pmap_copy_page copies the specified (machine independent)
1991 * page by mapping the page into virtual memory and using
1992 * bcopy to copy the page, one machine dependent page at a
1996 pmap_copy_page(vm_page_t msrc, vm_page_t mdst)
2000 src = (void *)pmap_page_to_va(msrc);
2001 dst = (void *)pmap_page_to_va(mdst);
2002 bcopy(src, dst, PAGE_SIZE);
2005 int unmapped_buf_allowed;
2008 pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[],
2009 vm_offset_t b_offset, int xfersize)
2012 vm_offset_t a_pg_offset, b_pg_offset;
2015 while (xfersize > 0) {
2016 a_pg_offset = a_offset & PAGE_MASK;
2017 cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
2018 a_cp = (char *)pmap_page_to_va(ma[a_offset >> PAGE_SHIFT]) +
2020 b_pg_offset = b_offset & PAGE_MASK;
2021 cnt = min(cnt, PAGE_SIZE - b_pg_offset);
2022 b_cp = (char *)pmap_page_to_va(mb[b_offset >> PAGE_SHIFT]) +
2024 bcopy(a_cp, b_cp, cnt);
2032 * Returns true if the pmap's pv is one of the first
2033 * 16 pvs linked to from this page. This count may
2034 * be changed upwards or downwards in the future; it
2035 * is only necessary that true be returned for a small
2036 * subset of pmaps for proper page aging.
2039 pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
2045 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2046 ("pmap_page_exists_quick: page %p is not managed", m));
2048 rw_wlock(&pvh_global_lock);
2049 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2050 if (PV_PMAP(pv) == pmap) {
2058 rw_wunlock(&pvh_global_lock);
2063 * pmap_page_wired_mappings:
2065 * Return the number of managed mappings to the given physical page
2069 pmap_page_wired_mappings(vm_page_t m)
2071 struct ia64_lpte *pte;
2072 pmap_t oldpmap, pmap;
2077 if ((m->oflags & VPO_UNMANAGED) != 0)
2079 rw_wlock(&pvh_global_lock);
2080 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2083 oldpmap = pmap_switch(pmap);
2084 pte = pmap_find_vhpt(pv->pv_va);
2085 KASSERT(pte != NULL, ("pte"));
2086 if (pmap_wired(pte))
2088 pmap_switch(oldpmap);
2091 rw_wunlock(&pvh_global_lock);
2096 * Remove all pages from specified address space
2097 * this aids process exit speeds. Also, this code
2098 * is special cased for current process only, but
2099 * can have the more generic (and slightly slower)
2100 * mode enabled. This is much faster than pmap_remove
2101 * in the case of running down an entire address space.
2104 pmap_remove_pages(pmap_t pmap)
2106 struct pv_chunk *pc, *npc;
2107 struct ia64_lpte *pte;
2112 u_long inuse, bitmask;
2113 int allfree, bit, field, idx;
2115 rw_wlock(&pvh_global_lock);
2117 oldpmap = pmap_switch(pmap);
2118 TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {
2120 for (field = 0; field < _NPCM; field++) {
2121 inuse = ~pc->pc_map[field] & pc_freemask[field];
2122 while (inuse != 0) {
2123 bit = ffsl(inuse) - 1;
2124 bitmask = 1UL << bit;
2125 idx = field * sizeof(inuse) * NBBY + bit;
2126 pv = &pc->pc_pventry[idx];
2129 pte = pmap_find_vhpt(va);
2130 KASSERT(pte != NULL, ("pte"));
2131 if (pmap_wired(pte)) {
2135 pmap_remove_vhpt(va);
2136 pmap_invalidate_page(va);
2137 m = PHYS_TO_VM_PAGE(pmap_ppn(pte));
2138 if (pmap_dirty(pte))
2140 pmap_free_pte(pte, va);
2142 PV_STAT(pv_entry_frees++);
2143 PV_STAT(pv_entry_spare++);
2145 pc->pc_map[field] |= bitmask;
2146 pmap->pm_stats.resident_count--;
2147 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
2148 if (TAILQ_EMPTY(&m->md.pv_list))
2149 vm_page_aflag_clear(m, PGA_WRITEABLE);
2153 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
2157 pmap_switch(oldpmap);
2159 rw_wunlock(&pvh_global_lock);
2163 * pmap_ts_referenced:
2165 * Return a count of reference bits for a page, clearing those bits.
2166 * It is not necessary for every reference bit to be cleared, but it
2167 * is necessary that 0 only be returned when there are truly no
2168 * reference bits set.
2170 * XXX: The exact number of bits to check and clear is a matter that
2171 * should be tested and standardized at some point in the future for
2172 * optimal aging of shared pages.
2175 pmap_ts_referenced(vm_page_t m)
2177 struct ia64_lpte *pte;
2178 pmap_t oldpmap, pmap;
2182 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2183 ("pmap_ts_referenced: page %p is not managed", m));
2184 rw_wlock(&pvh_global_lock);
2185 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2188 oldpmap = pmap_switch(pmap);
2189 pte = pmap_find_vhpt(pv->pv_va);
2190 KASSERT(pte != NULL, ("pte"));
2191 if (pmap_accessed(pte)) {
2193 pmap_clear_accessed(pte);
2194 pmap_invalidate_page(pv->pv_va);
2196 pmap_switch(oldpmap);
2199 rw_wunlock(&pvh_global_lock);
2206 * Return whether or not the specified physical page was modified
2207 * in any physical maps.
2210 pmap_is_modified(vm_page_t m)
2212 struct ia64_lpte *pte;
2213 pmap_t oldpmap, pmap;
2217 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2218 ("pmap_is_modified: page %p is not managed", m));
2222 * If the page is not exclusive busied, then PGA_WRITEABLE cannot be
2223 * concurrently set while the object is locked. Thus, if PGA_WRITEABLE
2224 * is clear, no PTEs can be dirty.
2226 VM_OBJECT_ASSERT_WLOCKED(m->object);
2227 if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0)
2229 rw_wlock(&pvh_global_lock);
2230 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2233 oldpmap = pmap_switch(pmap);
2234 pte = pmap_find_vhpt(pv->pv_va);
2235 pmap_switch(oldpmap);
2236 KASSERT(pte != NULL, ("pte"));
2237 rv = pmap_dirty(pte) ? TRUE : FALSE;
2242 rw_wunlock(&pvh_global_lock);
2247 * pmap_is_prefaultable:
2249 * Return whether or not the specified virtual address is elgible
2253 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)
2255 struct ia64_lpte *pte;
2257 pte = pmap_find_vhpt(addr);
2258 if (pte != NULL && pmap_present(pte))
2264 * pmap_is_referenced:
2266 * Return whether or not the specified physical page was referenced
2267 * in any physical maps.
2270 pmap_is_referenced(vm_page_t m)
2272 struct ia64_lpte *pte;
2273 pmap_t oldpmap, pmap;
2277 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2278 ("pmap_is_referenced: page %p is not managed", m));
2280 rw_wlock(&pvh_global_lock);
2281 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2284 oldpmap = pmap_switch(pmap);
2285 pte = pmap_find_vhpt(pv->pv_va);
2286 pmap_switch(oldpmap);
2287 KASSERT(pte != NULL, ("pte"));
2288 rv = pmap_accessed(pte) ? TRUE : FALSE;
2293 rw_wunlock(&pvh_global_lock);
2298 * Apply the given advice to the specified range of addresses within the
2299 * given pmap. Depending on the advice, clear the referenced and/or
2300 * modified flags in each mapping and set the mapped page's dirty field.
2303 pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
2305 struct ia64_lpte *pte;
2310 oldpmap = pmap_switch(pmap);
2311 for (; sva < eva; sva += PAGE_SIZE) {
2312 /* If page is invalid, skip this page. */
2313 pte = pmap_find_vhpt(sva);
2317 /* If it isn't managed, skip it too. */
2318 if (!pmap_managed(pte))
2321 /* Clear its modified and referenced bits. */
2322 if (pmap_dirty(pte)) {
2323 if (advice == MADV_DONTNEED) {
2325 * Future calls to pmap_is_modified() can be
2326 * avoided by making the page dirty now.
2328 m = PHYS_TO_VM_PAGE(pmap_ppn(pte));
2331 pmap_clear_dirty(pte);
2332 } else if (!pmap_accessed(pte))
2334 pmap_clear_accessed(pte);
2335 pmap_invalidate_page(sva);
2337 pmap_switch(oldpmap);
2342 * Clear the modify bits on the specified physical page.
2345 pmap_clear_modify(vm_page_t m)
2347 struct ia64_lpte *pte;
2348 pmap_t oldpmap, pmap;
2351 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2352 ("pmap_clear_modify: page %p is not managed", m));
2353 VM_OBJECT_ASSERT_WLOCKED(m->object);
2354 KASSERT(!vm_page_xbusied(m),
2355 ("pmap_clear_modify: page %p is exclusive busied", m));
2358 * If the page is not PGA_WRITEABLE, then no PTEs can be modified.
2359 * If the object containing the page is locked and the page is not
2360 * exclusive busied, then PGA_WRITEABLE cannot be concurrently set.
2362 if ((m->aflags & PGA_WRITEABLE) == 0)
2364 rw_wlock(&pvh_global_lock);
2365 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2368 oldpmap = pmap_switch(pmap);
2369 pte = pmap_find_vhpt(pv->pv_va);
2370 KASSERT(pte != NULL, ("pte"));
2371 if (pmap_dirty(pte)) {
2372 pmap_clear_dirty(pte);
2373 pmap_invalidate_page(pv->pv_va);
2375 pmap_switch(oldpmap);
2378 rw_wunlock(&pvh_global_lock);
2382 * Clear the write and modified bits in each of the given page's mappings.
2385 pmap_remove_write(vm_page_t m)
2387 struct ia64_lpte *pte;
2388 pmap_t oldpmap, pmap;
2392 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2393 ("pmap_remove_write: page %p is not managed", m));
2396 * If the page is not exclusive busied, then PGA_WRITEABLE cannot be
2397 * set by another thread while the object is locked. Thus,
2398 * if PGA_WRITEABLE is clear, no page table entries need updating.
2400 VM_OBJECT_ASSERT_WLOCKED(m->object);
2401 if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0)
2403 rw_wlock(&pvh_global_lock);
2404 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2407 oldpmap = pmap_switch(pmap);
2408 pte = pmap_find_vhpt(pv->pv_va);
2409 KASSERT(pte != NULL, ("pte"));
2410 prot = pmap_prot(pte);
2411 if ((prot & VM_PROT_WRITE) != 0) {
2412 if (pmap_dirty(pte)) {
2414 pmap_clear_dirty(pte);
2416 prot &= ~VM_PROT_WRITE;
2417 pmap_pte_prot(pmap, pte, prot);
2418 pmap_pte_attr(pte, m->md.memattr);
2419 pmap_invalidate_page(pv->pv_va);
2421 pmap_switch(oldpmap);
2424 vm_page_aflag_clear(m, PGA_WRITEABLE);
2425 rw_wunlock(&pvh_global_lock);
2429 * Map a set of physical memory pages into the kernel virtual
2430 * address space. Return a pointer to where it is mapped. This
2431 * routine is intended to be used for mapping device memory,
2435 pmap_mapdev(vm_paddr_t pa, vm_size_t sz)
2437 static void *last_va = NULL;
2438 static vm_paddr_t last_pa = 0;
2439 static vm_size_t last_sz = 0;
2443 if (pa == last_pa && sz == last_sz)
2446 md = efi_md_find(pa);
2448 printf("%s: [%#lx..%#lx] not covered by memory descriptor\n",
2449 __func__, pa, pa + sz - 1);
2450 return ((void *)IA64_PHYS_TO_RR6(pa));
2453 if (md->md_type == EFI_MD_TYPE_FREE) {
2454 printf("%s: [%#lx..%#lx] is in DRAM\n", __func__, pa,
2459 va = (md->md_attr & EFI_MD_ATTR_WB) ? IA64_PHYS_TO_RR7(pa) :
2460 IA64_PHYS_TO_RR6(pa);
2462 last_va = (void *)va;
2469 * 'Unmap' a range mapped by pmap_mapdev().
2472 pmap_unmapdev(vm_offset_t va, vm_size_t size)
2477 * Sets the memory attribute for the specified page.
2480 pmap_page_set_memattr_1(void *arg)
2482 struct ia64_pal_result res;
2484 uintptr_t pp = (uintptr_t)arg;
2486 is = intr_disable();
2487 res = ia64_call_pal_static(pp, 0, 0, 0);
2492 pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma)
2494 struct ia64_lpte *pte;
2495 pmap_t oldpmap, pmap;
2499 rw_wlock(&pvh_global_lock);
2501 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2504 oldpmap = pmap_switch(pmap);
2505 pte = pmap_find_vhpt(pv->pv_va);
2506 KASSERT(pte != NULL, ("pte"));
2507 pmap_pte_attr(pte, ma);
2508 pmap_invalidate_page(pv->pv_va);
2509 pmap_switch(oldpmap);
2512 rw_wunlock(&pvh_global_lock);
2514 if (ma == VM_MEMATTR_UNCACHEABLE) {
2516 smp_rendezvous(NULL, pmap_page_set_memattr_1, NULL,
2517 (void *)PAL_PREFETCH_VISIBILITY);
2519 pmap_page_set_memattr_1((void *)PAL_PREFETCH_VISIBILITY);
2521 va = (void *)pmap_page_to_va(m);
2523 cpu_flush_dcache(va, PAGE_SIZE);
2526 smp_rendezvous(NULL, pmap_page_set_memattr_1, NULL,
2527 (void *)PAL_MC_DRAIN);
2529 pmap_page_set_memattr_1((void *)PAL_MC_DRAIN);
2535 * perform the pmap work for mincore
2538 pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa)
2541 struct ia64_lpte *pte, tpte;
2547 oldpmap = pmap_switch(pmap);
2548 pte = pmap_find_vhpt(addr);
2553 pmap_switch(oldpmap);
2554 if (pte == NULL || !pmap_present(pte)) {
2558 val = MINCORE_INCORE;
2559 if (pmap_dirty(pte))
2560 val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER;
2561 if (pmap_accessed(pte))
2562 val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER;
2563 if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) !=
2564 (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) &&
2565 pmap_managed(pte)) {
2567 /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */
2568 if (vm_page_pa_tryrelock(pmap, pa, locked_pa))
2572 PA_UNLOCK_COND(*locked_pa);
2578 pmap_activate(struct thread *td)
2580 pmap_switch(vmspace_pmap(td->td_proc->p_vmspace));
2584 pmap_switch(pmap_t pm)
2590 prevpm = PCPU_GET(md.current_pmap);
2594 for (i = 0; i < IA64_VM_MINKERN_REGION; i++) {
2595 ia64_set_rr(IA64_RR_BASE(i),
2596 (i << 8)|(PAGE_SHIFT << 2)|1);
2599 for (i = 0; i < IA64_VM_MINKERN_REGION; i++) {
2600 ia64_set_rr(IA64_RR_BASE(i),
2601 (pm->pm_rid[i] << 8)|(PAGE_SHIFT << 2)|1);
2604 PCPU_SET(md.current_pmap, pm);
2613 pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz)
2616 struct ia64_lpte *pte;
2622 sz = (sz + 31) & ~31;
2625 oldpm = pmap_switch(pm);
2627 lim = round_page(va);
2628 len = MIN(lim - va, sz);
2629 pte = pmap_find_vhpt(va);
2630 if (pte != NULL && pmap_present(pte))
2631 ia64_sync_icache(va, len);
2640 * Increase the starting virtual address of the given mapping if a
2641 * different alignment might result in more superpage mappings.
2644 pmap_align_superpage(vm_object_t object, vm_ooffset_t offset,
2645 vm_offset_t *addr, vm_size_t size)
2649 #include "opt_ddb.h"
2653 #include <ddb/ddb.h>
2655 static const char* psnames[] = {
2656 "1B", "2B", "4B", "8B",
2657 "16B", "32B", "64B", "128B",
2658 "256B", "512B", "1K", "2K",
2659 "4K", "8K", "16K", "32K",
2660 "64K", "128K", "256K", "512K",
2661 "1M", "2M", "4M", "8M",
2662 "16M", "32M", "64M", "128M",
2663 "256M", "512M", "1G", "2G"
2669 struct ia64_pal_result res;
2677 static const char *manames[] = {
2678 "WB", "bad", "bad", "bad",
2679 "UC", "UCE", "WC", "NaT",
2682 res = ia64_call_pal_static(PAL_VM_SUMMARY, 0, 0, 0);
2683 if (res.pal_status != 0) {
2684 db_printf("Can't get VM summary\n");
2689 maxtr = (res.pal_result[0] >> 40) & 0xff;
2691 maxtr = (res.pal_result[0] >> 32) & 0xff;
2693 db_printf("V RID Virtual Page Physical Page PgSz ED AR PL D A MA P KEY\n");
2694 for (i = 0; i <= maxtr; i++) {
2695 bzero(&buf, sizeof(buf));
2696 res = ia64_pal_physical(PAL_VM_TR_READ, i, type,
2697 ia64_tpa((uint64_t)&buf));
2698 if (!(res.pal_result[0] & 1))
2699 buf.pte &= ~PTE_AR_MASK;
2700 if (!(res.pal_result[0] & 2))
2701 buf.pte &= ~PTE_PL_MASK;
2702 if (!(res.pal_result[0] & 4))
2703 pmap_clear_dirty(&buf);
2704 if (!(res.pal_result[0] & 8))
2705 buf.pte &= ~PTE_MA_MASK;
2706 db_printf("%d %06x %013lx %013lx %4s %d %d %d %d %d %-3s "
2707 "%d %06x\n", (int)buf.ifa & 1, buf.rr.rr_rid,
2708 buf.ifa >> 12, (buf.pte & PTE_PPN_MASK) >> 12,
2709 psnames[(buf.itir & ITIR_PS_MASK) >> 2],
2710 (buf.pte & PTE_ED) ? 1 : 0,
2711 (int)(buf.pte & PTE_AR_MASK) >> 9,
2712 (int)(buf.pte & PTE_PL_MASK) >> 7,
2713 (pmap_dirty(&buf)) ? 1 : 0,
2714 (pmap_accessed(&buf)) ? 1 : 0,
2715 manames[(buf.pte & PTE_MA_MASK) >> 2],
2716 (pmap_present(&buf)) ? 1 : 0,
2717 (int)((buf.itir & ITIR_KEY_MASK) >> 8));
2721 DB_COMMAND(itr, db_itr)
2726 DB_COMMAND(dtr, db_dtr)
2731 DB_COMMAND(rr, db_rr)
2737 printf("RR RID PgSz VE\n");
2738 for (i = 0; i < 8; i++) {
2739 __asm __volatile ("mov %0=rr[%1]"
2741 : "r"(IA64_RR_BASE(i)));
2742 *(uint64_t *) &rr = t;
2743 printf("%d %06x %4s %d\n",
2744 i, rr.rr_rid, psnames[rr.rr_ps], rr.rr_ve);
2748 DB_COMMAND(thash, db_thash)
2753 db_printf("%p\n", (void *) ia64_thash(addr));
2756 DB_COMMAND(ttag, db_ttag)
2761 db_printf("0x%lx\n", ia64_ttag(addr));
2764 DB_COMMAND(kpte, db_kpte)
2766 struct ia64_lpte *pte;
2769 db_printf("usage: kpte <kva>\n");
2772 if (addr < VM_INIT_KERNEL_ADDRESS) {
2773 db_printf("kpte: error: invalid <kva>\n");
2776 pte = pmap_find_kpte(addr);
2777 db_printf("kpte at %p:\n", pte);
2778 db_printf(" pte =%016lx\n", pte->pte);
2779 db_printf(" itir =%016lx\n", pte->itir);
2780 db_printf(" tag =%016lx\n", pte->tag);
2781 db_printf(" chain=%016lx\n", pte->chain);