2 * Copyright (c) 1991 Regents of the University of California.
4 * Copyright (c) 1994 John S. Dyson
6 * Copyright (c) 1994 David Greenman
8 * Copyright (c) 1998,2000 Doug Rabson
11 * This code is derived from software contributed to Berkeley by
12 * the Systems Programming Group of the University of Utah Computer
13 * Science Department and William Jolitz of UUNET Technologies Inc.
15 * Redistribution and use in source and binary forms, with or without
16 * modification, are permitted provided that the following conditions
18 * 1. Redistributions of source code must retain the above copyright
19 * notice, this list of conditions and the following disclaimer.
20 * 2. Redistributions in binary form must reproduce the above copyright
21 * notice, this list of conditions and the following disclaimer in the
22 * documentation and/or other materials provided with the distribution.
23 * 3. All advertising materials mentioning features or use of this software
24 * must display the following acknowledgement:
25 * This product includes software developed by the University of
26 * California, Berkeley and its contributors.
27 * 4. Neither the name of the University nor the names of its contributors
28 * may be used to endorse or promote products derived from this software
29 * without specific prior written permission.
31 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
32 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
33 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
34 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
35 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
36 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
37 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
38 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
39 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
40 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
43 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91
44 * from: i386 Id: pmap.c,v 1.193 1998/04/19 15:22:48 bde Exp
45 * with some ideas from NetBSD's alpha pmap
48 #include <sys/cdefs.h>
49 __FBSDID("$FreeBSD$");
53 #include <sys/param.h>
54 #include <sys/kernel.h>
57 #include <sys/mutex.h>
59 #include <sys/rwlock.h>
61 #include <sys/sysctl.h>
62 #include <sys/systm.h>
65 #include <vm/vm_param.h>
66 #include <vm/vm_page.h>
67 #include <vm/vm_map.h>
68 #include <vm/vm_object.h>
69 #include <vm/vm_pageout.h>
72 #include <machine/bootinfo.h>
73 #include <machine/efi.h>
74 #include <machine/md_var.h>
75 #include <machine/pal.h>
78 * Manages physical address maps.
80 * Since the information managed by this module is
81 * also stored by the logical address mapping module,
82 * this module may throw away valid virtual-to-physical
83 * mappings at almost any time. However, invalidations
84 * of virtual-to-physical mappings must be done as
87 * In order to cope with hardware architectures which
88 * make virtual-to-physical map invalidates expensive,
89 * this module may delay invalidate or reduced protection
90 * operations until such time as they are actually
91 * necessary. This module is given full information as
92 * to which processors are currently using which maps,
93 * and to when physical maps must be made correct.
97 * Following the Linux model, region IDs are allocated in groups of
98 * eight so that a single region ID can be used for as many RRs as we
99 * want by encoding the RR number into the low bits of the ID.
101 * We reserve region ID 0 for the kernel and allocate the remaining
102 * IDs for user pmaps.
104 * Region 0-3: User virtually mapped
105 * Region 4: PBVM and special mappings
106 * Region 5: Kernel virtual memory
107 * Region 6: Direct-mapped uncacheable
108 * Region 7: Direct-mapped cacheable
111 /* XXX move to a header. */
112 extern uint64_t ia64_gateway_page[];
114 #if !defined(DIAGNOSTIC)
115 #define PMAP_INLINE __inline
121 #define PV_STAT(x) do { x ; } while (0)
123 #define PV_STAT(x) do { } while (0)
126 #define pmap_accessed(lpte) ((lpte)->pte & PTE_ACCESSED)
127 #define pmap_dirty(lpte) ((lpte)->pte & PTE_DIRTY)
128 #define pmap_exec(lpte) ((lpte)->pte & PTE_AR_RX)
129 #define pmap_managed(lpte) ((lpte)->pte & PTE_MANAGED)
130 #define pmap_ppn(lpte) ((lpte)->pte & PTE_PPN_MASK)
131 #define pmap_present(lpte) ((lpte)->pte & PTE_PRESENT)
132 #define pmap_prot(lpte) (((lpte)->pte & PTE_PROT_MASK) >> 56)
133 #define pmap_wired(lpte) ((lpte)->pte & PTE_WIRED)
135 #define pmap_clear_accessed(lpte) (lpte)->pte &= ~PTE_ACCESSED
136 #define pmap_clear_dirty(lpte) (lpte)->pte &= ~PTE_DIRTY
137 #define pmap_clear_present(lpte) (lpte)->pte &= ~PTE_PRESENT
138 #define pmap_clear_wired(lpte) (lpte)->pte &= ~PTE_WIRED
140 #define pmap_set_wired(lpte) (lpte)->pte |= PTE_WIRED
143 * Individual PV entries are stored in per-pmap chunks. This saves
144 * space by eliminating the need to record the pmap within every PV
147 #if PAGE_SIZE == 8192
151 #elif PAGE_SIZE == 16384
158 TAILQ_ENTRY(pv_chunk) pc_list;
159 u_long pc_map[_NPCM]; /* bitmap; 1 = free */
160 TAILQ_ENTRY(pv_chunk) pc_lru;
161 u_long pc_spare[_NPCS];
162 struct pv_entry pc_pventry[_NPCPV];
166 * The VHPT bucket head structure.
175 * Statically allocated kernel pmap
177 struct pmap kernel_pmap_store;
179 vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */
180 vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */
183 * Kernel virtual memory management.
186 extern struct ia64_lpte ***ia64_kptdir;
188 #define KPTE_DIR0_INDEX(va) \
189 (((va) >> (3*PAGE_SHIFT-8)) & ((1<<(PAGE_SHIFT-3))-1))
190 #define KPTE_DIR1_INDEX(va) \
191 (((va) >> (2*PAGE_SHIFT-5)) & ((1<<(PAGE_SHIFT-3))-1))
192 #define KPTE_PTE_INDEX(va) \
193 (((va) >> PAGE_SHIFT) & ((1<<(PAGE_SHIFT-5))-1))
194 #define NKPTEPG (PAGE_SIZE / sizeof(struct ia64_lpte))
196 vm_offset_t kernel_vm_end;
198 /* Values for ptc.e. XXX values for SKI. */
199 static uint64_t pmap_ptc_e_base = 0x100000000;
200 static uint64_t pmap_ptc_e_count1 = 3;
201 static uint64_t pmap_ptc_e_count2 = 2;
202 static uint64_t pmap_ptc_e_stride1 = 0x2000;
203 static uint64_t pmap_ptc_e_stride2 = 0x100000000;
205 struct mtx pmap_ptc_mutex;
208 * Data for the RID allocator
210 static int pmap_ridcount;
211 static int pmap_rididx;
212 static int pmap_ridmapsz;
213 static int pmap_ridmax;
214 static uint64_t *pmap_ridmap;
215 struct mtx pmap_ridmutex;
217 static struct rwlock_padalign pvh_global_lock;
220 * Data for the pv entry allocation mechanism
222 static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks);
223 static int pv_entry_count;
226 * Data for allocating PTEs for user processes.
228 static uma_zone_t ptezone;
231 * Virtual Hash Page Table (VHPT) data.
233 /* SYSCTL_DECL(_machdep); */
234 static SYSCTL_NODE(_machdep, OID_AUTO, vhpt, CTLFLAG_RD, 0, "");
236 struct ia64_bucket *pmap_vhpt_bucket;
238 int pmap_vhpt_nbuckets;
239 SYSCTL_INT(_machdep_vhpt, OID_AUTO, nbuckets, CTLFLAG_RD,
240 &pmap_vhpt_nbuckets, 0, "");
242 int pmap_vhpt_log2size = 0;
243 TUNABLE_INT("machdep.vhpt.log2size", &pmap_vhpt_log2size);
244 SYSCTL_INT(_machdep_vhpt, OID_AUTO, log2size, CTLFLAG_RD,
245 &pmap_vhpt_log2size, 0, "");
247 static int pmap_vhpt_inserts;
248 SYSCTL_INT(_machdep_vhpt, OID_AUTO, inserts, CTLFLAG_RD,
249 &pmap_vhpt_inserts, 0, "");
251 static int pmap_vhpt_population(SYSCTL_HANDLER_ARGS);
252 SYSCTL_PROC(_machdep_vhpt, OID_AUTO, population, CTLTYPE_INT | CTLFLAG_RD,
253 NULL, 0, pmap_vhpt_population, "I", "");
255 static struct ia64_lpte *pmap_find_vhpt(vm_offset_t va);
257 static void free_pv_chunk(struct pv_chunk *pc);
258 static void free_pv_entry(pmap_t pmap, pv_entry_t pv);
259 static pv_entry_t get_pv_entry(pmap_t pmap, boolean_t try);
260 static vm_page_t pmap_pv_reclaim(pmap_t locked_pmap);
262 static void pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
263 vm_page_t m, vm_prot_t prot);
264 static void pmap_free_pte(struct ia64_lpte *pte, vm_offset_t va);
265 static void pmap_invalidate_all(void);
266 static int pmap_remove_pte(pmap_t pmap, struct ia64_lpte *pte,
267 vm_offset_t va, pv_entry_t pv, int freepte);
268 static int pmap_remove_vhpt(vm_offset_t va);
269 static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,
273 pmap_initialize_vhpt(vm_offset_t vhpt)
275 struct ia64_lpte *pte;
278 pte = (struct ia64_lpte *)vhpt;
279 for (i = 0; i < pmap_vhpt_nbuckets; i++) {
282 pte[i].tag = 1UL << 63; /* Invalid tag */
283 pte[i].chain = (uintptr_t)(pmap_vhpt_bucket + i);
289 pmap_alloc_vhpt(void)
295 size = 1UL << pmap_vhpt_log2size;
296 m = vm_page_alloc_contig(NULL, 0, VM_ALLOC_SYSTEM | VM_ALLOC_NOOBJ |
297 VM_ALLOC_WIRED, atop(size), 0UL, ~0UL, size, 0UL,
300 vhpt = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m));
301 pmap_initialize_vhpt(vhpt);
309 * Bootstrap the system enough to run with virtual memory.
314 struct ia64_pal_result res;
320 * Query the PAL Code to find the loop parameters for the
323 res = ia64_call_pal_static(PAL_PTCE_INFO, 0, 0, 0);
324 if (res.pal_status != 0)
325 panic("Can't configure ptc.e parameters");
326 pmap_ptc_e_base = res.pal_result[0];
327 pmap_ptc_e_count1 = res.pal_result[1] >> 32;
328 pmap_ptc_e_count2 = res.pal_result[1] & ((1L<<32) - 1);
329 pmap_ptc_e_stride1 = res.pal_result[2] >> 32;
330 pmap_ptc_e_stride2 = res.pal_result[2] & ((1L<<32) - 1);
332 printf("ptc.e base=0x%lx, count1=%ld, count2=%ld, "
333 "stride1=0x%lx, stride2=0x%lx\n",
340 mtx_init(&pmap_ptc_mutex, "PTC.G mutex", NULL, MTX_SPIN);
343 * Setup RIDs. RIDs 0..7 are reserved for the kernel.
345 * We currently need at least 19 bits in the RID because PID_MAX
346 * can only be encoded in 17 bits and we need RIDs for 4 regions
347 * per process. With PID_MAX equalling 99999 this means that we
348 * need to be able to encode 399996 (=4*PID_MAX).
349 * The Itanium processor only has 18 bits and the architected
350 * minimum is exactly that. So, we cannot use a PID based scheme
351 * in those cases. Enter pmap_ridmap...
352 * We should avoid the map when running on a processor that has
353 * implemented enough bits. This means that we should pass the
354 * process/thread ID to pmap. This we currently don't do, so we
355 * use the map anyway. However, we don't want to allocate a map
356 * that is large enough to cover the range dictated by the number
357 * of bits in the RID, because that may result in a RID map of
358 * 2MB in size for a 24-bit RID. A 64KB map is enough.
359 * The bottomline: we create a 32KB map when the processor only
360 * implements 18 bits (or when we can't figure it out). Otherwise
361 * we create a 64KB map.
363 res = ia64_call_pal_static(PAL_VM_SUMMARY, 0, 0, 0);
364 if (res.pal_status != 0) {
366 printf("Can't read VM Summary - assuming 18 Region ID bits\n");
367 ridbits = 18; /* guaranteed minimum */
369 ridbits = (res.pal_result[1] >> 8) & 0xff;
371 printf("Processor supports %d Region ID bits\n",
377 pmap_ridmax = (1 << ridbits);
378 pmap_ridmapsz = pmap_ridmax / 64;
379 pmap_ridmap = ia64_physmem_alloc(pmap_ridmax / 8, PAGE_SIZE);
380 pmap_ridmap[0] |= 0xff;
383 mtx_init(&pmap_ridmutex, "RID allocator lock", NULL, MTX_DEF);
386 * Allocate some memory for initial kernel 'page tables'.
388 ia64_kptdir = ia64_physmem_alloc(PAGE_SIZE, PAGE_SIZE);
390 kernel_vm_end = VM_INIT_KERNEL_ADDRESS;
393 * Determine a valid (mappable) VHPT size.
395 TUNABLE_INT_FETCH("machdep.vhpt.log2size", &pmap_vhpt_log2size);
396 if (pmap_vhpt_log2size == 0)
397 pmap_vhpt_log2size = 20;
398 else if (pmap_vhpt_log2size < 16)
399 pmap_vhpt_log2size = 16;
400 else if (pmap_vhpt_log2size > 28)
401 pmap_vhpt_log2size = 28;
402 if (pmap_vhpt_log2size & 1)
403 pmap_vhpt_log2size--;
405 size = 1UL << pmap_vhpt_log2size;
406 base = (uintptr_t)ia64_physmem_alloc(size, size);
408 panic("Unable to allocate VHPT");
410 PCPU_SET(md.vhpt, base);
412 printf("VHPT: address=%#lx, size=%#lx\n", base, size);
414 pmap_vhpt_nbuckets = size / sizeof(struct ia64_lpte);
415 pmap_vhpt_bucket = ia64_physmem_alloc(pmap_vhpt_nbuckets *
416 sizeof(struct ia64_bucket), PAGE_SIZE);
417 for (i = 0; i < pmap_vhpt_nbuckets; i++) {
418 /* Stolen memory is zeroed. */
419 mtx_init(&pmap_vhpt_bucket[i].mutex, "VHPT bucket lock", NULL,
420 MTX_NOWITNESS | MTX_SPIN);
423 pmap_initialize_vhpt(base);
425 ia64_set_pta(base + (1 << 8) + (pmap_vhpt_log2size << 2) + 1);
428 virtual_avail = VM_INIT_KERNEL_ADDRESS;
429 virtual_end = VM_MAX_KERNEL_ADDRESS;
432 * Initialize the kernel pmap (which is statically allocated).
434 PMAP_LOCK_INIT(kernel_pmap);
435 for (i = 0; i < IA64_VM_MINKERN_REGION; i++)
436 kernel_pmap->pm_rid[i] = 0;
437 TAILQ_INIT(&kernel_pmap->pm_pvchunk);
438 PCPU_SET(md.current_pmap, kernel_pmap);
441 * Initialize the global pv list lock.
443 rw_init(&pvh_global_lock, "pmap pv global");
445 /* Region 5 is mapped via the VHPT. */
446 ia64_set_rr(IA64_RR_BASE(5), (5 << 8) | (PAGE_SHIFT << 2) | 1);
449 * Clear out any random TLB entries left over from booting.
451 pmap_invalidate_all();
457 pmap_vhpt_population(SYSCTL_HANDLER_ARGS)
462 for (i = 0; i < pmap_vhpt_nbuckets; i++)
463 count += pmap_vhpt_bucket[i].length;
465 error = SYSCTL_OUT(req, &count, sizeof(count));
470 pmap_page_to_va(vm_page_t m)
475 pa = VM_PAGE_TO_PHYS(m);
476 va = (m->md.memattr == VM_MEMATTR_UNCACHEABLE) ? IA64_PHYS_TO_RR6(pa) :
477 IA64_PHYS_TO_RR7(pa);
482 * Initialize a vm_page's machine-dependent fields.
485 pmap_page_init(vm_page_t m)
488 TAILQ_INIT(&m->md.pv_list);
489 m->md.memattr = VM_MEMATTR_DEFAULT;
493 * Initialize the pmap module.
494 * Called by vm_init, to initialize any structures that the pmap
495 * system needs to map virtual memory.
501 ptezone = uma_zcreate("PT ENTRY", sizeof (struct ia64_lpte),
502 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM|UMA_ZONE_NOFREE);
506 /***************************************************
507 * Manipulate TLBs for a pmap
508 ***************************************************/
511 pmap_invalidate_page(vm_offset_t va)
513 struct ia64_lpte *pte;
520 vhpt_ofs = ia64_thash(va) - PCPU_GET(md.vhpt);
522 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
523 pte = (struct ia64_lpte *)(pc->pc_md.vhpt + vhpt_ofs);
524 atomic_cmpset_64(&pte->tag, tag, 1UL << 63);
527 mtx_lock_spin(&pmap_ptc_mutex);
529 ia64_ptc_ga(va, PAGE_SHIFT << 2);
533 mtx_unlock_spin(&pmap_ptc_mutex);
541 pmap_invalidate_all_1(void *arg)
547 addr = pmap_ptc_e_base;
548 for (i = 0; i < pmap_ptc_e_count1; i++) {
549 for (j = 0; j < pmap_ptc_e_count2; j++) {
551 addr += pmap_ptc_e_stride2;
553 addr += pmap_ptc_e_stride1;
559 pmap_invalidate_all(void)
564 smp_rendezvous(NULL, pmap_invalidate_all_1, NULL, NULL);
568 pmap_invalidate_all_1(NULL);
572 pmap_allocate_rid(void)
577 mtx_lock(&pmap_ridmutex);
578 if (pmap_ridcount == pmap_ridmax)
579 panic("pmap_allocate_rid: All Region IDs used");
581 /* Find an index with a free bit. */
582 while ((bits = pmap_ridmap[pmap_rididx]) == ~0UL) {
584 if (pmap_rididx == pmap_ridmapsz)
587 rid = pmap_rididx * 64;
589 /* Find a free bit. */
596 pmap_ridmap[pmap_rididx] |= bit;
598 mtx_unlock(&pmap_ridmutex);
604 pmap_free_rid(uint32_t rid)
610 bit = ~(1UL << (rid & 63));
612 mtx_lock(&pmap_ridmutex);
613 pmap_ridmap[idx] &= bit;
615 mtx_unlock(&pmap_ridmutex);
618 /***************************************************
619 * Page table page management routines.....
620 ***************************************************/
623 pmap_pinit0(struct pmap *pmap)
626 PMAP_LOCK_INIT(pmap);
627 /* kernel_pmap is the same as any other pmap. */
632 * Initialize a preallocated and zeroed pmap structure,
633 * such as one in a vmspace structure.
636 pmap_pinit(struct pmap *pmap)
640 for (i = 0; i < IA64_VM_MINKERN_REGION; i++)
641 pmap->pm_rid[i] = pmap_allocate_rid();
642 TAILQ_INIT(&pmap->pm_pvchunk);
643 bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
647 /***************************************************
648 * Pmap allocation/deallocation routines.
649 ***************************************************/
652 * Release any resources held by the given physical map.
653 * Called when a pmap initialized by pmap_pinit is being released.
654 * Should only be called if the map contains no valid mappings.
657 pmap_release(pmap_t pmap)
661 for (i = 0; i < IA64_VM_MINKERN_REGION; i++)
663 pmap_free_rid(pmap->pm_rid[i]);
667 * grow the number of kernel page table entries, if needed
670 pmap_growkernel(vm_offset_t addr)
672 struct ia64_lpte **dir1;
673 struct ia64_lpte *leaf;
676 while (kernel_vm_end <= addr) {
677 if (nkpt == PAGE_SIZE/8 + PAGE_SIZE*PAGE_SIZE/64)
678 panic("%s: out of kernel address space", __func__);
680 dir1 = ia64_kptdir[KPTE_DIR0_INDEX(kernel_vm_end)];
682 nkpg = vm_page_alloc(NULL, nkpt++,
683 VM_ALLOC_NOOBJ|VM_ALLOC_INTERRUPT|VM_ALLOC_WIRED);
685 panic("%s: cannot add dir. page", __func__);
687 dir1 = (struct ia64_lpte **)pmap_page_to_va(nkpg);
688 bzero(dir1, PAGE_SIZE);
689 ia64_kptdir[KPTE_DIR0_INDEX(kernel_vm_end)] = dir1;
692 nkpg = vm_page_alloc(NULL, nkpt++,
693 VM_ALLOC_NOOBJ|VM_ALLOC_INTERRUPT|VM_ALLOC_WIRED);
695 panic("%s: cannot add PTE page", __func__);
697 leaf = (struct ia64_lpte *)pmap_page_to_va(nkpg);
698 bzero(leaf, PAGE_SIZE);
699 dir1[KPTE_DIR1_INDEX(kernel_vm_end)] = leaf;
701 kernel_vm_end += PAGE_SIZE * NKPTEPG;
705 /***************************************************
706 * page management routines.
707 ***************************************************/
709 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE);
711 static __inline struct pv_chunk *
712 pv_to_chunk(pv_entry_t pv)
715 return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK));
718 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
720 #define PC_FREE_FULL 0xfffffffffffffffful
721 #define PC_FREE_PARTIAL \
722 ((1UL << (_NPCPV - sizeof(u_long) * 8 * (_NPCM - 1))) - 1)
724 #if PAGE_SIZE == 8192
725 static const u_long pc_freemask[_NPCM] = {
726 PC_FREE_FULL, PC_FREE_FULL, PC_FREE_FULL,
727 PC_FREE_FULL, PC_FREE_FULL, PC_FREE_PARTIAL
729 #elif PAGE_SIZE == 16384
730 static const u_long pc_freemask[_NPCM] = {
731 PC_FREE_FULL, PC_FREE_FULL, PC_FREE_FULL,
732 PC_FREE_FULL, PC_FREE_FULL, PC_FREE_FULL,
733 PC_FREE_FULL, PC_FREE_FULL, PC_FREE_FULL,
734 PC_FREE_FULL, PC_FREE_PARTIAL
738 static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");
740 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0,
741 "Current number of pv entries");
744 static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail;
746 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0,
747 "Current number of pv entry chunks");
748 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0,
749 "Current number of pv entry chunks allocated");
750 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0,
751 "Current number of pv entry chunks frees");
752 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, &pc_chunk_tryfail, 0,
753 "Number of times tried to get a chunk page but failed.");
755 static long pv_entry_frees, pv_entry_allocs;
756 static int pv_entry_spare;
758 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, &pv_entry_frees, 0,
759 "Current number of pv entry frees");
760 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, &pv_entry_allocs, 0,
761 "Current number of pv entry allocs");
762 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0,
763 "Current number of spare pv entries");
767 * We are in a serious low memory condition. Resort to
768 * drastic measures to free some pages so we can allocate
769 * another pv entry chunk.
772 pmap_pv_reclaim(pmap_t locked_pmap)
776 struct ia64_lpte *pte;
782 int bit, field, freed, idx;
784 PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED);
787 TAILQ_INIT(&newtail);
788 while ((pc = TAILQ_FIRST(&pv_chunks)) != NULL) {
789 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
790 if (pmap != pc->pc_pmap) {
792 if (pmap != locked_pmap) {
793 pmap_switch(locked_pmap);
798 /* Avoid deadlock and lock recursion. */
799 if (pmap > locked_pmap)
801 else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap)) {
803 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
810 * Destroy every non-wired, 8 KB page mapping in the chunk.
813 for (field = 0; field < _NPCM; field++) {
814 for (inuse = ~pc->pc_map[field] & pc_freemask[field];
815 inuse != 0; inuse &= ~(1UL << bit)) {
816 bit = ffsl(inuse) - 1;
817 idx = field * sizeof(inuse) * NBBY + bit;
818 pv = &pc->pc_pventry[idx];
820 pte = pmap_find_vhpt(va);
821 KASSERT(pte != NULL, ("pte"));
824 pmap_remove_vhpt(va);
825 pmap_invalidate_page(va);
826 m = PHYS_TO_VM_PAGE(pmap_ppn(pte));
827 if (pmap_accessed(pte))
828 vm_page_aflag_set(m, PGA_REFERENCED);
831 pmap_free_pte(pte, va);
832 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
833 if (TAILQ_EMPTY(&m->md.pv_list))
834 vm_page_aflag_clear(m, PGA_WRITEABLE);
835 pc->pc_map[field] |= 1UL << bit;
840 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
843 /* Every freed mapping is for a 8 KB page. */
844 pmap->pm_stats.resident_count -= freed;
845 PV_STAT(pv_entry_frees += freed);
846 PV_STAT(pv_entry_spare += freed);
847 pv_entry_count -= freed;
848 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
849 for (field = 0; field < _NPCM; field++)
850 if (pc->pc_map[field] != pc_freemask[field]) {
851 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,
853 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
856 * One freed pv entry in locked_pmap is
859 if (pmap == locked_pmap)
863 if (field == _NPCM) {
864 PV_STAT(pv_entry_spare -= _NPCPV);
865 PV_STAT(pc_chunk_count--);
866 PV_STAT(pc_chunk_frees++);
867 /* Entire chunk is free; return it. */
868 m_pc = PHYS_TO_VM_PAGE(IA64_RR_MASK((vm_offset_t)pc));
873 TAILQ_CONCAT(&pv_chunks, &newtail, pc_lru);
875 if (pmap != locked_pmap) {
876 pmap_switch(locked_pmap);
884 * free the pv_entry back to the free list
887 free_pv_entry(pmap_t pmap, pv_entry_t pv)
892 rw_assert(&pvh_global_lock, RA_WLOCKED);
893 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
894 PV_STAT(pv_entry_frees++);
895 PV_STAT(pv_entry_spare++);
897 pc = pv_to_chunk(pv);
898 idx = pv - &pc->pc_pventry[0];
899 field = idx / (sizeof(u_long) * NBBY);
900 bit = idx % (sizeof(u_long) * NBBY);
901 pc->pc_map[field] |= 1ul << bit;
902 for (idx = 0; idx < _NPCM; idx++)
903 if (pc->pc_map[idx] != pc_freemask[idx]) {
905 * 98% of the time, pc is already at the head of the
906 * list. If it isn't already, move it to the head.
908 if (__predict_false(TAILQ_FIRST(&pmap->pm_pvchunk) !=
910 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
911 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,
916 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
921 free_pv_chunk(struct pv_chunk *pc)
925 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
926 PV_STAT(pv_entry_spare -= _NPCPV);
927 PV_STAT(pc_chunk_count--);
928 PV_STAT(pc_chunk_frees++);
929 /* entire chunk is free, return it */
930 m = PHYS_TO_VM_PAGE(IA64_RR_MASK((vm_offset_t)pc));
931 vm_page_unwire(m, 0);
936 * get a new pv_entry, allocating a block from the system
940 get_pv_entry(pmap_t pmap, boolean_t try)
947 rw_assert(&pvh_global_lock, RA_WLOCKED);
948 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
949 PV_STAT(pv_entry_allocs++);
952 pc = TAILQ_FIRST(&pmap->pm_pvchunk);
954 for (field = 0; field < _NPCM; field++) {
955 if (pc->pc_map[field]) {
956 bit = ffsl(pc->pc_map[field]) - 1;
961 idx = field * sizeof(pc->pc_map[field]) * NBBY + bit;
962 pv = &pc->pc_pventry[idx];
963 pc->pc_map[field] &= ~(1ul << bit);
964 /* If this was the last item, move it to tail */
965 for (field = 0; field < _NPCM; field++)
966 if (pc->pc_map[field] != 0) {
967 PV_STAT(pv_entry_spare--);
968 return (pv); /* not full, return */
970 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
971 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
972 PV_STAT(pv_entry_spare--);
976 /* No free items, allocate another chunk */
977 m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ |
982 PV_STAT(pc_chunk_tryfail++);
985 m = pmap_pv_reclaim(pmap);
989 PV_STAT(pc_chunk_count++);
990 PV_STAT(pc_chunk_allocs++);
991 pc = (struct pv_chunk *)IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m));
993 pc->pc_map[0] = pc_freemask[0] & ~1ul; /* preallocated bit 0 */
994 for (field = 1; field < _NPCM; field++)
995 pc->pc_map[field] = pc_freemask[field];
996 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
997 pv = &pc->pc_pventry[0];
998 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
999 PV_STAT(pv_entry_spare += _NPCPV - 1);
1004 * Conditionally create a pv entry.
1007 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
1011 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1012 rw_assert(&pvh_global_lock, RA_WLOCKED);
1013 if ((pv = get_pv_entry(pmap, TRUE)) != NULL) {
1015 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
1022 * Add an ia64_lpte to the VHPT.
1025 pmap_enter_vhpt(struct ia64_lpte *pte, vm_offset_t va)
1027 struct ia64_bucket *bckt;
1028 struct ia64_lpte *vhpte;
1031 /* Can fault, so get it out of the way. */
1032 pte_pa = ia64_tpa((vm_offset_t)pte);
1034 vhpte = (struct ia64_lpte *)ia64_thash(va);
1035 bckt = (struct ia64_bucket *)vhpte->chain;
1037 mtx_lock_spin(&bckt->mutex);
1038 pte->chain = bckt->chain;
1040 bckt->chain = pte_pa;
1042 pmap_vhpt_inserts++;
1044 mtx_unlock_spin(&bckt->mutex);
1048 * Remove the ia64_lpte matching va from the VHPT. Return zero if it
1049 * worked or an appropriate error code otherwise.
1052 pmap_remove_vhpt(vm_offset_t va)
1054 struct ia64_bucket *bckt;
1055 struct ia64_lpte *pte;
1056 struct ia64_lpte *lpte;
1057 struct ia64_lpte *vhpte;
1058 uint64_t chain, tag;
1060 tag = ia64_ttag(va);
1061 vhpte = (struct ia64_lpte *)ia64_thash(va);
1062 bckt = (struct ia64_bucket *)vhpte->chain;
1065 mtx_lock_spin(&bckt->mutex);
1066 chain = bckt->chain;
1067 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
1068 while (chain != 0 && pte->tag != tag) {
1071 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
1074 mtx_unlock_spin(&bckt->mutex);
1078 /* Snip this pv_entry out of the collision chain. */
1080 bckt->chain = pte->chain;
1082 lpte->chain = pte->chain;
1086 mtx_unlock_spin(&bckt->mutex);
1091 * Find the ia64_lpte for the given va, if any.
1093 static struct ia64_lpte *
1094 pmap_find_vhpt(vm_offset_t va)
1096 struct ia64_bucket *bckt;
1097 struct ia64_lpte *pte;
1098 uint64_t chain, tag;
1100 tag = ia64_ttag(va);
1101 pte = (struct ia64_lpte *)ia64_thash(va);
1102 bckt = (struct ia64_bucket *)pte->chain;
1104 mtx_lock_spin(&bckt->mutex);
1105 chain = bckt->chain;
1106 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
1107 while (chain != 0 && pte->tag != tag) {
1109 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
1111 mtx_unlock_spin(&bckt->mutex);
1112 return ((chain != 0) ? pte : NULL);
1116 * Remove an entry from the list of managed mappings.
1119 pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va, pv_entry_t pv)
1122 rw_assert(&pvh_global_lock, RA_WLOCKED);
1124 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
1125 if (pmap == PV_PMAP(pv) && va == pv->pv_va)
1131 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
1132 if (TAILQ_FIRST(&m->md.pv_list) == NULL)
1133 vm_page_aflag_clear(m, PGA_WRITEABLE);
1135 free_pv_entry(pmap, pv);
1143 * Create a pv entry for page at pa for
1147 pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
1151 rw_assert(&pvh_global_lock, RA_WLOCKED);
1152 pv = get_pv_entry(pmap, FALSE);
1154 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
1158 * Routine: pmap_extract
1160 * Extract the physical page address associated
1161 * with the given map/virtual_address pair.
1164 pmap_extract(pmap_t pmap, vm_offset_t va)
1166 struct ia64_lpte *pte;
1172 oldpmap = pmap_switch(pmap);
1173 pte = pmap_find_vhpt(va);
1174 if (pte != NULL && pmap_present(pte))
1176 pmap_switch(oldpmap);
1182 * Routine: pmap_extract_and_hold
1184 * Atomically extract and hold the physical page
1185 * with the given pmap and virtual address pair
1186 * if that mapping permits the given protection.
1189 pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
1191 struct ia64_lpte *pte;
1199 oldpmap = pmap_switch(pmap);
1201 pte = pmap_find_vhpt(va);
1202 if (pte != NULL && pmap_present(pte) &&
1203 (pmap_prot(pte) & prot) == prot) {
1204 m = PHYS_TO_VM_PAGE(pmap_ppn(pte));
1205 if (vm_page_pa_tryrelock(pmap, pmap_ppn(pte), &pa))
1210 pmap_switch(oldpmap);
1215 /***************************************************
1216 * Low level mapping routines.....
1217 ***************************************************/
1220 * Find the kernel lpte for mapping the given virtual address, which
1221 * must be in the part of region 5 which we can cover with our kernel
1224 static struct ia64_lpte *
1225 pmap_find_kpte(vm_offset_t va)
1227 struct ia64_lpte **dir1;
1228 struct ia64_lpte *leaf;
1230 KASSERT((va >> 61) == 5,
1231 ("kernel mapping 0x%lx not in region 5", va));
1232 KASSERT(va < kernel_vm_end,
1233 ("kernel mapping 0x%lx out of range", va));
1235 dir1 = ia64_kptdir[KPTE_DIR0_INDEX(va)];
1236 leaf = dir1[KPTE_DIR1_INDEX(va)];
1237 return (&leaf[KPTE_PTE_INDEX(va)]);
1241 * Find a pte suitable for mapping a user-space address. If one exists
1242 * in the VHPT, that one will be returned, otherwise a new pte is
1245 static struct ia64_lpte *
1246 pmap_find_pte(vm_offset_t va)
1248 struct ia64_lpte *pte;
1250 if (va >= VM_MAXUSER_ADDRESS)
1251 return pmap_find_kpte(va);
1253 pte = pmap_find_vhpt(va);
1255 pte = uma_zalloc(ptezone, M_NOWAIT | M_ZERO);
1256 pte->tag = 1UL << 63;
1262 * Free a pte which is now unused. This simply returns it to the zone
1263 * allocator if it is a user mapping. For kernel mappings, clear the
1264 * valid bit to make it clear that the mapping is not currently used.
1267 pmap_free_pte(struct ia64_lpte *pte, vm_offset_t va)
1269 if (va < VM_MAXUSER_ADDRESS)
1270 uma_zfree(ptezone, pte);
1272 pmap_clear_present(pte);
1275 static PMAP_INLINE void
1276 pmap_pte_prot(pmap_t pm, struct ia64_lpte *pte, vm_prot_t prot)
1278 static long prot2ar[4] = {
1279 PTE_AR_R, /* VM_PROT_NONE */
1280 PTE_AR_RW, /* VM_PROT_WRITE */
1281 PTE_AR_RX|PTE_ED, /* VM_PROT_EXECUTE */
1282 PTE_AR_RWX|PTE_ED /* VM_PROT_WRITE|VM_PROT_EXECUTE */
1285 pte->pte &= ~(PTE_PROT_MASK | PTE_PL_MASK | PTE_AR_MASK | PTE_ED);
1286 pte->pte |= (uint64_t)(prot & VM_PROT_ALL) << 56;
1287 pte->pte |= (prot == VM_PROT_NONE || pm == kernel_pmap)
1288 ? PTE_PL_KERN : PTE_PL_USER;
1289 pte->pte |= prot2ar[(prot & VM_PROT_ALL) >> 1];
1292 static PMAP_INLINE void
1293 pmap_pte_attr(struct ia64_lpte *pte, vm_memattr_t ma)
1296 pte->pte &= ~PTE_MA_MASK;
1297 pte->pte |= (ma & PTE_MA_MASK);
1301 * Set a pte to contain a valid mapping and enter it in the VHPT. If
1302 * the pte was orginally valid, then its assumed to already be in the
1304 * This functions does not set the protection bits. It's expected
1305 * that those have been set correctly prior to calling this function.
1308 pmap_set_pte(struct ia64_lpte *pte, vm_offset_t va, vm_offset_t pa,
1309 boolean_t wired, boolean_t managed)
1312 pte->pte &= PTE_PROT_MASK | PTE_MA_MASK | PTE_PL_MASK |
1313 PTE_AR_MASK | PTE_ED;
1314 pte->pte |= PTE_PRESENT;
1315 pte->pte |= (managed) ? PTE_MANAGED : (PTE_DIRTY | PTE_ACCESSED);
1316 pte->pte |= (wired) ? PTE_WIRED : 0;
1317 pte->pte |= pa & PTE_PPN_MASK;
1319 pte->itir = PAGE_SHIFT << 2;
1321 pte->tag = ia64_ttag(va);
1325 * Remove the (possibly managed) mapping represented by pte from the
1329 pmap_remove_pte(pmap_t pmap, struct ia64_lpte *pte, vm_offset_t va,
1330 pv_entry_t pv, int freepte)
1336 * First remove from the VHPT.
1338 error = pmap_remove_vhpt(va);
1342 pmap_invalidate_page(va);
1344 if (pmap_wired(pte))
1345 pmap->pm_stats.wired_count -= 1;
1347 pmap->pm_stats.resident_count -= 1;
1348 if (pmap_managed(pte)) {
1349 m = PHYS_TO_VM_PAGE(pmap_ppn(pte));
1350 if (pmap_dirty(pte))
1352 if (pmap_accessed(pte))
1353 vm_page_aflag_set(m, PGA_REFERENCED);
1355 error = pmap_remove_entry(pmap, m, va, pv);
1358 pmap_free_pte(pte, va);
1364 * Extract the physical page address associated with a kernel
1368 pmap_kextract(vm_offset_t va)
1370 struct ia64_lpte *pte;
1371 uint64_t *pbvm_pgtbl;
1375 KASSERT(va >= VM_MAXUSER_ADDRESS, ("Must be kernel VA"));
1377 /* Regions 6 and 7 are direct mapped. */
1378 if (va >= IA64_RR_BASE(6)) {
1379 pa = IA64_RR_MASK(va);
1383 /* Region 5 is our KVA. Bail out if the VA is beyond our limits. */
1384 if (va >= kernel_vm_end)
1386 if (va >= VM_INIT_KERNEL_ADDRESS) {
1387 pte = pmap_find_kpte(va);
1388 pa = pmap_present(pte) ? pmap_ppn(pte) | (va & PAGE_MASK) : 0;
1392 /* The PBVM page table. */
1393 if (va >= IA64_PBVM_PGTBL + bootinfo->bi_pbvm_pgtblsz)
1395 if (va >= IA64_PBVM_PGTBL) {
1396 pa = (va - IA64_PBVM_PGTBL) + bootinfo->bi_pbvm_pgtbl;
1400 /* The PBVM itself. */
1401 if (va >= IA64_PBVM_BASE) {
1402 pbvm_pgtbl = (void *)IA64_PBVM_PGTBL;
1403 idx = (va - IA64_PBVM_BASE) >> IA64_PBVM_PAGE_SHIFT;
1404 if (idx >= (bootinfo->bi_pbvm_pgtblsz >> 3))
1406 if ((pbvm_pgtbl[idx] & PTE_PRESENT) == 0)
1408 pa = (pbvm_pgtbl[idx] & PTE_PPN_MASK) +
1409 (va & IA64_PBVM_PAGE_MASK);
1414 printf("XXX: %s: va=%#lx is invalid\n", __func__, va);
1423 * Add a list of wired pages to the kva this routine is only used for
1424 * temporary kernel mappings that do not need to have page modification
1425 * or references recorded. Note that old mappings are simply written
1426 * over. The page is effectively wired, but it's customary to not have
1427 * the PTE reflect that, nor update statistics.
1430 pmap_qenter(vm_offset_t va, vm_page_t *m, int count)
1432 struct ia64_lpte *pte;
1435 for (i = 0; i < count; i++) {
1436 pte = pmap_find_kpte(va);
1437 if (pmap_present(pte))
1438 pmap_invalidate_page(va);
1440 pmap_enter_vhpt(pte, va);
1441 pmap_pte_prot(kernel_pmap, pte, VM_PROT_ALL);
1442 pmap_pte_attr(pte, m[i]->md.memattr);
1443 pmap_set_pte(pte, va, VM_PAGE_TO_PHYS(m[i]), FALSE, FALSE);
1449 * this routine jerks page mappings from the
1450 * kernel -- it is meant only for temporary mappings.
1453 pmap_qremove(vm_offset_t va, int count)
1455 struct ia64_lpte *pte;
1458 for (i = 0; i < count; i++) {
1459 pte = pmap_find_kpte(va);
1460 if (pmap_present(pte)) {
1461 pmap_remove_vhpt(va);
1462 pmap_invalidate_page(va);
1463 pmap_clear_present(pte);
1470 * Add a wired page to the kva. As for pmap_qenter(), it's customary
1471 * to not have the PTE reflect that, nor update statistics.
1474 pmap_kenter(vm_offset_t va, vm_offset_t pa)
1476 struct ia64_lpte *pte;
1478 pte = pmap_find_kpte(va);
1479 if (pmap_present(pte))
1480 pmap_invalidate_page(va);
1482 pmap_enter_vhpt(pte, va);
1483 pmap_pte_prot(kernel_pmap, pte, VM_PROT_ALL);
1484 pmap_pte_attr(pte, VM_MEMATTR_DEFAULT);
1485 pmap_set_pte(pte, va, pa, FALSE, FALSE);
1489 * Remove a page from the kva
1492 pmap_kremove(vm_offset_t va)
1494 struct ia64_lpte *pte;
1496 pte = pmap_find_kpte(va);
1497 if (pmap_present(pte)) {
1498 pmap_remove_vhpt(va);
1499 pmap_invalidate_page(va);
1500 pmap_clear_present(pte);
1505 * Used to map a range of physical addresses into kernel
1506 * virtual address space.
1508 * The value passed in '*virt' is a suggested virtual address for
1509 * the mapping. Architectures which can support a direct-mapped
1510 * physical to virtual region can return the appropriate address
1511 * within that region, leaving '*virt' unchanged. Other
1512 * architectures should map the pages starting at '*virt' and
1513 * update '*virt' with the first usable address after the mapped
1517 pmap_map(vm_offset_t *virt, vm_offset_t start, vm_offset_t end, int prot)
1519 return IA64_PHYS_TO_RR7(start);
1523 * Remove the given range of addresses from the specified map.
1525 * It is assumed that the start and end are properly
1526 * rounded to the page size.
1528 * Sparsely used ranges are inefficiently removed. The VHPT is
1529 * probed for every page within the range. XXX
1532 pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
1536 struct ia64_lpte *pte;
1539 * Perform an unsynchronized read. This is, however, safe.
1541 if (pmap->pm_stats.resident_count == 0)
1544 rw_wlock(&pvh_global_lock);
1546 oldpmap = pmap_switch(pmap);
1547 for (va = sva; va < eva; va += PAGE_SIZE) {
1548 pte = pmap_find_vhpt(va);
1550 pmap_remove_pte(pmap, pte, va, 0, 1);
1552 rw_wunlock(&pvh_global_lock);
1553 pmap_switch(oldpmap);
1558 * Routine: pmap_remove_all
1560 * Removes this physical page from
1561 * all physical maps in which it resides.
1562 * Reflects back modify bits to the pager.
1565 * Original versions of this routine were very
1566 * inefficient because they iteratively called
1567 * pmap_remove (slow...)
1571 pmap_remove_all(vm_page_t m)
1576 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
1577 ("pmap_remove_all: page %p is not managed", m));
1578 rw_wlock(&pvh_global_lock);
1579 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
1580 struct ia64_lpte *pte;
1581 pmap_t pmap = PV_PMAP(pv);
1582 vm_offset_t va = pv->pv_va;
1585 oldpmap = pmap_switch(pmap);
1586 pte = pmap_find_vhpt(va);
1587 KASSERT(pte != NULL, ("pte"));
1588 if (pmap_ppn(pte) != VM_PAGE_TO_PHYS(m))
1589 panic("pmap_remove_all: pv_table for %lx is inconsistent", VM_PAGE_TO_PHYS(m));
1590 pmap_remove_pte(pmap, pte, va, pv, 1);
1591 pmap_switch(oldpmap);
1594 vm_page_aflag_clear(m, PGA_WRITEABLE);
1595 rw_wunlock(&pvh_global_lock);
1599 * Set the physical protection on the
1600 * specified range of this map as requested.
1603 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
1606 struct ia64_lpte *pte;
1608 if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
1609 pmap_remove(pmap, sva, eva);
1613 if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) ==
1614 (VM_PROT_WRITE|VM_PROT_EXECUTE))
1617 if ((sva & PAGE_MASK) || (eva & PAGE_MASK))
1618 panic("pmap_protect: unaligned addresses");
1621 oldpmap = pmap_switch(pmap);
1622 for ( ; sva < eva; sva += PAGE_SIZE) {
1623 /* If page is invalid, skip this page */
1624 pte = pmap_find_vhpt(sva);
1628 /* If there's no change, skip it too */
1629 if (pmap_prot(pte) == prot)
1632 if ((prot & VM_PROT_WRITE) == 0 &&
1633 pmap_managed(pte) && pmap_dirty(pte)) {
1634 vm_paddr_t pa = pmap_ppn(pte);
1635 vm_page_t m = PHYS_TO_VM_PAGE(pa);
1638 pmap_clear_dirty(pte);
1641 if (prot & VM_PROT_EXECUTE)
1642 ia64_sync_icache(sva, PAGE_SIZE);
1644 pmap_pte_prot(pmap, pte, prot);
1645 pmap_invalidate_page(sva);
1647 pmap_switch(oldpmap);
1652 * Insert the given physical page (p) at
1653 * the specified virtual address (v) in the
1654 * target physical map with the protection requested.
1656 * If specified, the page will be wired down, meaning
1657 * that the related pte can not be reclaimed.
1659 * NB: This is the only routine which MAY NOT lazy-evaluate
1660 * or lose information. That is, this routine must actually
1661 * insert this page into the given map NOW.
1664 pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t m,
1665 vm_prot_t prot, boolean_t wired)
1670 struct ia64_lpte origpte;
1671 struct ia64_lpte *pte;
1672 boolean_t icache_inval, managed;
1674 rw_wlock(&pvh_global_lock);
1676 oldpmap = pmap_switch(pmap);
1679 KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig"));
1680 KASSERT((m->oflags & VPO_UNMANAGED) != 0 || vm_page_xbusied(m),
1681 ("pmap_enter: page %p is not busy", m));
1684 * Find (or create) a pte for the given mapping.
1686 while ((pte = pmap_find_pte(va)) == NULL) {
1687 pmap_switch(oldpmap);
1689 rw_wunlock(&pvh_global_lock);
1691 rw_wlock(&pvh_global_lock);
1693 oldpmap = pmap_switch(pmap);
1696 if (!pmap_present(pte)) {
1698 pmap_enter_vhpt(pte, va);
1700 opa = pmap_ppn(pte);
1702 pa = VM_PAGE_TO_PHYS(m);
1704 icache_inval = (prot & VM_PROT_EXECUTE) ? TRUE : FALSE;
1707 * Mapping has not changed, must be protection or wiring change.
1711 * Wiring change, just update stats. We don't worry about
1712 * wiring PT pages as they remain resident as long as there
1713 * are valid mappings in them. Hence, if a user page is wired,
1714 * the PT page will be also.
1716 if (wired && !pmap_wired(&origpte))
1717 pmap->pm_stats.wired_count++;
1718 else if (!wired && pmap_wired(&origpte))
1719 pmap->pm_stats.wired_count--;
1721 managed = (pmap_managed(&origpte)) ? TRUE : FALSE;
1724 * We might be turning off write access to the page,
1725 * so we go ahead and sense modify status. Otherwise,
1726 * we can avoid I-cache invalidation if the page
1727 * already allowed execution.
1729 if (managed && pmap_dirty(&origpte))
1731 else if (pmap_exec(&origpte))
1732 icache_inval = FALSE;
1734 pmap_invalidate_page(va);
1739 * Mapping has changed, invalidate old range and fall
1740 * through to handle validating new mapping.
1743 pmap_remove_pte(pmap, pte, va, 0, 0);
1744 pmap_enter_vhpt(pte, va);
1748 * Enter on the PV list if part of our managed memory.
1750 if ((m->oflags & VPO_UNMANAGED) == 0) {
1751 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva,
1752 ("pmap_enter: managed mapping within the clean submap"));
1753 pmap_insert_entry(pmap, va, m);
1758 * Increment counters
1760 pmap->pm_stats.resident_count++;
1762 pmap->pm_stats.wired_count++;
1767 * Now validate mapping with desired protection/wiring. This
1768 * adds the pte to the VHPT if necessary.
1770 pmap_pte_prot(pmap, pte, prot);
1771 pmap_pte_attr(pte, m->md.memattr);
1772 pmap_set_pte(pte, va, pa, wired, managed);
1774 /* Invalidate the I-cache when needed. */
1776 ia64_sync_icache(va, PAGE_SIZE);
1778 if ((prot & VM_PROT_WRITE) != 0 && managed)
1779 vm_page_aflag_set(m, PGA_WRITEABLE);
1780 rw_wunlock(&pvh_global_lock);
1781 pmap_switch(oldpmap);
1786 * Maps a sequence of resident pages belonging to the same object.
1787 * The sequence begins with the given page m_start. This page is
1788 * mapped at the given virtual address start. Each subsequent page is
1789 * mapped at a virtual address that is offset from start by the same
1790 * amount as the page is offset from m_start within the object. The
1791 * last page in the sequence is the page with the largest offset from
1792 * m_start that can be mapped at a virtual address less than the given
1793 * virtual address end. Not every virtual page between start and end
1794 * is mapped; only those for which a resident page exists with the
1795 * corresponding offset from m_start are mapped.
1798 pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,
1799 vm_page_t m_start, vm_prot_t prot)
1803 vm_pindex_t diff, psize;
1805 VM_OBJECT_ASSERT_LOCKED(m_start->object);
1807 psize = atop(end - start);
1809 rw_wlock(&pvh_global_lock);
1811 oldpmap = pmap_switch(pmap);
1812 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
1813 pmap_enter_quick_locked(pmap, start + ptoa(diff), m, prot);
1814 m = TAILQ_NEXT(m, listq);
1816 rw_wunlock(&pvh_global_lock);
1817 pmap_switch(oldpmap);
1822 * this code makes some *MAJOR* assumptions:
1823 * 1. Current pmap & pmap exists.
1826 * 4. No page table pages.
1827 * but is *MUCH* faster than pmap_enter...
1831 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
1835 rw_wlock(&pvh_global_lock);
1837 oldpmap = pmap_switch(pmap);
1838 pmap_enter_quick_locked(pmap, va, m, prot);
1839 rw_wunlock(&pvh_global_lock);
1840 pmap_switch(oldpmap);
1845 pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
1848 struct ia64_lpte *pte;
1851 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva ||
1852 (m->oflags & VPO_UNMANAGED) != 0,
1853 ("pmap_enter_quick_locked: managed mapping within the clean submap"));
1854 rw_assert(&pvh_global_lock, RA_WLOCKED);
1855 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1857 if ((pte = pmap_find_pte(va)) == NULL)
1860 if (!pmap_present(pte)) {
1861 /* Enter on the PV list if the page is managed. */
1862 if ((m->oflags & VPO_UNMANAGED) == 0) {
1863 if (!pmap_try_insert_pv_entry(pmap, va, m)) {
1864 pmap_free_pte(pte, va);
1871 /* Increment counters. */
1872 pmap->pm_stats.resident_count++;
1874 /* Initialise with R/O protection and enter into VHPT. */
1875 pmap_enter_vhpt(pte, va);
1876 pmap_pte_prot(pmap, pte,
1877 prot & (VM_PROT_READ | VM_PROT_EXECUTE));
1878 pmap_pte_attr(pte, m->md.memattr);
1879 pmap_set_pte(pte, va, VM_PAGE_TO_PHYS(m), FALSE, managed);
1881 if (prot & VM_PROT_EXECUTE)
1882 ia64_sync_icache(va, PAGE_SIZE);
1887 * pmap_object_init_pt preloads the ptes for a given object
1888 * into the specified pmap. This eliminates the blast of soft
1889 * faults on process startup and immediately after an mmap.
1892 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr,
1893 vm_object_t object, vm_pindex_t pindex,
1897 VM_OBJECT_ASSERT_WLOCKED(object);
1898 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG,
1899 ("pmap_object_init_pt: non-device object"));
1903 * Routine: pmap_change_wiring
1904 * Function: Change the wiring attribute for a map/virtual-address
1906 * In/out conditions:
1907 * The mapping must already exist in the pmap.
1910 pmap_change_wiring(pmap, va, wired)
1911 register pmap_t pmap;
1916 struct ia64_lpte *pte;
1919 oldpmap = pmap_switch(pmap);
1921 pte = pmap_find_vhpt(va);
1922 KASSERT(pte != NULL, ("pte"));
1923 if (wired && !pmap_wired(pte)) {
1924 pmap->pm_stats.wired_count++;
1925 pmap_set_wired(pte);
1926 } else if (!wired && pmap_wired(pte)) {
1927 pmap->pm_stats.wired_count--;
1928 pmap_clear_wired(pte);
1931 pmap_switch(oldpmap);
1938 * Copy the range specified by src_addr/len
1939 * from the source map to the range dst_addr/len
1940 * in the destination map.
1942 * This routine is only advisory and need not do anything.
1946 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
1947 vm_offset_t src_addr)
1953 * pmap_zero_page zeros the specified hardware page by
1954 * mapping it into virtual memory and using bzero to clear
1959 pmap_zero_page(vm_page_t m)
1963 p = (void *)pmap_page_to_va(m);
1964 bzero(p, PAGE_SIZE);
1969 * pmap_zero_page_area zeros the specified hardware page by
1970 * mapping it into virtual memory and using bzero to clear
1973 * off and size must reside within a single page.
1977 pmap_zero_page_area(vm_page_t m, int off, int size)
1981 p = (void *)pmap_page_to_va(m);
1982 bzero(p + off, size);
1987 * pmap_zero_page_idle zeros the specified hardware page by
1988 * mapping it into virtual memory and using bzero to clear
1989 * its contents. This is for the vm_idlezero process.
1993 pmap_zero_page_idle(vm_page_t m)
1997 p = (void *)pmap_page_to_va(m);
1998 bzero(p, PAGE_SIZE);
2003 * pmap_copy_page copies the specified (machine independent)
2004 * page by mapping the page into virtual memory and using
2005 * bcopy to copy the page, one machine dependent page at a
2009 pmap_copy_page(vm_page_t msrc, vm_page_t mdst)
2013 src = (void *)pmap_page_to_va(msrc);
2014 dst = (void *)pmap_page_to_va(mdst);
2015 bcopy(src, dst, PAGE_SIZE);
2018 int unmapped_buf_allowed;
2021 pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[],
2022 vm_offset_t b_offset, int xfersize)
2025 vm_offset_t a_pg_offset, b_pg_offset;
2028 while (xfersize > 0) {
2029 a_pg_offset = a_offset & PAGE_MASK;
2030 cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
2031 a_cp = (char *)pmap_page_to_va(ma[a_offset >> PAGE_SHIFT]) +
2033 b_pg_offset = b_offset & PAGE_MASK;
2034 cnt = min(cnt, PAGE_SIZE - b_pg_offset);
2035 b_cp = (char *)pmap_page_to_va(mb[b_offset >> PAGE_SHIFT]) +
2037 bcopy(a_cp, b_cp, cnt);
2045 * Returns true if the pmap's pv is one of the first
2046 * 16 pvs linked to from this page. This count may
2047 * be changed upwards or downwards in the future; it
2048 * is only necessary that true be returned for a small
2049 * subset of pmaps for proper page aging.
2052 pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
2058 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2059 ("pmap_page_exists_quick: page %p is not managed", m));
2061 rw_wlock(&pvh_global_lock);
2062 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2063 if (PV_PMAP(pv) == pmap) {
2071 rw_wunlock(&pvh_global_lock);
2076 * pmap_page_wired_mappings:
2078 * Return the number of managed mappings to the given physical page
2082 pmap_page_wired_mappings(vm_page_t m)
2084 struct ia64_lpte *pte;
2085 pmap_t oldpmap, pmap;
2090 if ((m->oflags & VPO_UNMANAGED) != 0)
2092 rw_wlock(&pvh_global_lock);
2093 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2096 oldpmap = pmap_switch(pmap);
2097 pte = pmap_find_vhpt(pv->pv_va);
2098 KASSERT(pte != NULL, ("pte"));
2099 if (pmap_wired(pte))
2101 pmap_switch(oldpmap);
2104 rw_wunlock(&pvh_global_lock);
2109 * Remove all pages from specified address space
2110 * this aids process exit speeds. Also, this code
2111 * is special cased for current process only, but
2112 * can have the more generic (and slightly slower)
2113 * mode enabled. This is much faster than pmap_remove
2114 * in the case of running down an entire address space.
2117 pmap_remove_pages(pmap_t pmap)
2119 struct pv_chunk *pc, *npc;
2120 struct ia64_lpte *pte;
2124 u_long inuse, bitmask;
2125 int allfree, bit, field, idx;
2127 if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace)) {
2128 printf("warning: %s called with non-current pmap\n",
2132 rw_wlock(&pvh_global_lock);
2134 TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {
2136 for (field = 0; field < _NPCM; field++) {
2137 inuse = ~pc->pc_map[field] & pc_freemask[field];
2138 while (inuse != 0) {
2139 bit = ffsl(inuse) - 1;
2140 bitmask = 1UL << bit;
2141 idx = field * sizeof(inuse) * NBBY + bit;
2142 pv = &pc->pc_pventry[idx];
2145 pte = pmap_find_vhpt(va);
2146 KASSERT(pte != NULL, ("pte"));
2147 if (pmap_wired(pte)) {
2151 pmap_remove_vhpt(va);
2152 pmap_invalidate_page(va);
2153 m = PHYS_TO_VM_PAGE(pmap_ppn(pte));
2154 if (pmap_dirty(pte))
2156 pmap_free_pte(pte, va);
2158 PV_STAT(pv_entry_frees++);
2159 PV_STAT(pv_entry_spare++);
2161 pc->pc_map[field] |= bitmask;
2162 pmap->pm_stats.resident_count--;
2163 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
2164 if (TAILQ_EMPTY(&m->md.pv_list))
2165 vm_page_aflag_clear(m, PGA_WRITEABLE);
2169 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
2173 rw_wunlock(&pvh_global_lock);
2178 * pmap_ts_referenced:
2180 * Return a count of reference bits for a page, clearing those bits.
2181 * It is not necessary for every reference bit to be cleared, but it
2182 * is necessary that 0 only be returned when there are truly no
2183 * reference bits set.
2185 * XXX: The exact number of bits to check and clear is a matter that
2186 * should be tested and standardized at some point in the future for
2187 * optimal aging of shared pages.
2190 pmap_ts_referenced(vm_page_t m)
2192 struct ia64_lpte *pte;
2193 pmap_t oldpmap, pmap;
2197 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2198 ("pmap_ts_referenced: page %p is not managed", m));
2199 rw_wlock(&pvh_global_lock);
2200 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2203 oldpmap = pmap_switch(pmap);
2204 pte = pmap_find_vhpt(pv->pv_va);
2205 KASSERT(pte != NULL, ("pte"));
2206 if (pmap_accessed(pte)) {
2208 pmap_clear_accessed(pte);
2209 pmap_invalidate_page(pv->pv_va);
2211 pmap_switch(oldpmap);
2214 rw_wunlock(&pvh_global_lock);
2221 * Return whether or not the specified physical page was modified
2222 * in any physical maps.
2225 pmap_is_modified(vm_page_t m)
2227 struct ia64_lpte *pte;
2228 pmap_t oldpmap, pmap;
2232 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2233 ("pmap_is_modified: page %p is not managed", m));
2237 * If the page is not exclusive busied, then PGA_WRITEABLE cannot be
2238 * concurrently set while the object is locked. Thus, if PGA_WRITEABLE
2239 * is clear, no PTEs can be dirty.
2241 VM_OBJECT_ASSERT_WLOCKED(m->object);
2242 if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0)
2244 rw_wlock(&pvh_global_lock);
2245 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2248 oldpmap = pmap_switch(pmap);
2249 pte = pmap_find_vhpt(pv->pv_va);
2250 pmap_switch(oldpmap);
2251 KASSERT(pte != NULL, ("pte"));
2252 rv = pmap_dirty(pte) ? TRUE : FALSE;
2257 rw_wunlock(&pvh_global_lock);
2262 * pmap_is_prefaultable:
2264 * Return whether or not the specified virtual address is elgible
2268 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)
2270 struct ia64_lpte *pte;
2272 pte = pmap_find_vhpt(addr);
2273 if (pte != NULL && pmap_present(pte))
2279 * pmap_is_referenced:
2281 * Return whether or not the specified physical page was referenced
2282 * in any physical maps.
2285 pmap_is_referenced(vm_page_t m)
2287 struct ia64_lpte *pte;
2288 pmap_t oldpmap, pmap;
2292 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2293 ("pmap_is_referenced: page %p is not managed", m));
2295 rw_wlock(&pvh_global_lock);
2296 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2299 oldpmap = pmap_switch(pmap);
2300 pte = pmap_find_vhpt(pv->pv_va);
2301 pmap_switch(oldpmap);
2302 KASSERT(pte != NULL, ("pte"));
2303 rv = pmap_accessed(pte) ? TRUE : FALSE;
2308 rw_wunlock(&pvh_global_lock);
2313 * Apply the given advice to the specified range of addresses within the
2314 * given pmap. Depending on the advice, clear the referenced and/or
2315 * modified flags in each mapping and set the mapped page's dirty field.
2318 pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
2320 struct ia64_lpte *pte;
2325 oldpmap = pmap_switch(pmap);
2326 for (; sva < eva; sva += PAGE_SIZE) {
2327 /* If page is invalid, skip this page. */
2328 pte = pmap_find_vhpt(sva);
2332 /* If it isn't managed, skip it too. */
2333 if (!pmap_managed(pte))
2336 /* Clear its modified and referenced bits. */
2337 if (pmap_dirty(pte)) {
2338 if (advice == MADV_DONTNEED) {
2340 * Future calls to pmap_is_modified() can be
2341 * avoided by making the page dirty now.
2343 m = PHYS_TO_VM_PAGE(pmap_ppn(pte));
2346 pmap_clear_dirty(pte);
2347 } else if (!pmap_accessed(pte))
2349 pmap_clear_accessed(pte);
2350 pmap_invalidate_page(sva);
2352 pmap_switch(oldpmap);
2357 * Clear the modify bits on the specified physical page.
2360 pmap_clear_modify(vm_page_t m)
2362 struct ia64_lpte *pte;
2363 pmap_t oldpmap, pmap;
2366 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2367 ("pmap_clear_modify: page %p is not managed", m));
2368 VM_OBJECT_ASSERT_WLOCKED(m->object);
2369 KASSERT(!vm_page_xbusied(m),
2370 ("pmap_clear_modify: page %p is exclusive busied", m));
2373 * If the page is not PGA_WRITEABLE, then no PTEs can be modified.
2374 * If the object containing the page is locked and the page is not
2375 * exclusive busied, then PGA_WRITEABLE cannot be concurrently set.
2377 if ((m->aflags & PGA_WRITEABLE) == 0)
2379 rw_wlock(&pvh_global_lock);
2380 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2383 oldpmap = pmap_switch(pmap);
2384 pte = pmap_find_vhpt(pv->pv_va);
2385 KASSERT(pte != NULL, ("pte"));
2386 if (pmap_dirty(pte)) {
2387 pmap_clear_dirty(pte);
2388 pmap_invalidate_page(pv->pv_va);
2390 pmap_switch(oldpmap);
2393 rw_wunlock(&pvh_global_lock);
2397 * Clear the write and modified bits in each of the given page's mappings.
2400 pmap_remove_write(vm_page_t m)
2402 struct ia64_lpte *pte;
2403 pmap_t oldpmap, pmap;
2407 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2408 ("pmap_remove_write: page %p is not managed", m));
2411 * If the page is not exclusive busied, then PGA_WRITEABLE cannot be
2412 * set by another thread while the object is locked. Thus,
2413 * if PGA_WRITEABLE is clear, no page table entries need updating.
2415 VM_OBJECT_ASSERT_WLOCKED(m->object);
2416 if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0)
2418 rw_wlock(&pvh_global_lock);
2419 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2422 oldpmap = pmap_switch(pmap);
2423 pte = pmap_find_vhpt(pv->pv_va);
2424 KASSERT(pte != NULL, ("pte"));
2425 prot = pmap_prot(pte);
2426 if ((prot & VM_PROT_WRITE) != 0) {
2427 if (pmap_dirty(pte)) {
2429 pmap_clear_dirty(pte);
2431 prot &= ~VM_PROT_WRITE;
2432 pmap_pte_prot(pmap, pte, prot);
2433 pmap_pte_attr(pte, m->md.memattr);
2434 pmap_invalidate_page(pv->pv_va);
2436 pmap_switch(oldpmap);
2439 vm_page_aflag_clear(m, PGA_WRITEABLE);
2440 rw_wunlock(&pvh_global_lock);
2444 * Map a set of physical memory pages into the kernel virtual
2445 * address space. Return a pointer to where it is mapped. This
2446 * routine is intended to be used for mapping device memory,
2450 pmap_mapdev(vm_paddr_t pa, vm_size_t sz)
2452 static void *last_va = NULL;
2453 static vm_paddr_t last_pa = 0;
2454 static vm_size_t last_sz = 0;
2458 if (pa == last_pa && sz == last_sz)
2461 md = efi_md_find(pa);
2463 printf("%s: [%#lx..%#lx] not covered by memory descriptor\n",
2464 __func__, pa, pa + sz - 1);
2465 return ((void *)IA64_PHYS_TO_RR6(pa));
2468 if (md->md_type == EFI_MD_TYPE_FREE) {
2469 printf("%s: [%#lx..%#lx] is in DRAM\n", __func__, pa,
2474 va = (md->md_attr & EFI_MD_ATTR_WB) ? IA64_PHYS_TO_RR7(pa) :
2475 IA64_PHYS_TO_RR6(pa);
2477 last_va = (void *)va;
2484 * 'Unmap' a range mapped by pmap_mapdev().
2487 pmap_unmapdev(vm_offset_t va, vm_size_t size)
2492 * Sets the memory attribute for the specified page.
2495 pmap_page_set_memattr_1(void *arg)
2497 struct ia64_pal_result res;
2499 uintptr_t pp = (uintptr_t)arg;
2501 is = intr_disable();
2502 res = ia64_call_pal_static(pp, 0, 0, 0);
2507 pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma)
2509 struct ia64_lpte *pte;
2510 pmap_t oldpmap, pmap;
2514 rw_wlock(&pvh_global_lock);
2516 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2519 oldpmap = pmap_switch(pmap);
2520 pte = pmap_find_vhpt(pv->pv_va);
2521 KASSERT(pte != NULL, ("pte"));
2522 pmap_pte_attr(pte, ma);
2523 pmap_invalidate_page(pv->pv_va);
2524 pmap_switch(oldpmap);
2527 rw_wunlock(&pvh_global_lock);
2529 if (ma == VM_MEMATTR_UNCACHEABLE) {
2531 smp_rendezvous(NULL, pmap_page_set_memattr_1, NULL,
2532 (void *)PAL_PREFETCH_VISIBILITY);
2534 pmap_page_set_memattr_1((void *)PAL_PREFETCH_VISIBILITY);
2536 va = (void *)pmap_page_to_va(m);
2538 cpu_flush_dcache(va, PAGE_SIZE);
2541 smp_rendezvous(NULL, pmap_page_set_memattr_1, NULL,
2542 (void *)PAL_MC_DRAIN);
2544 pmap_page_set_memattr_1((void *)PAL_MC_DRAIN);
2550 * perform the pmap work for mincore
2553 pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa)
2556 struct ia64_lpte *pte, tpte;
2562 oldpmap = pmap_switch(pmap);
2563 pte = pmap_find_vhpt(addr);
2568 pmap_switch(oldpmap);
2569 if (pte == NULL || !pmap_present(pte)) {
2573 val = MINCORE_INCORE;
2574 if (pmap_dirty(pte))
2575 val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER;
2576 if (pmap_accessed(pte))
2577 val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER;
2578 if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) !=
2579 (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) &&
2580 pmap_managed(pte)) {
2582 /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */
2583 if (vm_page_pa_tryrelock(pmap, pa, locked_pa))
2587 PA_UNLOCK_COND(*locked_pa);
2593 pmap_activate(struct thread *td)
2595 pmap_switch(vmspace_pmap(td->td_proc->p_vmspace));
2599 pmap_switch(pmap_t pm)
2605 prevpm = PCPU_GET(md.current_pmap);
2609 for (i = 0; i < IA64_VM_MINKERN_REGION; i++) {
2610 ia64_set_rr(IA64_RR_BASE(i),
2611 (i << 8)|(PAGE_SHIFT << 2)|1);
2614 for (i = 0; i < IA64_VM_MINKERN_REGION; i++) {
2615 ia64_set_rr(IA64_RR_BASE(i),
2616 (pm->pm_rid[i] << 8)|(PAGE_SHIFT << 2)|1);
2619 PCPU_SET(md.current_pmap, pm);
2628 pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz)
2631 struct ia64_lpte *pte;
2637 sz = (sz + 31) & ~31;
2640 oldpm = pmap_switch(pm);
2642 lim = round_page(va);
2643 len = MIN(lim - va, sz);
2644 pte = pmap_find_vhpt(va);
2645 if (pte != NULL && pmap_present(pte))
2646 ia64_sync_icache(va, len);
2655 * Increase the starting virtual address of the given mapping if a
2656 * different alignment might result in more superpage mappings.
2659 pmap_align_superpage(vm_object_t object, vm_ooffset_t offset,
2660 vm_offset_t *addr, vm_size_t size)
2664 #include "opt_ddb.h"
2668 #include <ddb/ddb.h>
2670 static const char* psnames[] = {
2671 "1B", "2B", "4B", "8B",
2672 "16B", "32B", "64B", "128B",
2673 "256B", "512B", "1K", "2K",
2674 "4K", "8K", "16K", "32K",
2675 "64K", "128K", "256K", "512K",
2676 "1M", "2M", "4M", "8M",
2677 "16M", "32M", "64M", "128M",
2678 "256M", "512M", "1G", "2G"
2684 struct ia64_pal_result res;
2692 static const char *manames[] = {
2693 "WB", "bad", "bad", "bad",
2694 "UC", "UCE", "WC", "NaT",
2697 res = ia64_call_pal_static(PAL_VM_SUMMARY, 0, 0, 0);
2698 if (res.pal_status != 0) {
2699 db_printf("Can't get VM summary\n");
2704 maxtr = (res.pal_result[0] >> 40) & 0xff;
2706 maxtr = (res.pal_result[0] >> 32) & 0xff;
2708 db_printf("V RID Virtual Page Physical Page PgSz ED AR PL D A MA P KEY\n");
2709 for (i = 0; i <= maxtr; i++) {
2710 bzero(&buf, sizeof(buf));
2711 res = ia64_pal_physical(PAL_VM_TR_READ, i, type,
2712 ia64_tpa((uint64_t)&buf));
2713 if (!(res.pal_result[0] & 1))
2714 buf.pte &= ~PTE_AR_MASK;
2715 if (!(res.pal_result[0] & 2))
2716 buf.pte &= ~PTE_PL_MASK;
2717 if (!(res.pal_result[0] & 4))
2718 pmap_clear_dirty(&buf);
2719 if (!(res.pal_result[0] & 8))
2720 buf.pte &= ~PTE_MA_MASK;
2721 db_printf("%d %06x %013lx %013lx %4s %d %d %d %d %d %-3s "
2722 "%d %06x\n", (int)buf.ifa & 1, buf.rr.rr_rid,
2723 buf.ifa >> 12, (buf.pte & PTE_PPN_MASK) >> 12,
2724 psnames[(buf.itir & ITIR_PS_MASK) >> 2],
2725 (buf.pte & PTE_ED) ? 1 : 0,
2726 (int)(buf.pte & PTE_AR_MASK) >> 9,
2727 (int)(buf.pte & PTE_PL_MASK) >> 7,
2728 (pmap_dirty(&buf)) ? 1 : 0,
2729 (pmap_accessed(&buf)) ? 1 : 0,
2730 manames[(buf.pte & PTE_MA_MASK) >> 2],
2731 (pmap_present(&buf)) ? 1 : 0,
2732 (int)((buf.itir & ITIR_KEY_MASK) >> 8));
2736 DB_COMMAND(itr, db_itr)
2741 DB_COMMAND(dtr, db_dtr)
2746 DB_COMMAND(rr, db_rr)
2752 printf("RR RID PgSz VE\n");
2753 for (i = 0; i < 8; i++) {
2754 __asm __volatile ("mov %0=rr[%1]"
2756 : "r"(IA64_RR_BASE(i)));
2757 *(uint64_t *) &rr = t;
2758 printf("%d %06x %4s %d\n",
2759 i, rr.rr_rid, psnames[rr.rr_ps], rr.rr_ve);
2763 DB_COMMAND(thash, db_thash)
2768 db_printf("%p\n", (void *) ia64_thash(addr));
2771 DB_COMMAND(ttag, db_ttag)
2776 db_printf("0x%lx\n", ia64_ttag(addr));
2779 DB_COMMAND(kpte, db_kpte)
2781 struct ia64_lpte *pte;
2784 db_printf("usage: kpte <kva>\n");
2787 if (addr < VM_INIT_KERNEL_ADDRESS) {
2788 db_printf("kpte: error: invalid <kva>\n");
2791 pte = pmap_find_kpte(addr);
2792 db_printf("kpte at %p:\n", pte);
2793 db_printf(" pte =%016lx\n", pte->pte);
2794 db_printf(" itir =%016lx\n", pte->itir);
2795 db_printf(" tag =%016lx\n", pte->tag);
2796 db_printf(" chain=%016lx\n", pte->chain);