2 * Copyright (c) 1991 Regents of the University of California.
4 * Copyright (c) 1994 John S. Dyson
6 * Copyright (c) 1994 David Greenman
8 * Copyright (c) 1998,2000 Doug Rabson
11 * This code is derived from software contributed to Berkeley by
12 * the Systems Programming Group of the University of Utah Computer
13 * Science Department and William Jolitz of UUNET Technologies Inc.
15 * Redistribution and use in source and binary forms, with or without
16 * modification, are permitted provided that the following conditions
18 * 1. Redistributions of source code must retain the above copyright
19 * notice, this list of conditions and the following disclaimer.
20 * 2. Redistributions in binary form must reproduce the above copyright
21 * notice, this list of conditions and the following disclaimer in the
22 * documentation and/or other materials provided with the distribution.
23 * 3. All advertising materials mentioning features or use of this software
24 * must display the following acknowledgement:
25 * This product includes software developed by the University of
26 * California, Berkeley and its contributors.
27 * 4. Neither the name of the University nor the names of its contributors
28 * may be used to endorse or promote products derived from this software
29 * without specific prior written permission.
31 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
32 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
33 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
34 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
35 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
36 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
37 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
38 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
39 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
40 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
43 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91
44 * from: i386 Id: pmap.c,v 1.193 1998/04/19 15:22:48 bde Exp
45 * with some ideas from NetBSD's alpha pmap
48 #include <sys/cdefs.h>
49 __FBSDID("$FreeBSD$");
51 #include <sys/param.h>
52 #include <sys/kernel.h>
55 #include <sys/mutex.h>
58 #include <sys/sysctl.h>
59 #include <sys/systm.h>
62 #include <vm/vm_page.h>
63 #include <vm/vm_map.h>
64 #include <vm/vm_object.h>
65 #include <vm/vm_pageout.h>
68 #include <machine/md_var.h>
69 #include <machine/pal.h>
72 * Manages physical address maps.
74 * In addition to hardware address maps, this
75 * module is called upon to provide software-use-only
76 * maps which may or may not be stored in the same
77 * form as hardware maps. These pseudo-maps are
78 * used to store intermediate results from copy
79 * operations to and from address spaces.
81 * Since the information managed by this module is
82 * also stored by the logical address mapping module,
83 * this module may throw away valid virtual-to-physical
84 * mappings at almost any time. However, invalidations
85 * of virtual-to-physical mappings must be done as
88 * In order to cope with hardware architectures which
89 * make virtual-to-physical map invalidates expensive,
90 * this module may delay invalidate or reduced protection
91 * operations until such time as they are actually
92 * necessary. This module is given full information as
93 * to which processors are currently using which maps,
94 * and to when physical maps must be made correct.
98 * Following the Linux model, region IDs are allocated in groups of
99 * eight so that a single region ID can be used for as many RRs as we
100 * want by encoding the RR number into the low bits of the ID.
102 * We reserve region ID 0 for the kernel and allocate the remaining
103 * IDs for user pmaps.
106 * User virtually mapped
109 * Kernel virtually mapped
112 * Kernel physically mapped uncacheable
115 * Kernel physically mapped cacheable
118 /* XXX move to a header. */
119 extern uint64_t ia64_gateway_page[];
121 MALLOC_DEFINE(M_PMAP, "PMAP", "PMAP Structures");
123 #ifndef PMAP_SHPGPERPROC
124 #define PMAP_SHPGPERPROC 200
127 #if !defined(DIAGNOSTIC)
128 #define PMAP_INLINE __inline
133 #define pmap_accessed(lpte) ((lpte)->pte & PTE_ACCESSED)
134 #define pmap_dirty(lpte) ((lpte)->pte & PTE_DIRTY)
135 #define pmap_managed(lpte) ((lpte)->pte & PTE_MANAGED)
136 #define pmap_ppn(lpte) ((lpte)->pte & PTE_PPN_MASK)
137 #define pmap_present(lpte) ((lpte)->pte & PTE_PRESENT)
138 #define pmap_prot(lpte) (((lpte)->pte & PTE_PROT_MASK) >> 56)
139 #define pmap_wired(lpte) ((lpte)->pte & PTE_WIRED)
141 #define pmap_clear_accessed(lpte) (lpte)->pte &= ~PTE_ACCESSED
142 #define pmap_clear_dirty(lpte) (lpte)->pte &= ~PTE_DIRTY
143 #define pmap_clear_present(lpte) (lpte)->pte &= ~PTE_PRESENT
144 #define pmap_clear_wired(lpte) (lpte)->pte &= ~PTE_WIRED
146 #define pmap_set_wired(lpte) (lpte)->pte |= PTE_WIRED
149 * The VHPT bucket head structure.
158 * Statically allocated kernel pmap
160 struct pmap kernel_pmap_store;
162 vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */
163 vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */
166 * Kernel virtual memory management.
169 struct ia64_lpte ***ia64_kptdir;
170 #define KPTE_DIR0_INDEX(va) \
171 (((va) >> (3*PAGE_SHIFT-8)) & ((1<<(PAGE_SHIFT-3))-1))
172 #define KPTE_DIR1_INDEX(va) \
173 (((va) >> (2*PAGE_SHIFT-5)) & ((1<<(PAGE_SHIFT-3))-1))
174 #define KPTE_PTE_INDEX(va) \
175 (((va) >> PAGE_SHIFT) & ((1<<(PAGE_SHIFT-5))-1))
176 #define NKPTEPG (PAGE_SIZE / sizeof(struct ia64_lpte))
178 vm_offset_t kernel_vm_end;
180 /* Values for ptc.e. XXX values for SKI. */
181 static uint64_t pmap_ptc_e_base = 0x100000000;
182 static uint64_t pmap_ptc_e_count1 = 3;
183 static uint64_t pmap_ptc_e_count2 = 2;
184 static uint64_t pmap_ptc_e_stride1 = 0x2000;
185 static uint64_t pmap_ptc_e_stride2 = 0x100000000;
186 struct mtx pmap_ptcmutex;
189 * Data for the RID allocator
191 static int pmap_ridcount;
192 static int pmap_rididx;
193 static int pmap_ridmapsz;
194 static int pmap_ridmax;
195 static uint64_t *pmap_ridmap;
196 struct mtx pmap_ridmutex;
199 * Data for the pv entry allocation mechanism
201 static uma_zone_t pvzone;
202 static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0;
205 * Data for allocating PTEs for user processes.
207 static uma_zone_t ptezone;
210 * Virtual Hash Page Table (VHPT) data.
212 /* SYSCTL_DECL(_machdep); */
213 SYSCTL_NODE(_machdep, OID_AUTO, vhpt, CTLFLAG_RD, 0, "");
215 struct ia64_bucket *pmap_vhpt_bucket;
217 int pmap_vhpt_nbuckets;
218 SYSCTL_INT(_machdep_vhpt, OID_AUTO, nbuckets, CTLFLAG_RD,
219 &pmap_vhpt_nbuckets, 0, "");
221 uint64_t pmap_vhpt_base[MAXCPU];
223 int pmap_vhpt_log2size = 0;
224 TUNABLE_INT("machdep.vhpt.log2size", &pmap_vhpt_log2size);
225 SYSCTL_INT(_machdep_vhpt, OID_AUTO, log2size, CTLFLAG_RD,
226 &pmap_vhpt_log2size, 0, "");
228 static int pmap_vhpt_inserts;
229 SYSCTL_INT(_machdep_vhpt, OID_AUTO, inserts, CTLFLAG_RD,
230 &pmap_vhpt_inserts, 0, "");
232 static int pmap_vhpt_population(SYSCTL_HANDLER_ARGS);
233 SYSCTL_PROC(_machdep_vhpt, OID_AUTO, population, CTLTYPE_INT | CTLFLAG_RD,
234 NULL, 0, pmap_vhpt_population, "I", "");
236 static struct ia64_lpte *pmap_find_vhpt(vm_offset_t va);
238 static PMAP_INLINE void free_pv_entry(pv_entry_t pv);
239 static pv_entry_t get_pv_entry(pmap_t locked_pmap);
241 static void pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
242 vm_page_t m, vm_prot_t prot);
243 static pmap_t pmap_install(pmap_t);
244 static void pmap_invalidate_all(pmap_t pmap);
245 static int pmap_remove_pte(pmap_t pmap, struct ia64_lpte *pte,
246 vm_offset_t va, pv_entry_t pv, int freepte);
247 static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,
251 pmap_steal_memory(vm_size_t size)
256 size = round_page(size);
258 bank_size = phys_avail[1] - phys_avail[0];
259 while (size > bank_size) {
261 for (i = 0; phys_avail[i+2]; i+= 2) {
262 phys_avail[i] = phys_avail[i+2];
263 phys_avail[i+1] = phys_avail[i+3];
268 panic("pmap_steal_memory: out of memory");
269 bank_size = phys_avail[1] - phys_avail[0];
273 phys_avail[0] += size;
275 va = IA64_PHYS_TO_RR7(pa);
276 bzero((caddr_t) va, size);
281 * Bootstrap the system enough to run with virtual memory.
286 struct ia64_pal_result res;
287 struct ia64_lpte *pte;
288 vm_offset_t base, limit;
290 int i, j, count, ridbits;
293 * Query the PAL Code to find the loop parameters for the
296 res = ia64_call_pal_static(PAL_PTCE_INFO, 0, 0, 0);
297 if (res.pal_status != 0)
298 panic("Can't configure ptc.e parameters");
299 pmap_ptc_e_base = res.pal_result[0];
300 pmap_ptc_e_count1 = res.pal_result[1] >> 32;
301 pmap_ptc_e_count2 = res.pal_result[1] & ((1L<<32) - 1);
302 pmap_ptc_e_stride1 = res.pal_result[2] >> 32;
303 pmap_ptc_e_stride2 = res.pal_result[2] & ((1L<<32) - 1);
305 printf("ptc.e base=0x%lx, count1=%ld, count2=%ld, "
306 "stride1=0x%lx, stride2=0x%lx\n",
312 mtx_init(&pmap_ptcmutex, "Global PTC lock", NULL, MTX_SPIN);
315 * Setup RIDs. RIDs 0..7 are reserved for the kernel.
317 * We currently need at least 19 bits in the RID because PID_MAX
318 * can only be encoded in 17 bits and we need RIDs for 5 regions
319 * per process. With PID_MAX equalling 99999 this means that we
320 * need to be able to encode 499995 (=5*PID_MAX).
321 * The Itanium processor only has 18 bits and the architected
322 * minimum is exactly that. So, we cannot use a PID based scheme
323 * in those cases. Enter pmap_ridmap...
324 * We should avoid the map when running on a processor that has
325 * implemented enough bits. This means that we should pass the
326 * process/thread ID to pmap. This we currently don't do, so we
327 * use the map anyway. However, we don't want to allocate a map
328 * that is large enough to cover the range dictated by the number
329 * of bits in the RID, because that may result in a RID map of
330 * 2MB in size for a 24-bit RID. A 64KB map is enough.
331 * The bottomline: we create a 32KB map when the processor only
332 * implements 18 bits (or when we can't figure it out). Otherwise
333 * we create a 64KB map.
335 res = ia64_call_pal_static(PAL_VM_SUMMARY, 0, 0, 0);
336 if (res.pal_status != 0) {
338 printf("Can't read VM Summary - assuming 18 Region ID bits\n");
339 ridbits = 18; /* guaranteed minimum */
341 ridbits = (res.pal_result[1] >> 8) & 0xff;
343 printf("Processor supports %d Region ID bits\n",
349 pmap_ridmax = (1 << ridbits);
350 pmap_ridmapsz = pmap_ridmax / 64;
351 pmap_ridmap = (uint64_t *)pmap_steal_memory(pmap_ridmax / 8);
352 pmap_ridmap[0] |= 0xff;
355 mtx_init(&pmap_ridmutex, "RID allocator lock", NULL, MTX_DEF);
358 * Allocate some memory for initial kernel 'page tables'.
360 ia64_kptdir = (void *)pmap_steal_memory(PAGE_SIZE);
362 kernel_vm_end = VM_MIN_KERNEL_ADDRESS - VM_GATEWAY_SIZE;
364 for (i = 0; phys_avail[i+2]; i+= 2)
369 * Figure out a useful size for the VHPT, based on the size of
370 * physical memory and try to locate a region which is large
371 * enough to contain the VHPT (which must be a power of two in
372 * size and aligned to a natural boundary).
373 * We silently bump up the VHPT size to the minimum size if the
374 * user has set the tunable too small. Likewise, the VHPT size
375 * is silently capped to the maximum allowed.
377 TUNABLE_INT_FETCH("machdep.vhpt.log2size", &pmap_vhpt_log2size);
378 if (pmap_vhpt_log2size == 0) {
379 pmap_vhpt_log2size = 15;
380 size = 1UL << pmap_vhpt_log2size;
381 while (size < Maxmem * 32) {
382 pmap_vhpt_log2size++;
385 } else if (pmap_vhpt_log2size < 15)
386 pmap_vhpt_log2size = 15;
387 if (pmap_vhpt_log2size > 61)
388 pmap_vhpt_log2size = 61;
390 pmap_vhpt_base[0] = 0;
392 size = 1UL << pmap_vhpt_log2size;
393 while (pmap_vhpt_base[0] == 0) {
395 printf("Trying VHPT size 0x%lx\n", size);
396 for (i = 0; i < count; i += 2) {
397 base = (phys_avail[i] + size - 1) & ~(size - 1);
398 limit = base + MAXCPU * size;
399 if (limit <= phys_avail[i+1])
401 * VHPT can fit in this region
405 if (!phys_avail[i]) {
406 /* Can't fit, try next smaller size. */
407 pmap_vhpt_log2size--;
410 pmap_vhpt_base[0] = IA64_PHYS_TO_RR7(base);
412 if (pmap_vhpt_log2size < 15)
413 panic("Can't find space for VHPT");
416 printf("Putting VHPT at 0x%lx\n", base);
418 if (base != phys_avail[i]) {
419 /* Split this region. */
421 printf("Splitting [%p-%p]\n", (void *)phys_avail[i],
422 (void *)phys_avail[i+1]);
423 for (j = count; j > i; j -= 2) {
424 phys_avail[j] = phys_avail[j-2];
425 phys_avail[j+1] = phys_avail[j-2+1];
427 phys_avail[i+1] = base;
428 phys_avail[i+2] = limit;
430 phys_avail[i] = limit;
432 pmap_vhpt_nbuckets = size / sizeof(struct ia64_lpte);
434 pmap_vhpt_bucket = (void *)pmap_steal_memory(pmap_vhpt_nbuckets *
435 sizeof(struct ia64_bucket));
436 pte = (struct ia64_lpte *)pmap_vhpt_base[0];
437 for (i = 0; i < pmap_vhpt_nbuckets; i++) {
440 pte[i].tag = 1UL << 63; /* Invalid tag */
441 pte[i].chain = (uintptr_t)(pmap_vhpt_bucket + i);
442 /* Stolen memory is zeroed! */
443 mtx_init(&pmap_vhpt_bucket[i].mutex, "VHPT bucket lock", NULL,
447 for (i = 1; i < MAXCPU; i++) {
448 pmap_vhpt_base[i] = pmap_vhpt_base[i - 1] + size;
449 bcopy((void *)pmap_vhpt_base[i - 1], (void *)pmap_vhpt_base[i],
453 __asm __volatile("mov cr.pta=%0;; srlz.i;;" ::
454 "r" (pmap_vhpt_base[0] + (1<<8) + (pmap_vhpt_log2size<<2) + 1));
456 virtual_avail = VM_MIN_KERNEL_ADDRESS;
457 virtual_end = VM_MAX_KERNEL_ADDRESS;
460 * Initialize the kernel pmap (which is statically allocated).
462 PMAP_LOCK_INIT(kernel_pmap);
463 for (i = 0; i < 5; i++)
464 kernel_pmap->pm_rid[i] = 0;
465 kernel_pmap->pm_active = 1;
466 TAILQ_INIT(&kernel_pmap->pm_pvlist);
467 PCPU_SET(current_pmap, kernel_pmap);
470 * Region 5 is mapped via the vhpt.
472 ia64_set_rr(IA64_RR_BASE(5),
473 (5 << 8) | (PAGE_SHIFT << 2) | 1);
476 * Region 6 is direct mapped UC and region 7 is direct mapped
477 * WC. The details of this is controlled by the Alt {I,D}TLB
478 * handlers. Here we just make sure that they have the largest
479 * possible page size to minimise TLB usage.
481 ia64_set_rr(IA64_RR_BASE(6), (6 << 8) | (IA64_ID_PAGE_SHIFT << 2));
482 ia64_set_rr(IA64_RR_BASE(7), (7 << 8) | (IA64_ID_PAGE_SHIFT << 2));
485 * Clear out any random TLB entries left over from booting.
487 pmap_invalidate_all(kernel_pmap);
493 pmap_vhpt_population(SYSCTL_HANDLER_ARGS)
498 for (i = 0; i < pmap_vhpt_nbuckets; i++)
499 count += pmap_vhpt_bucket[i].length;
501 error = SYSCTL_OUT(req, &count, sizeof(count));
506 * Initialize a vm_page's machine-dependent fields.
509 pmap_page_init(vm_page_t m)
512 TAILQ_INIT(&m->md.pv_list);
513 m->md.pv_list_count = 0;
517 * Initialize the pmap module.
518 * Called by vm_init, to initialize any structures that the pmap
519 * system needs to map virtual memory.
524 int shpgperproc = PMAP_SHPGPERPROC;
527 * Initialize the address space (zone) for the pv entries. Set a
528 * high water mark so that the system can recover from excessive
529 * numbers of pv entries.
531 pvzone = uma_zcreate("PV ENTRY", sizeof(struct pv_entry), NULL, NULL,
532 NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE);
533 TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc);
534 pv_entry_max = shpgperproc * maxproc + cnt.v_page_count;
535 TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max);
536 pv_entry_high_water = 9 * (pv_entry_max / 10);
538 ptezone = uma_zcreate("PT ENTRY", sizeof (struct ia64_lpte),
539 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM|UMA_ZONE_NOFREE);
543 /***************************************************
544 * Manipulate TLBs for a pmap
545 ***************************************************/
549 pmap_invalidate_page_locally(void *arg)
551 vm_offset_t va = (uintptr_t)arg;
552 struct ia64_lpte *pte;
554 pte = (struct ia64_lpte *)ia64_thash(va);
555 if (pte->tag == ia64_ttag(va))
556 pte->tag = 1UL << 63;
557 ia64_ptc_l(va, PAGE_SHIFT << 2);
562 pmap_invalidate_page_1(void *arg)
568 oldpmap = pmap_install(args[0]);
569 pmap_invalidate_page_locally(args[1]);
570 pmap_install(oldpmap);
576 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
579 KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)),
580 ("invalidating TLB for non-current pmap"));
586 args[1] = (void *)va;
587 smp_rendezvous(NULL, pmap_invalidate_page_1, NULL, args);
590 pmap_invalidate_page_locally((void *)va);
595 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
597 struct ia64_lpte *pte;
600 KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)),
601 ("invalidating TLB for non-current pmap"));
603 vhpt_ofs = ia64_thash(va) - pmap_vhpt_base[PCPU_GET(cpuid)];
605 for (i = 0; i < MAXCPU; i++) {
606 pte = (struct ia64_lpte *)(pmap_vhpt_base[i] + vhpt_ofs);
607 if (pte->tag == ia64_ttag(va))
608 pte->tag = 1UL << 63;
611 mtx_lock_spin(&pmap_ptcmutex);
612 ia64_ptc_ga(va, PAGE_SHIFT << 2);
613 mtx_unlock_spin(&pmap_ptcmutex);
617 pmap_invalidate_all_1(void *arg)
623 addr = pmap_ptc_e_base;
624 for (i = 0; i < pmap_ptc_e_count1; i++) {
625 for (j = 0; j < pmap_ptc_e_count2; j++) {
627 addr += pmap_ptc_e_stride2;
629 addr += pmap_ptc_e_stride1;
635 pmap_invalidate_all(pmap_t pmap)
638 KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)),
639 ("invalidating TLB for non-current pmap"));
643 smp_rendezvous(NULL, pmap_invalidate_all_1, NULL, NULL);
646 pmap_invalidate_all_1(NULL);
650 pmap_allocate_rid(void)
655 mtx_lock(&pmap_ridmutex);
656 if (pmap_ridcount == pmap_ridmax)
657 panic("pmap_allocate_rid: All Region IDs used");
659 /* Find an index with a free bit. */
660 while ((bits = pmap_ridmap[pmap_rididx]) == ~0UL) {
662 if (pmap_rididx == pmap_ridmapsz)
665 rid = pmap_rididx * 64;
667 /* Find a free bit. */
674 pmap_ridmap[pmap_rididx] |= bit;
676 mtx_unlock(&pmap_ridmutex);
682 pmap_free_rid(uint32_t rid)
688 bit = ~(1UL << (rid & 63));
690 mtx_lock(&pmap_ridmutex);
691 pmap_ridmap[idx] &= bit;
693 mtx_unlock(&pmap_ridmutex);
696 /***************************************************
697 * Page table page management routines.....
698 ***************************************************/
701 pmap_pinit0(struct pmap *pmap)
703 /* kernel_pmap is the same as any other pmap. */
708 * Initialize a preallocated and zeroed pmap structure,
709 * such as one in a vmspace structure.
712 pmap_pinit(struct pmap *pmap)
716 PMAP_LOCK_INIT(pmap);
717 for (i = 0; i < 5; i++)
718 pmap->pm_rid[i] = pmap_allocate_rid();
720 TAILQ_INIT(&pmap->pm_pvlist);
721 bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
724 /***************************************************
725 * Pmap allocation/deallocation routines.
726 ***************************************************/
729 * Release any resources held by the given physical map.
730 * Called when a pmap initialized by pmap_pinit is being released.
731 * Should only be called if the map contains no valid mappings.
734 pmap_release(pmap_t pmap)
738 for (i = 0; i < 5; i++)
740 pmap_free_rid(pmap->pm_rid[i]);
741 PMAP_LOCK_DESTROY(pmap);
745 * grow the number of kernel page table entries, if needed
748 pmap_growkernel(vm_offset_t addr)
750 struct ia64_lpte **dir1;
751 struct ia64_lpte *leaf;
754 while (kernel_vm_end <= addr) {
755 if (nkpt == PAGE_SIZE/8 + PAGE_SIZE*PAGE_SIZE/64)
756 panic("%s: out of kernel address space", __func__);
758 dir1 = ia64_kptdir[KPTE_DIR0_INDEX(kernel_vm_end)];
760 nkpg = vm_page_alloc(NULL, nkpt++,
761 VM_ALLOC_NOOBJ|VM_ALLOC_INTERRUPT|VM_ALLOC_WIRED);
763 panic("%s: cannot add dir. page", __func__);
765 dir1 = (struct ia64_lpte **)
766 IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(nkpg));
767 bzero(dir1, PAGE_SIZE);
768 ia64_kptdir[KPTE_DIR0_INDEX(kernel_vm_end)] = dir1;
771 nkpg = vm_page_alloc(NULL, nkpt++,
772 VM_ALLOC_NOOBJ|VM_ALLOC_INTERRUPT|VM_ALLOC_WIRED);
774 panic("%s: cannot add PTE page", __func__);
776 leaf = (struct ia64_lpte *)
777 IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(nkpg));
778 bzero(leaf, PAGE_SIZE);
779 dir1[KPTE_DIR1_INDEX(kernel_vm_end)] = leaf;
781 kernel_vm_end += PAGE_SIZE * NKPTEPG;
785 /***************************************************
786 * page management routines.
787 ***************************************************/
790 * free the pv_entry back to the free list
792 static PMAP_INLINE void
793 free_pv_entry(pv_entry_t pv)
796 uma_zfree(pvzone, pv);
800 * get a new pv_entry, allocating a block from the system
804 get_pv_entry(pmap_t locked_pmap)
806 static const struct timeval printinterval = { 60, 0 };
807 static struct timeval lastprint;
808 struct vpgqueues *vpq;
809 struct ia64_lpte *pte;
810 pmap_t oldpmap, pmap;
811 pv_entry_t allocated_pv, next_pv, pv;
815 PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED);
816 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
817 allocated_pv = uma_zalloc(pvzone, M_NOWAIT);
818 if (allocated_pv != NULL) {
820 if (pv_entry_count > pv_entry_high_water)
823 return (allocated_pv);
827 * Reclaim pv entries: At first, destroy mappings to inactive
828 * pages. After that, if a pv entry is still needed, destroy
829 * mappings to active pages.
831 if (ratecheck(&lastprint, &printinterval))
832 printf("Approaching the limit on PV entries, "
833 "increase the vm.pmap.shpgperproc tunable.\n");
834 vpq = &vm_page_queues[PQ_INACTIVE];
836 TAILQ_FOREACH(m, &vpq->pl, pageq) {
837 if (m->hold_count || m->busy)
839 TAILQ_FOREACH_SAFE(pv, &m->md.pv_list, pv_list, next_pv) {
842 /* Avoid deadlock and lock recursion. */
843 if (pmap > locked_pmap)
845 else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap))
847 oldpmap = pmap_install(pmap);
848 pte = pmap_find_vhpt(va);
849 KASSERT(pte != NULL, ("pte"));
850 pmap_remove_pte(pmap, pte, va, pv, 1);
851 pmap_install(oldpmap);
852 if (pmap != locked_pmap)
854 if (allocated_pv == NULL)
860 if (allocated_pv == NULL) {
861 if (vpq == &vm_page_queues[PQ_INACTIVE]) {
862 vpq = &vm_page_queues[PQ_ACTIVE];
865 panic("get_pv_entry: increase the vm.pmap.shpgperproc tunable");
867 return (allocated_pv);
871 * Conditionally create a pv entry.
874 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
878 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
879 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
880 if (pv_entry_count < pv_entry_high_water &&
881 (pv = uma_zalloc(pvzone, M_NOWAIT)) != NULL) {
885 TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist);
886 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
887 m->md.pv_list_count++;
894 * Add an ia64_lpte to the VHPT.
897 pmap_enter_vhpt(struct ia64_lpte *pte, vm_offset_t va)
899 struct ia64_bucket *bckt;
900 struct ia64_lpte *vhpte;
903 /* Can fault, so get it out of the way. */
904 pte_pa = ia64_tpa((vm_offset_t)pte);
906 vhpte = (struct ia64_lpte *)ia64_thash(va);
907 bckt = (struct ia64_bucket *)vhpte->chain;
909 mtx_lock_spin(&bckt->mutex);
910 pte->chain = bckt->chain;
912 bckt->chain = pte_pa;
916 mtx_unlock_spin(&bckt->mutex);
920 * Remove the ia64_lpte matching va from the VHPT. Return zero if it
921 * worked or an appropriate error code otherwise.
924 pmap_remove_vhpt(vm_offset_t va)
926 struct ia64_bucket *bckt;
927 struct ia64_lpte *pte;
928 struct ia64_lpte *lpte;
929 struct ia64_lpte *vhpte;
933 vhpte = (struct ia64_lpte *)ia64_thash(va);
934 bckt = (struct ia64_bucket *)vhpte->chain;
937 mtx_lock_spin(&bckt->mutex);
939 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
940 while (chain != 0 && pte->tag != tag) {
943 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
946 mtx_unlock_spin(&bckt->mutex);
950 /* Snip this pv_entry out of the collision chain. */
952 bckt->chain = pte->chain;
954 lpte->chain = pte->chain;
958 mtx_unlock_spin(&bckt->mutex);
963 * Find the ia64_lpte for the given va, if any.
965 static struct ia64_lpte *
966 pmap_find_vhpt(vm_offset_t va)
968 struct ia64_bucket *bckt;
969 struct ia64_lpte *pte;
973 pte = (struct ia64_lpte *)ia64_thash(va);
974 bckt = (struct ia64_bucket *)pte->chain;
976 mtx_lock_spin(&bckt->mutex);
978 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
979 while (chain != 0 && pte->tag != tag) {
981 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
983 mtx_unlock_spin(&bckt->mutex);
984 return ((chain != 0) ? pte : NULL);
988 * Remove an entry from the list of managed mappings.
991 pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va, pv_entry_t pv)
994 if (m->md.pv_list_count < pmap->pm_stats.resident_count) {
995 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
996 if (pmap == pv->pv_pmap && va == pv->pv_va)
1000 TAILQ_FOREACH(pv, &pmap->pm_pvlist, pv_plist) {
1001 if (va == pv->pv_va)
1008 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
1009 m->md.pv_list_count--;
1010 if (TAILQ_FIRST(&m->md.pv_list) == NULL)
1011 vm_page_flag_clear(m, PG_WRITEABLE);
1013 TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist);
1022 * Create a pv entry for page at pa for
1026 pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
1030 pv = get_pv_entry(pmap);
1034 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1035 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1036 TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist);
1037 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
1038 m->md.pv_list_count++;
1042 * Routine: pmap_extract
1044 * Extract the physical page address associated
1045 * with the given map/virtual_address pair.
1048 pmap_extract(pmap_t pmap, vm_offset_t va)
1050 struct ia64_lpte *pte;
1056 oldpmap = pmap_install(pmap);
1057 pte = pmap_find_vhpt(va);
1058 if (pte != NULL && pmap_present(pte))
1060 pmap_install(oldpmap);
1066 * Routine: pmap_extract_and_hold
1068 * Atomically extract and hold the physical page
1069 * with the given pmap and virtual address pair
1070 * if that mapping permits the given protection.
1073 pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
1075 struct ia64_lpte *pte;
1080 vm_page_lock_queues();
1082 oldpmap = pmap_install(pmap);
1083 pte = pmap_find_vhpt(va);
1084 if (pte != NULL && pmap_present(pte) &&
1085 (pmap_prot(pte) & prot) == prot) {
1086 m = PHYS_TO_VM_PAGE(pmap_ppn(pte));
1089 vm_page_unlock_queues();
1090 pmap_install(oldpmap);
1095 /***************************************************
1096 * Low level mapping routines.....
1097 ***************************************************/
1100 * Find the kernel lpte for mapping the given virtual address, which
1101 * must be in the part of region 5 which we can cover with our kernel
1104 static struct ia64_lpte *
1105 pmap_find_kpte(vm_offset_t va)
1107 struct ia64_lpte **dir1;
1108 struct ia64_lpte *leaf;
1110 KASSERT((va >> 61) == 5,
1111 ("kernel mapping 0x%lx not in region 5", va));
1112 KASSERT(va < kernel_vm_end,
1113 ("kernel mapping 0x%lx out of range", va));
1115 dir1 = ia64_kptdir[KPTE_DIR0_INDEX(va)];
1116 leaf = dir1[KPTE_DIR1_INDEX(va)];
1117 return (&leaf[KPTE_PTE_INDEX(va)]);
1121 * Find a pte suitable for mapping a user-space address. If one exists
1122 * in the VHPT, that one will be returned, otherwise a new pte is
1125 static struct ia64_lpte *
1126 pmap_find_pte(vm_offset_t va)
1128 struct ia64_lpte *pte;
1130 if (va >= VM_MAXUSER_ADDRESS)
1131 return pmap_find_kpte(va);
1133 pte = pmap_find_vhpt(va);
1135 pte = uma_zalloc(ptezone, M_NOWAIT | M_ZERO);
1136 pte->tag = 1UL << 63;
1142 * Free a pte which is now unused. This simply returns it to the zone
1143 * allocator if it is a user mapping. For kernel mappings, clear the
1144 * valid bit to make it clear that the mapping is not currently used.
1147 pmap_free_pte(struct ia64_lpte *pte, vm_offset_t va)
1149 if (va < VM_MAXUSER_ADDRESS)
1150 uma_zfree(ptezone, pte);
1152 pmap_clear_present(pte);
1155 static PMAP_INLINE void
1156 pmap_pte_prot(pmap_t pm, struct ia64_lpte *pte, vm_prot_t prot)
1158 static long prot2ar[4] = {
1159 PTE_AR_R, /* VM_PROT_NONE */
1160 PTE_AR_RW, /* VM_PROT_WRITE */
1161 PTE_AR_RX|PTE_ED, /* VM_PROT_EXECUTE */
1162 PTE_AR_RWX|PTE_ED /* VM_PROT_WRITE|VM_PROT_EXECUTE */
1165 pte->pte &= ~(PTE_PROT_MASK | PTE_PL_MASK | PTE_AR_MASK | PTE_ED);
1166 pte->pte |= (uint64_t)(prot & VM_PROT_ALL) << 56;
1167 pte->pte |= (prot == VM_PROT_NONE || pm == kernel_pmap)
1168 ? PTE_PL_KERN : PTE_PL_USER;
1169 pte->pte |= prot2ar[(prot & VM_PROT_ALL) >> 1];
1173 * Set a pte to contain a valid mapping and enter it in the VHPT. If
1174 * the pte was orginally valid, then its assumed to already be in the
1176 * This functions does not set the protection bits. It's expected
1177 * that those have been set correctly prior to calling this function.
1180 pmap_set_pte(struct ia64_lpte *pte, vm_offset_t va, vm_offset_t pa,
1181 boolean_t wired, boolean_t managed)
1184 pte->pte &= PTE_PROT_MASK | PTE_PL_MASK | PTE_AR_MASK | PTE_ED;
1185 pte->pte |= PTE_PRESENT | PTE_MA_WB;
1186 pte->pte |= (managed) ? PTE_MANAGED : (PTE_DIRTY | PTE_ACCESSED);
1187 pte->pte |= (wired) ? PTE_WIRED : 0;
1188 pte->pte |= pa & PTE_PPN_MASK;
1190 pte->itir = PAGE_SHIFT << 2;
1192 pte->tag = ia64_ttag(va);
1196 * Remove the (possibly managed) mapping represented by pte from the
1200 pmap_remove_pte(pmap_t pmap, struct ia64_lpte *pte, vm_offset_t va,
1201 pv_entry_t pv, int freepte)
1206 KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)),
1207 ("removing pte for non-current pmap"));
1210 * First remove from the VHPT.
1212 error = pmap_remove_vhpt(va);
1216 pmap_invalidate_page(pmap, va);
1218 if (pmap_wired(pte))
1219 pmap->pm_stats.wired_count -= 1;
1221 pmap->pm_stats.resident_count -= 1;
1222 if (pmap_managed(pte)) {
1223 m = PHYS_TO_VM_PAGE(pmap_ppn(pte));
1224 if (pmap_dirty(pte))
1226 if (pmap_accessed(pte))
1227 vm_page_flag_set(m, PG_REFERENCED);
1229 error = pmap_remove_entry(pmap, m, va, pv);
1232 pmap_free_pte(pte, va);
1238 * Extract the physical page address associated with a kernel
1242 pmap_kextract(vm_offset_t va)
1244 struct ia64_lpte *pte;
1247 KASSERT(va >= IA64_RR_BASE(5), ("Must be kernel VA"));
1249 /* Regions 6 and 7 are direct mapped. */
1250 if (va >= IA64_RR_BASE(6))
1251 return (IA64_RR_MASK(va));
1253 /* EPC gateway page? */
1254 gwpage = (vm_offset_t)ia64_get_k5();
1255 if (va >= gwpage && va < gwpage + VM_GATEWAY_SIZE)
1256 return (IA64_RR_MASK((vm_offset_t)ia64_gateway_page));
1258 /* Bail out if the virtual address is beyond our limits. */
1259 if (va >= kernel_vm_end)
1262 pte = pmap_find_kpte(va);
1263 if (!pmap_present(pte))
1265 return (pmap_ppn(pte) | (va & PAGE_MASK));
1269 * Add a list of wired pages to the kva this routine is only used for
1270 * temporary kernel mappings that do not need to have page modification
1271 * or references recorded. Note that old mappings are simply written
1272 * over. The page is effectively wired, but it's customary to not have
1273 * the PTE reflect that, nor update statistics.
1276 pmap_qenter(vm_offset_t va, vm_page_t *m, int count)
1278 struct ia64_lpte *pte;
1281 for (i = 0; i < count; i++) {
1282 pte = pmap_find_kpte(va);
1283 if (pmap_present(pte))
1284 pmap_invalidate_page(kernel_pmap, va);
1286 pmap_enter_vhpt(pte, va);
1287 pmap_pte_prot(kernel_pmap, pte, VM_PROT_ALL);
1288 pmap_set_pte(pte, va, VM_PAGE_TO_PHYS(m[i]), FALSE, FALSE);
1294 * this routine jerks page mappings from the
1295 * kernel -- it is meant only for temporary mappings.
1298 pmap_qremove(vm_offset_t va, int count)
1300 struct ia64_lpte *pte;
1303 for (i = 0; i < count; i++) {
1304 pte = pmap_find_kpte(va);
1305 if (pmap_present(pte)) {
1306 pmap_remove_vhpt(va);
1307 pmap_invalidate_page(kernel_pmap, va);
1308 pmap_clear_present(pte);
1315 * Add a wired page to the kva. As for pmap_qenter(), it's customary
1316 * to not have the PTE reflect that, nor update statistics.
1319 pmap_kenter(vm_offset_t va, vm_offset_t pa)
1321 struct ia64_lpte *pte;
1323 pte = pmap_find_kpte(va);
1324 if (pmap_present(pte))
1325 pmap_invalidate_page(kernel_pmap, va);
1327 pmap_enter_vhpt(pte, va);
1328 pmap_pte_prot(kernel_pmap, pte, VM_PROT_ALL);
1329 pmap_set_pte(pte, va, pa, FALSE, FALSE);
1333 * Remove a page from the kva
1336 pmap_kremove(vm_offset_t va)
1338 struct ia64_lpte *pte;
1340 pte = pmap_find_kpte(va);
1341 if (pmap_present(pte)) {
1342 pmap_remove_vhpt(va);
1343 pmap_invalidate_page(kernel_pmap, va);
1344 pmap_clear_present(pte);
1349 * Used to map a range of physical addresses into kernel
1350 * virtual address space.
1352 * The value passed in '*virt' is a suggested virtual address for
1353 * the mapping. Architectures which can support a direct-mapped
1354 * physical to virtual region can return the appropriate address
1355 * within that region, leaving '*virt' unchanged. Other
1356 * architectures should map the pages starting at '*virt' and
1357 * update '*virt' with the first usable address after the mapped
1361 pmap_map(vm_offset_t *virt, vm_offset_t start, vm_offset_t end, int prot)
1363 return IA64_PHYS_TO_RR7(start);
1367 * Remove a single page from a process address space
1370 pmap_remove_page(pmap_t pmap, vm_offset_t va)
1372 struct ia64_lpte *pte;
1374 KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)),
1375 ("removing page for non-current pmap"));
1377 pte = pmap_find_vhpt(va);
1379 pmap_remove_pte(pmap, pte, va, 0, 1);
1384 * Remove the given range of addresses from the specified map.
1386 * It is assumed that the start and end are properly
1387 * rounded to the page size.
1390 pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
1395 struct ia64_lpte *pte;
1397 if (pmap->pm_stats.resident_count == 0)
1400 vm_page_lock_queues();
1402 oldpmap = pmap_install(pmap);
1405 * special handling of removing one page. a very
1406 * common operation and easy to short circuit some
1409 if (sva + PAGE_SIZE == eva) {
1410 pmap_remove_page(pmap, sva);
1414 if (pmap->pm_stats.resident_count < ((eva - sva) >> PAGE_SHIFT)) {
1415 TAILQ_FOREACH_SAFE(pv, &pmap->pm_pvlist, pv_plist, npv) {
1417 if (va >= sva && va < eva) {
1418 pte = pmap_find_vhpt(va);
1419 KASSERT(pte != NULL, ("pte"));
1420 pmap_remove_pte(pmap, pte, va, pv, 1);
1424 for (va = sva; va < eva; va += PAGE_SIZE) {
1425 pte = pmap_find_vhpt(va);
1427 pmap_remove_pte(pmap, pte, va, 0, 1);
1432 vm_page_unlock_queues();
1433 pmap_install(oldpmap);
1438 * Routine: pmap_remove_all
1440 * Removes this physical page from
1441 * all physical maps in which it resides.
1442 * Reflects back modify bits to the pager.
1445 * Original versions of this routine were very
1446 * inefficient because they iteratively called
1447 * pmap_remove (slow...)
1451 pmap_remove_all(vm_page_t m)
1456 #if defined(DIAGNOSTIC)
1458 * XXX this makes pmap_page_protect(NONE) illegal for non-managed
1461 if (m->flags & PG_FICTITIOUS) {
1462 panic("pmap_page_protect: illegal for unmanaged page, va: 0x%lx", VM_PAGE_TO_PHYS(m));
1465 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1466 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
1467 struct ia64_lpte *pte;
1468 pmap_t pmap = pv->pv_pmap;
1469 vm_offset_t va = pv->pv_va;
1472 oldpmap = pmap_install(pmap);
1473 pte = pmap_find_vhpt(va);
1474 KASSERT(pte != NULL, ("pte"));
1475 if (pmap_ppn(pte) != VM_PAGE_TO_PHYS(m))
1476 panic("pmap_remove_all: pv_table for %lx is inconsistent", VM_PAGE_TO_PHYS(m));
1477 pmap_remove_pte(pmap, pte, va, pv, 1);
1478 pmap_install(oldpmap);
1481 vm_page_flag_clear(m, PG_WRITEABLE);
1485 * Set the physical protection on the
1486 * specified range of this map as requested.
1489 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
1492 struct ia64_lpte *pte;
1494 if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
1495 pmap_remove(pmap, sva, eva);
1499 if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) ==
1500 (VM_PROT_WRITE|VM_PROT_EXECUTE))
1503 if ((sva & PAGE_MASK) || (eva & PAGE_MASK))
1504 panic("pmap_protect: unaligned addresses");
1506 vm_page_lock_queues();
1508 oldpmap = pmap_install(pmap);
1511 * If page is invalid, skip this page
1513 pte = pmap_find_vhpt(sva);
1519 if (pmap_prot(pte) != prot) {
1520 if (pmap_managed(pte)) {
1521 vm_offset_t pa = pmap_ppn(pte);
1522 vm_page_t m = PHYS_TO_VM_PAGE(pa);
1523 if (pmap_dirty(pte)) {
1525 pmap_clear_dirty(pte);
1527 if (pmap_accessed(pte)) {
1528 vm_page_flag_set(m, PG_REFERENCED);
1529 pmap_clear_accessed(pte);
1532 pmap_pte_prot(pmap, pte, prot);
1533 pmap_invalidate_page(pmap, sva);
1538 vm_page_unlock_queues();
1539 pmap_install(oldpmap);
1544 * Insert the given physical page (p) at
1545 * the specified virtual address (v) in the
1546 * target physical map with the protection requested.
1548 * If specified, the page will be wired down, meaning
1549 * that the related pte can not be reclaimed.
1551 * NB: This is the only routine which MAY NOT lazy-evaluate
1552 * or lose information. That is, this routine must actually
1553 * insert this page into the given map NOW.
1556 pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
1562 struct ia64_lpte origpte;
1563 struct ia64_lpte *pte;
1566 vm_page_lock_queues();
1568 oldpmap = pmap_install(pmap);
1572 if (va > VM_MAX_KERNEL_ADDRESS)
1573 panic("pmap_enter: toobig");
1577 * Find (or create) a pte for the given mapping.
1579 while ((pte = pmap_find_pte(va)) == NULL) {
1580 pmap_install(oldpmap);
1582 vm_page_unlock_queues();
1584 vm_page_lock_queues();
1586 oldpmap = pmap_install(pmap);
1589 if (!pmap_present(pte)) {
1591 pmap_enter_vhpt(pte, va);
1593 opa = pmap_ppn(pte);
1595 pa = VM_PAGE_TO_PHYS(m);
1598 * Mapping has not changed, must be protection or wiring change.
1602 * Wiring change, just update stats. We don't worry about
1603 * wiring PT pages as they remain resident as long as there
1604 * are valid mappings in them. Hence, if a user page is wired,
1605 * the PT page will be also.
1607 if (wired && !pmap_wired(&origpte))
1608 pmap->pm_stats.wired_count++;
1609 else if (!wired && pmap_wired(&origpte))
1610 pmap->pm_stats.wired_count--;
1612 managed = (pmap_managed(&origpte)) ? TRUE : FALSE;
1615 * We might be turning off write access to the page,
1616 * so we go ahead and sense modify status.
1618 if (managed && pmap_dirty(&origpte))
1621 pmap_invalidate_page(pmap, va);
1626 * Mapping has changed, invalidate old range and fall
1627 * through to handle validating new mapping.
1630 pmap_remove_pte(pmap, pte, va, 0, 0);
1631 pmap_enter_vhpt(pte, va);
1635 * Enter on the PV list if part of our managed memory.
1637 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) {
1638 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva,
1639 ("pmap_enter: managed mapping within the clean submap"));
1640 pmap_insert_entry(pmap, va, m);
1645 * Increment counters
1647 pmap->pm_stats.resident_count++;
1649 pmap->pm_stats.wired_count++;
1654 * Now validate mapping with desired protection/wiring. This
1655 * adds the pte to the VHPT if necessary.
1657 pmap_pte_prot(pmap, pte, prot);
1658 pmap_set_pte(pte, va, pa, wired, managed);
1660 if ((prot & VM_PROT_WRITE) != 0)
1661 vm_page_flag_set(m, PG_WRITEABLE);
1662 vm_page_unlock_queues();
1663 pmap_install(oldpmap);
1668 * Maps a sequence of resident pages belonging to the same object.
1669 * The sequence begins with the given page m_start. This page is
1670 * mapped at the given virtual address start. Each subsequent page is
1671 * mapped at a virtual address that is offset from start by the same
1672 * amount as the page is offset from m_start within the object. The
1673 * last page in the sequence is the page with the largest offset from
1674 * m_start that can be mapped at a virtual address less than the given
1675 * virtual address end. Not every virtual page between start and end
1676 * is mapped; only those for which a resident page exists with the
1677 * corresponding offset from m_start are mapped.
1680 pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,
1681 vm_page_t m_start, vm_prot_t prot)
1685 vm_pindex_t diff, psize;
1687 VM_OBJECT_LOCK_ASSERT(m_start->object, MA_OWNED);
1688 psize = atop(end - start);
1691 oldpmap = pmap_install(pmap);
1692 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
1693 pmap_enter_quick_locked(pmap, start + ptoa(diff), m, prot);
1694 m = TAILQ_NEXT(m, listq);
1696 pmap_install(oldpmap);
1701 * this code makes some *MAJOR* assumptions:
1702 * 1. Current pmap & pmap exists.
1705 * 4. No page table pages.
1706 * but is *MUCH* faster than pmap_enter...
1710 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
1715 oldpmap = pmap_install(pmap);
1716 pmap_enter_quick_locked(pmap, va, m, prot);
1717 pmap_install(oldpmap);
1722 pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
1725 struct ia64_lpte *pte;
1728 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva ||
1729 (m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0,
1730 ("pmap_enter_quick_locked: managed mapping within the clean submap"));
1731 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1732 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1734 if ((pte = pmap_find_pte(va)) == NULL)
1737 if (!pmap_present(pte)) {
1738 /* Enter on the PV list if the page is managed. */
1739 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) {
1740 if (!pmap_try_insert_pv_entry(pmap, va, m)) {
1741 pmap_free_pte(pte, va);
1748 /* Increment counters. */
1749 pmap->pm_stats.resident_count++;
1751 /* Initialise with R/O protection and enter into VHPT. */
1752 pmap_enter_vhpt(pte, va);
1753 pmap_pte_prot(pmap, pte,
1754 prot & (VM_PROT_READ | VM_PROT_EXECUTE));
1755 pmap_set_pte(pte, va, VM_PAGE_TO_PHYS(m), FALSE, managed);
1760 * pmap_object_init_pt preloads the ptes for a given object
1761 * into the specified pmap. This eliminates the blast of soft
1762 * faults on process startup and immediately after an mmap.
1765 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr,
1766 vm_object_t object, vm_pindex_t pindex,
1770 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1771 KASSERT(object->type == OBJT_DEVICE,
1772 ("pmap_object_init_pt: non-device object"));
1776 * Routine: pmap_change_wiring
1777 * Function: Change the wiring attribute for a map/virtual-address
1779 * In/out conditions:
1780 * The mapping must already exist in the pmap.
1783 pmap_change_wiring(pmap, va, wired)
1784 register pmap_t pmap;
1789 struct ia64_lpte *pte;
1792 oldpmap = pmap_install(pmap);
1794 pte = pmap_find_vhpt(va);
1795 KASSERT(pte != NULL, ("pte"));
1796 if (wired && !pmap_wired(pte)) {
1797 pmap->pm_stats.wired_count++;
1798 pmap_set_wired(pte);
1799 } else if (!wired && pmap_wired(pte)) {
1800 pmap->pm_stats.wired_count--;
1801 pmap_clear_wired(pte);
1804 pmap_install(oldpmap);
1811 * Copy the range specified by src_addr/len
1812 * from the source map to the range dst_addr/len
1813 * in the destination map.
1815 * This routine is only advisory and need not do anything.
1819 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
1820 vm_offset_t src_addr)
1826 * pmap_zero_page zeros the specified hardware page by
1827 * mapping it into virtual memory and using bzero to clear
1832 pmap_zero_page(vm_page_t m)
1834 vm_offset_t va = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m));
1835 bzero((caddr_t) va, PAGE_SIZE);
1840 * pmap_zero_page_area zeros the specified hardware page by
1841 * mapping it into virtual memory and using bzero to clear
1844 * off and size must reside within a single page.
1848 pmap_zero_page_area(vm_page_t m, int off, int size)
1850 vm_offset_t va = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m));
1851 bzero((char *)(caddr_t)va + off, size);
1856 * pmap_zero_page_idle zeros the specified hardware page by
1857 * mapping it into virtual memory and using bzero to clear
1858 * its contents. This is for the vm_idlezero process.
1862 pmap_zero_page_idle(vm_page_t m)
1864 vm_offset_t va = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m));
1865 bzero((caddr_t) va, PAGE_SIZE);
1870 * pmap_copy_page copies the specified (machine independent)
1871 * page by mapping the page into virtual memory and using
1872 * bcopy to copy the page, one machine dependent page at a
1876 pmap_copy_page(vm_page_t msrc, vm_page_t mdst)
1878 vm_offset_t src = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(msrc));
1879 vm_offset_t dst = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(mdst));
1880 bcopy((caddr_t) src, (caddr_t) dst, PAGE_SIZE);
1884 * Returns true if the pmap's pv is one of the first
1885 * 16 pvs linked to from this page. This count may
1886 * be changed upwards or downwards in the future; it
1887 * is only necessary that true be returned for a small
1888 * subset of pmaps for proper page aging.
1891 pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
1896 if (m->flags & PG_FICTITIOUS)
1900 * Not found, check current mappings returning immediately if found.
1902 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1903 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
1904 if (pv->pv_pmap == pmap) {
1915 * Remove all pages from specified address space
1916 * this aids process exit speeds. Also, this code
1917 * is special cased for current process only, but
1918 * can have the more generic (and slightly slower)
1919 * mode enabled. This is much faster than pmap_remove
1920 * in the case of running down an entire address space.
1923 pmap_remove_pages(pmap_t pmap)
1928 if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace)) {
1929 printf("warning: pmap_remove_pages called with non-current pmap\n");
1933 vm_page_lock_queues();
1935 oldpmap = pmap_install(pmap);
1937 for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) {
1938 struct ia64_lpte *pte;
1940 npv = TAILQ_NEXT(pv, pv_plist);
1942 pte = pmap_find_vhpt(pv->pv_va);
1943 KASSERT(pte != NULL, ("pte"));
1944 if (!pmap_wired(pte))
1945 pmap_remove_pte(pmap, pte, pv->pv_va, pv, 1);
1948 pmap_install(oldpmap);
1950 vm_page_unlock_queues();
1954 * pmap_ts_referenced:
1956 * Return a count of reference bits for a page, clearing those bits.
1957 * It is not necessary for every reference bit to be cleared, but it
1958 * is necessary that 0 only be returned when there are truly no
1959 * reference bits set.
1961 * XXX: The exact number of bits to check and clear is a matter that
1962 * should be tested and standardized at some point in the future for
1963 * optimal aging of shared pages.
1966 pmap_ts_referenced(vm_page_t m)
1968 struct ia64_lpte *pte;
1973 if (m->flags & PG_FICTITIOUS)
1976 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
1977 PMAP_LOCK(pv->pv_pmap);
1978 oldpmap = pmap_install(pv->pv_pmap);
1979 pte = pmap_find_vhpt(pv->pv_va);
1980 KASSERT(pte != NULL, ("pte"));
1981 if (pmap_accessed(pte)) {
1983 pmap_clear_accessed(pte);
1984 pmap_invalidate_page(pv->pv_pmap, pv->pv_va);
1986 pmap_install(oldpmap);
1987 PMAP_UNLOCK(pv->pv_pmap);
1996 * Return whether or not the specified physical page was modified
1997 * in any physical maps.
2000 pmap_is_modified(vm_page_t m)
2002 struct ia64_lpte *pte;
2008 if (m->flags & PG_FICTITIOUS)
2011 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2012 PMAP_LOCK(pv->pv_pmap);
2013 oldpmap = pmap_install(pv->pv_pmap);
2014 pte = pmap_find_vhpt(pv->pv_va);
2015 pmap_install(oldpmap);
2016 KASSERT(pte != NULL, ("pte"));
2017 rv = pmap_dirty(pte) ? TRUE : FALSE;
2018 PMAP_UNLOCK(pv->pv_pmap);
2027 * pmap_is_prefaultable:
2029 * Return whether or not the specified virtual address is elgible
2033 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)
2035 struct ia64_lpte *pte;
2037 pte = pmap_find_vhpt(addr);
2038 if (pte != NULL && pmap_present(pte))
2044 * Clear the modify bits on the specified physical page.
2047 pmap_clear_modify(vm_page_t m)
2049 struct ia64_lpte *pte;
2053 if (m->flags & PG_FICTITIOUS)
2056 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2057 PMAP_LOCK(pv->pv_pmap);
2058 oldpmap = pmap_install(pv->pv_pmap);
2059 pte = pmap_find_vhpt(pv->pv_va);
2060 KASSERT(pte != NULL, ("pte"));
2061 if (pmap_dirty(pte)) {
2062 pmap_clear_dirty(pte);
2063 pmap_invalidate_page(pv->pv_pmap, pv->pv_va);
2065 pmap_install(oldpmap);
2066 PMAP_UNLOCK(pv->pv_pmap);
2071 * pmap_clear_reference:
2073 * Clear the reference bit on the specified physical page.
2076 pmap_clear_reference(vm_page_t m)
2078 struct ia64_lpte *pte;
2082 if (m->flags & PG_FICTITIOUS)
2085 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2086 PMAP_LOCK(pv->pv_pmap);
2087 oldpmap = pmap_install(pv->pv_pmap);
2088 pte = pmap_find_vhpt(pv->pv_va);
2089 KASSERT(pte != NULL, ("pte"));
2090 if (pmap_accessed(pte)) {
2091 pmap_clear_accessed(pte);
2092 pmap_invalidate_page(pv->pv_pmap, pv->pv_va);
2094 pmap_install(oldpmap);
2095 PMAP_UNLOCK(pv->pv_pmap);
2100 * Clear the write and modified bits in each of the given page's mappings.
2103 pmap_remove_write(vm_page_t m)
2105 struct ia64_lpte *pte;
2106 pmap_t oldpmap, pmap;
2110 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
2111 if ((m->flags & PG_FICTITIOUS) != 0 ||
2112 (m->flags & PG_WRITEABLE) == 0)
2114 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2117 oldpmap = pmap_install(pmap);
2118 pte = pmap_find_vhpt(pv->pv_va);
2119 KASSERT(pte != NULL, ("pte"));
2120 prot = pmap_prot(pte);
2121 if ((prot & VM_PROT_WRITE) != 0) {
2122 if (pmap_dirty(pte)) {
2124 pmap_clear_dirty(pte);
2126 prot &= ~VM_PROT_WRITE;
2127 pmap_pte_prot(pmap, pte, prot);
2128 pmap_invalidate_page(pmap, pv->pv_va);
2130 pmap_install(oldpmap);
2133 vm_page_flag_clear(m, PG_WRITEABLE);
2137 * Map a set of physical memory pages into the kernel virtual
2138 * address space. Return a pointer to where it is mapped. This
2139 * routine is intended to be used for mapping device memory,
2143 pmap_mapdev(vm_offset_t pa, vm_size_t size)
2145 return (void*) IA64_PHYS_TO_RR6(pa);
2149 * 'Unmap' a range mapped by pmap_mapdev().
2152 pmap_unmapdev(vm_offset_t va, vm_size_t size)
2158 * perform the pmap work for mincore
2161 pmap_mincore(pmap_t pmap, vm_offset_t addr)
2164 struct ia64_lpte *pte, tpte;
2168 oldpmap = pmap_install(pmap);
2169 pte = pmap_find_vhpt(addr);
2174 pmap_install(oldpmap);
2180 if (pmap_present(pte)) {
2184 val = MINCORE_INCORE;
2185 if (!pmap_managed(pte))
2190 m = PHYS_TO_VM_PAGE(pa);
2195 if (pmap_dirty(pte))
2196 val |= MINCORE_MODIFIED|MINCORE_MODIFIED_OTHER;
2199 * Modified by someone
2201 vm_page_lock_queues();
2202 if (pmap_is_modified(m))
2203 val |= MINCORE_MODIFIED_OTHER;
2204 vm_page_unlock_queues();
2209 if (pmap_accessed(pte))
2210 val |= MINCORE_REFERENCED|MINCORE_REFERENCED_OTHER;
2213 * Referenced by someone
2215 vm_page_lock_queues();
2216 if (pmap_ts_referenced(m)) {
2217 val |= MINCORE_REFERENCED_OTHER;
2218 vm_page_flag_set(m, PG_REFERENCED);
2220 vm_page_unlock_queues();
2227 pmap_activate(struct thread *td)
2229 pmap_install(vmspace_pmap(td->td_proc->p_vmspace));
2233 pmap_switch(pmap_t pm)
2238 THREAD_LOCK_ASSERT(curthread, MA_OWNED);
2239 prevpm = PCPU_GET(current_pmap);
2243 atomic_clear_32(&prevpm->pm_active, PCPU_GET(cpumask));
2245 for (i = 0; i < 5; i++) {
2246 ia64_set_rr(IA64_RR_BASE(i),
2247 (i << 8)|(PAGE_SHIFT << 2)|1);
2250 for (i = 0; i < 5; i++) {
2251 ia64_set_rr(IA64_RR_BASE(i),
2252 (pm->pm_rid[i] << 8)|(PAGE_SHIFT << 2)|1);
2254 atomic_set_32(&pm->pm_active, PCPU_GET(cpumask));
2256 PCPU_SET(current_pmap, pm);
2257 __asm __volatile("srlz.d");
2262 pmap_install(pmap_t pm)
2269 prevpm = pmap_switch(pm);
2276 pmap_addr_hint(vm_object_t obj, vm_offset_t addr, vm_size_t size)
2282 #include "opt_ddb.h"
2286 #include <ddb/ddb.h>
2288 static const char* psnames[] = {
2289 "1B", "2B", "4B", "8B",
2290 "16B", "32B", "64B", "128B",
2291 "256B", "512B", "1K", "2K",
2292 "4K", "8K", "16K", "32K",
2293 "64K", "128K", "256K", "512K",
2294 "1M", "2M", "4M", "8M",
2295 "16M", "32M", "64M", "128M",
2296 "256M", "512M", "1G", "2G"
2302 struct ia64_pal_result res;
2310 static const char *manames[] = {
2311 "WB", "bad", "bad", "bad",
2312 "UC", "UCE", "WC", "NaT",
2315 res = ia64_call_pal_static(PAL_VM_SUMMARY, 0, 0, 0);
2316 if (res.pal_status != 0) {
2317 db_printf("Can't get VM summary\n");
2322 maxtr = (res.pal_result[0] >> 40) & 0xff;
2324 maxtr = (res.pal_result[0] >> 32) & 0xff;
2326 db_printf("V RID Virtual Page Physical Page PgSz ED AR PL D A MA P KEY\n");
2327 for (i = 0; i <= maxtr; i++) {
2328 bzero(&buf, sizeof(buf));
2329 res = ia64_call_pal_stacked_physical
2330 (PAL_VM_TR_READ, i, type, ia64_tpa((uint64_t) &buf));
2331 if (!(res.pal_result[0] & 1))
2332 buf.pte &= ~PTE_AR_MASK;
2333 if (!(res.pal_result[0] & 2))
2334 buf.pte &= ~PTE_PL_MASK;
2335 if (!(res.pal_result[0] & 4))
2336 pmap_clear_dirty(&buf);
2337 if (!(res.pal_result[0] & 8))
2338 buf.pte &= ~PTE_MA_MASK;
2339 db_printf("%d %06x %013lx %013lx %4s %d %d %d %d %d %-3s "
2340 "%d %06x\n", (int)buf.ifa & 1, buf.rr.rr_rid,
2341 buf.ifa >> 12, (buf.pte & PTE_PPN_MASK) >> 12,
2342 psnames[(buf.itir & ITIR_PS_MASK) >> 2],
2343 (buf.pte & PTE_ED) ? 1 : 0,
2344 (int)(buf.pte & PTE_AR_MASK) >> 9,
2345 (int)(buf.pte & PTE_PL_MASK) >> 7,
2346 (pmap_dirty(&buf)) ? 1 : 0,
2347 (pmap_accessed(&buf)) ? 1 : 0,
2348 manames[(buf.pte & PTE_MA_MASK) >> 2],
2349 (pmap_present(&buf)) ? 1 : 0,
2350 (int)((buf.itir & ITIR_KEY_MASK) >> 8));
2354 DB_COMMAND(itr, db_itr)
2359 DB_COMMAND(dtr, db_dtr)
2364 DB_COMMAND(rr, db_rr)
2370 printf("RR RID PgSz VE\n");
2371 for (i = 0; i < 8; i++) {
2372 __asm __volatile ("mov %0=rr[%1]"
2374 : "r"(IA64_RR_BASE(i)));
2375 *(uint64_t *) &rr = t;
2376 printf("%d %06x %4s %d\n",
2377 i, rr.rr_rid, psnames[rr.rr_ps], rr.rr_ve);
2381 DB_COMMAND(thash, db_thash)
2386 db_printf("%p\n", (void *) ia64_thash(addr));
2389 DB_COMMAND(ttag, db_ttag)
2394 db_printf("0x%lx\n", ia64_ttag(addr));
2397 DB_COMMAND(kpte, db_kpte)
2399 struct ia64_lpte *pte;
2402 db_printf("usage: kpte <kva>\n");
2405 if (addr < VM_MIN_KERNEL_ADDRESS) {
2406 db_printf("kpte: error: invalid <kva>\n");
2409 pte = pmap_find_kpte(addr);
2410 db_printf("kpte at %p:\n", pte);
2411 db_printf(" pte =%016lx\n", pte->pte);
2412 db_printf(" itir =%016lx\n", pte->itir);
2413 db_printf(" tag =%016lx\n", pte->tag);
2414 db_printf(" chain=%016lx\n", pte->chain);