2 * Copyright (c) 1991 Regents of the University of California.
4 * Copyright (c) 1994 John S. Dyson
6 * Copyright (c) 1994 David Greenman
8 * Copyright (c) 1998,2000 Doug Rabson
11 * This code is derived from software contributed to Berkeley by
12 * the Systems Programming Group of the University of Utah Computer
13 * Science Department and William Jolitz of UUNET Technologies Inc.
15 * Redistribution and use in source and binary forms, with or without
16 * modification, are permitted provided that the following conditions
18 * 1. Redistributions of source code must retain the above copyright
19 * notice, this list of conditions and the following disclaimer.
20 * 2. Redistributions in binary form must reproduce the above copyright
21 * notice, this list of conditions and the following disclaimer in the
22 * documentation and/or other materials provided with the distribution.
23 * 3. All advertising materials mentioning features or use of this software
24 * must display the following acknowledgement:
25 * This product includes software developed by the University of
26 * California, Berkeley and its contributors.
27 * 4. Neither the name of the University nor the names of its contributors
28 * may be used to endorse or promote products derived from this software
29 * without specific prior written permission.
31 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
32 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
33 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
34 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
35 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
36 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
37 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
38 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
39 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
40 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
43 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91
44 * from: i386 Id: pmap.c,v 1.193 1998/04/19 15:22:48 bde Exp
45 * with some ideas from NetBSD's alpha pmap
48 #include <sys/cdefs.h>
49 __FBSDID("$FreeBSD$");
51 #include <sys/param.h>
52 #include <sys/kernel.h>
55 #include <sys/mutex.h>
58 #include <sys/sysctl.h>
59 #include <sys/systm.h>
62 #include <vm/vm_page.h>
63 #include <vm/vm_map.h>
64 #include <vm/vm_object.h>
65 #include <vm/vm_pageout.h>
68 #include <machine/md_var.h>
69 #include <machine/pal.h>
72 * Manages physical address maps.
74 * In addition to hardware address maps, this
75 * module is called upon to provide software-use-only
76 * maps which may or may not be stored in the same
77 * form as hardware maps. These pseudo-maps are
78 * used to store intermediate results from copy
79 * operations to and from address spaces.
81 * Since the information managed by this module is
82 * also stored by the logical address mapping module,
83 * this module may throw away valid virtual-to-physical
84 * mappings at almost any time. However, invalidations
85 * of virtual-to-physical mappings must be done as
88 * In order to cope with hardware architectures which
89 * make virtual-to-physical map invalidates expensive,
90 * this module may delay invalidate or reduced protection
91 * operations until such time as they are actually
92 * necessary. This module is given full information as
93 * to which processors are currently using which maps,
94 * and to when physical maps must be made correct.
98 * Following the Linux model, region IDs are allocated in groups of
99 * eight so that a single region ID can be used for as many RRs as we
100 * want by encoding the RR number into the low bits of the ID.
102 * We reserve region ID 0 for the kernel and allocate the remaining
103 * IDs for user pmaps.
106 * User virtually mapped
109 * Kernel virtually mapped
112 * Kernel physically mapped uncacheable
115 * Kernel physically mapped cacheable
118 /* XXX move to a header. */
119 extern uint64_t ia64_gateway_page[];
121 MALLOC_DEFINE(M_PMAP, "PMAP", "PMAP Structures");
123 #ifndef PMAP_SHPGPERPROC
124 #define PMAP_SHPGPERPROC 200
127 #if !defined(DIAGNOSTIC)
128 #define PMAP_INLINE __inline
133 #define pmap_accessed(lpte) ((lpte)->pte & PTE_ACCESSED)
134 #define pmap_dirty(lpte) ((lpte)->pte & PTE_DIRTY)
135 #define pmap_exec(lpte) ((lpte)->pte & PTE_AR_RX)
136 #define pmap_managed(lpte) ((lpte)->pte & PTE_MANAGED)
137 #define pmap_ppn(lpte) ((lpte)->pte & PTE_PPN_MASK)
138 #define pmap_present(lpte) ((lpte)->pte & PTE_PRESENT)
139 #define pmap_prot(lpte) (((lpte)->pte & PTE_PROT_MASK) >> 56)
140 #define pmap_wired(lpte) ((lpte)->pte & PTE_WIRED)
142 #define pmap_clear_accessed(lpte) (lpte)->pte &= ~PTE_ACCESSED
143 #define pmap_clear_dirty(lpte) (lpte)->pte &= ~PTE_DIRTY
144 #define pmap_clear_present(lpte) (lpte)->pte &= ~PTE_PRESENT
145 #define pmap_clear_wired(lpte) (lpte)->pte &= ~PTE_WIRED
147 #define pmap_set_wired(lpte) (lpte)->pte |= PTE_WIRED
150 * The VHPT bucket head structure.
159 * Statically allocated kernel pmap
161 struct pmap kernel_pmap_store;
163 vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */
164 vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */
167 * Kernel virtual memory management.
170 struct ia64_lpte ***ia64_kptdir;
171 #define KPTE_DIR0_INDEX(va) \
172 (((va) >> (3*PAGE_SHIFT-8)) & ((1<<(PAGE_SHIFT-3))-1))
173 #define KPTE_DIR1_INDEX(va) \
174 (((va) >> (2*PAGE_SHIFT-5)) & ((1<<(PAGE_SHIFT-3))-1))
175 #define KPTE_PTE_INDEX(va) \
176 (((va) >> PAGE_SHIFT) & ((1<<(PAGE_SHIFT-5))-1))
177 #define NKPTEPG (PAGE_SIZE / sizeof(struct ia64_lpte))
179 vm_offset_t kernel_vm_end;
181 /* Values for ptc.e. XXX values for SKI. */
182 static uint64_t pmap_ptc_e_base = 0x100000000;
183 static uint64_t pmap_ptc_e_count1 = 3;
184 static uint64_t pmap_ptc_e_count2 = 2;
185 static uint64_t pmap_ptc_e_stride1 = 0x2000;
186 static uint64_t pmap_ptc_e_stride2 = 0x100000000;
187 struct mtx pmap_ptcmutex;
190 * Data for the RID allocator
192 static int pmap_ridcount;
193 static int pmap_rididx;
194 static int pmap_ridmapsz;
195 static int pmap_ridmax;
196 static uint64_t *pmap_ridmap;
197 struct mtx pmap_ridmutex;
200 * Data for the pv entry allocation mechanism
202 static uma_zone_t pvzone;
203 static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0;
206 * Data for allocating PTEs for user processes.
208 static uma_zone_t ptezone;
211 * Virtual Hash Page Table (VHPT) data.
213 /* SYSCTL_DECL(_machdep); */
214 SYSCTL_NODE(_machdep, OID_AUTO, vhpt, CTLFLAG_RD, 0, "");
216 struct ia64_bucket *pmap_vhpt_bucket;
218 int pmap_vhpt_nbuckets;
219 SYSCTL_INT(_machdep_vhpt, OID_AUTO, nbuckets, CTLFLAG_RD,
220 &pmap_vhpt_nbuckets, 0, "");
222 uint64_t pmap_vhpt_base[MAXCPU];
224 int pmap_vhpt_log2size = 0;
225 TUNABLE_INT("machdep.vhpt.log2size", &pmap_vhpt_log2size);
226 SYSCTL_INT(_machdep_vhpt, OID_AUTO, log2size, CTLFLAG_RD,
227 &pmap_vhpt_log2size, 0, "");
229 static int pmap_vhpt_inserts;
230 SYSCTL_INT(_machdep_vhpt, OID_AUTO, inserts, CTLFLAG_RD,
231 &pmap_vhpt_inserts, 0, "");
233 static int pmap_vhpt_population(SYSCTL_HANDLER_ARGS);
234 SYSCTL_PROC(_machdep_vhpt, OID_AUTO, population, CTLTYPE_INT | CTLFLAG_RD,
235 NULL, 0, pmap_vhpt_population, "I", "");
237 static struct ia64_lpte *pmap_find_vhpt(vm_offset_t va);
239 static PMAP_INLINE void free_pv_entry(pv_entry_t pv);
240 static pv_entry_t get_pv_entry(pmap_t locked_pmap);
242 static void pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
243 vm_page_t m, vm_prot_t prot);
244 static void pmap_invalidate_all(pmap_t pmap);
245 static int pmap_remove_pte(pmap_t pmap, struct ia64_lpte *pte,
246 vm_offset_t va, pv_entry_t pv, int freepte);
247 static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,
251 pmap_steal_memory(vm_size_t size)
256 size = round_page(size);
258 bank_size = phys_avail[1] - phys_avail[0];
259 while (size > bank_size) {
261 for (i = 0; phys_avail[i+2]; i+= 2) {
262 phys_avail[i] = phys_avail[i+2];
263 phys_avail[i+1] = phys_avail[i+3];
268 panic("pmap_steal_memory: out of memory");
269 bank_size = phys_avail[1] - phys_avail[0];
273 phys_avail[0] += size;
275 va = IA64_PHYS_TO_RR7(pa);
276 bzero((caddr_t) va, size);
281 * Bootstrap the system enough to run with virtual memory.
286 struct ia64_pal_result res;
287 struct ia64_lpte *pte;
288 vm_offset_t base, limit;
290 int i, j, count, ridbits;
293 * Query the PAL Code to find the loop parameters for the
296 res = ia64_call_pal_static(PAL_PTCE_INFO, 0, 0, 0);
297 if (res.pal_status != 0)
298 panic("Can't configure ptc.e parameters");
299 pmap_ptc_e_base = res.pal_result[0];
300 pmap_ptc_e_count1 = res.pal_result[1] >> 32;
301 pmap_ptc_e_count2 = res.pal_result[1] & ((1L<<32) - 1);
302 pmap_ptc_e_stride1 = res.pal_result[2] >> 32;
303 pmap_ptc_e_stride2 = res.pal_result[2] & ((1L<<32) - 1);
305 printf("ptc.e base=0x%lx, count1=%ld, count2=%ld, "
306 "stride1=0x%lx, stride2=0x%lx\n",
312 mtx_init(&pmap_ptcmutex, "Global PTC lock", NULL, MTX_SPIN);
315 * Setup RIDs. RIDs 0..7 are reserved for the kernel.
317 * We currently need at least 19 bits in the RID because PID_MAX
318 * can only be encoded in 17 bits and we need RIDs for 5 regions
319 * per process. With PID_MAX equalling 99999 this means that we
320 * need to be able to encode 499995 (=5*PID_MAX).
321 * The Itanium processor only has 18 bits and the architected
322 * minimum is exactly that. So, we cannot use a PID based scheme
323 * in those cases. Enter pmap_ridmap...
324 * We should avoid the map when running on a processor that has
325 * implemented enough bits. This means that we should pass the
326 * process/thread ID to pmap. This we currently don't do, so we
327 * use the map anyway. However, we don't want to allocate a map
328 * that is large enough to cover the range dictated by the number
329 * of bits in the RID, because that may result in a RID map of
330 * 2MB in size for a 24-bit RID. A 64KB map is enough.
331 * The bottomline: we create a 32KB map when the processor only
332 * implements 18 bits (or when we can't figure it out). Otherwise
333 * we create a 64KB map.
335 res = ia64_call_pal_static(PAL_VM_SUMMARY, 0, 0, 0);
336 if (res.pal_status != 0) {
338 printf("Can't read VM Summary - assuming 18 Region ID bits\n");
339 ridbits = 18; /* guaranteed minimum */
341 ridbits = (res.pal_result[1] >> 8) & 0xff;
343 printf("Processor supports %d Region ID bits\n",
349 pmap_ridmax = (1 << ridbits);
350 pmap_ridmapsz = pmap_ridmax / 64;
351 pmap_ridmap = (uint64_t *)pmap_steal_memory(pmap_ridmax / 8);
352 pmap_ridmap[0] |= 0xff;
355 mtx_init(&pmap_ridmutex, "RID allocator lock", NULL, MTX_DEF);
358 * Allocate some memory for initial kernel 'page tables'.
360 ia64_kptdir = (void *)pmap_steal_memory(PAGE_SIZE);
362 kernel_vm_end = VM_MIN_KERNEL_ADDRESS - VM_GATEWAY_SIZE;
364 for (i = 0; phys_avail[i+2]; i+= 2)
369 * Figure out a useful size for the VHPT, based on the size of
370 * physical memory and try to locate a region which is large
371 * enough to contain the VHPT (which must be a power of two in
372 * size and aligned to a natural boundary).
373 * We silently bump up the VHPT size to the minimum size if the
374 * user has set the tunable too small. Likewise, the VHPT size
375 * is silently capped to the maximum allowed.
377 TUNABLE_INT_FETCH("machdep.vhpt.log2size", &pmap_vhpt_log2size);
378 if (pmap_vhpt_log2size == 0) {
379 pmap_vhpt_log2size = 15;
380 size = 1UL << pmap_vhpt_log2size;
381 while (size < Maxmem * 32) {
382 pmap_vhpt_log2size++;
385 } else if (pmap_vhpt_log2size < 15)
386 pmap_vhpt_log2size = 15;
387 if (pmap_vhpt_log2size > 61)
388 pmap_vhpt_log2size = 61;
390 pmap_vhpt_base[0] = 0;
392 size = 1UL << pmap_vhpt_log2size;
393 while (pmap_vhpt_base[0] == 0) {
395 printf("Trying VHPT size 0x%lx\n", size);
396 for (i = 0; i < count; i += 2) {
397 base = (phys_avail[i] + size - 1) & ~(size - 1);
398 limit = base + MAXCPU * size;
399 if (limit <= phys_avail[i+1])
401 * VHPT can fit in this region
405 if (!phys_avail[i]) {
406 /* Can't fit, try next smaller size. */
407 pmap_vhpt_log2size--;
410 pmap_vhpt_base[0] = IA64_PHYS_TO_RR7(base);
412 if (pmap_vhpt_log2size < 15)
413 panic("Can't find space for VHPT");
416 printf("Putting VHPT at 0x%lx\n", base);
418 if (base != phys_avail[i]) {
419 /* Split this region. */
421 printf("Splitting [%p-%p]\n", (void *)phys_avail[i],
422 (void *)phys_avail[i+1]);
423 for (j = count; j > i; j -= 2) {
424 phys_avail[j] = phys_avail[j-2];
425 phys_avail[j+1] = phys_avail[j-2+1];
427 phys_avail[i+1] = base;
428 phys_avail[i+2] = limit;
430 phys_avail[i] = limit;
432 pmap_vhpt_nbuckets = size / sizeof(struct ia64_lpte);
434 pmap_vhpt_bucket = (void *)pmap_steal_memory(pmap_vhpt_nbuckets *
435 sizeof(struct ia64_bucket));
436 pte = (struct ia64_lpte *)pmap_vhpt_base[0];
437 for (i = 0; i < pmap_vhpt_nbuckets; i++) {
440 pte[i].tag = 1UL << 63; /* Invalid tag */
441 pte[i].chain = (uintptr_t)(pmap_vhpt_bucket + i);
442 /* Stolen memory is zeroed! */
443 mtx_init(&pmap_vhpt_bucket[i].mutex, "VHPT bucket lock", NULL,
447 for (i = 1; i < MAXCPU; i++) {
448 pmap_vhpt_base[i] = pmap_vhpt_base[i - 1] + size;
449 bcopy((void *)pmap_vhpt_base[i - 1], (void *)pmap_vhpt_base[i],
453 map_vhpt(pmap_vhpt_base[0]);
454 ia64_set_pta(pmap_vhpt_base[0] + (1 << 8) +
455 (pmap_vhpt_log2size << 2) + 1);
458 virtual_avail = VM_MIN_KERNEL_ADDRESS;
459 virtual_end = VM_MAX_KERNEL_ADDRESS;
462 * Initialize the kernel pmap (which is statically allocated).
464 PMAP_LOCK_INIT(kernel_pmap);
465 for (i = 0; i < 5; i++)
466 kernel_pmap->pm_rid[i] = 0;
467 kernel_pmap->pm_active = 1;
468 TAILQ_INIT(&kernel_pmap->pm_pvlist);
469 PCPU_SET(current_pmap, kernel_pmap);
472 * Region 5 is mapped via the vhpt.
474 ia64_set_rr(IA64_RR_BASE(5),
475 (5 << 8) | (PAGE_SHIFT << 2) | 1);
478 * Region 6 is direct mapped UC and region 7 is direct mapped
479 * WC. The details of this is controlled by the Alt {I,D}TLB
480 * handlers. Here we just make sure that they have the largest
481 * possible page size to minimise TLB usage.
483 ia64_set_rr(IA64_RR_BASE(6), (6 << 8) | (IA64_ID_PAGE_SHIFT << 2));
484 ia64_set_rr(IA64_RR_BASE(7), (7 << 8) | (IA64_ID_PAGE_SHIFT << 2));
488 * Clear out any random TLB entries left over from booting.
490 pmap_invalidate_all(kernel_pmap);
496 pmap_vhpt_population(SYSCTL_HANDLER_ARGS)
501 for (i = 0; i < pmap_vhpt_nbuckets; i++)
502 count += pmap_vhpt_bucket[i].length;
504 error = SYSCTL_OUT(req, &count, sizeof(count));
509 * Initialize a vm_page's machine-dependent fields.
512 pmap_page_init(vm_page_t m)
515 TAILQ_INIT(&m->md.pv_list);
516 m->md.pv_list_count = 0;
520 * Initialize the pmap module.
521 * Called by vm_init, to initialize any structures that the pmap
522 * system needs to map virtual memory.
527 int shpgperproc = PMAP_SHPGPERPROC;
530 * Initialize the address space (zone) for the pv entries. Set a
531 * high water mark so that the system can recover from excessive
532 * numbers of pv entries.
534 pvzone = uma_zcreate("PV ENTRY", sizeof(struct pv_entry), NULL, NULL,
535 NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE);
536 TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc);
537 pv_entry_max = shpgperproc * maxproc + cnt.v_page_count;
538 TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max);
539 pv_entry_high_water = 9 * (pv_entry_max / 10);
541 ptezone = uma_zcreate("PT ENTRY", sizeof (struct ia64_lpte),
542 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM|UMA_ZONE_NOFREE);
546 /***************************************************
547 * Manipulate TLBs for a pmap
548 ***************************************************/
551 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
553 struct ia64_lpte *pte;
556 KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)),
557 ("invalidating TLB for non-current pmap"));
559 vhpt_ofs = ia64_thash(va) - pmap_vhpt_base[PCPU_GET(cpuid)];
561 for (i = 0; i < MAXCPU; i++) {
562 pte = (struct ia64_lpte *)(pmap_vhpt_base[i] + vhpt_ofs);
563 if (pte->tag == ia64_ttag(va))
564 pte->tag = 1UL << 63;
567 mtx_lock_spin(&pmap_ptcmutex);
568 ia64_ptc_ga(va, PAGE_SHIFT << 2);
569 mtx_unlock_spin(&pmap_ptcmutex);
573 pmap_invalidate_all_1(void *arg)
579 addr = pmap_ptc_e_base;
580 for (i = 0; i < pmap_ptc_e_count1; i++) {
581 for (j = 0; j < pmap_ptc_e_count2; j++) {
583 addr += pmap_ptc_e_stride2;
585 addr += pmap_ptc_e_stride1;
591 pmap_invalidate_all(pmap_t pmap)
594 KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)),
595 ("invalidating TLB for non-current pmap"));
599 smp_rendezvous(NULL, pmap_invalidate_all_1, NULL, NULL);
602 pmap_invalidate_all_1(NULL);
606 pmap_allocate_rid(void)
611 mtx_lock(&pmap_ridmutex);
612 if (pmap_ridcount == pmap_ridmax)
613 panic("pmap_allocate_rid: All Region IDs used");
615 /* Find an index with a free bit. */
616 while ((bits = pmap_ridmap[pmap_rididx]) == ~0UL) {
618 if (pmap_rididx == pmap_ridmapsz)
621 rid = pmap_rididx * 64;
623 /* Find a free bit. */
630 pmap_ridmap[pmap_rididx] |= bit;
632 mtx_unlock(&pmap_ridmutex);
638 pmap_free_rid(uint32_t rid)
644 bit = ~(1UL << (rid & 63));
646 mtx_lock(&pmap_ridmutex);
647 pmap_ridmap[idx] &= bit;
649 mtx_unlock(&pmap_ridmutex);
652 /***************************************************
653 * Page table page management routines.....
654 ***************************************************/
657 pmap_pinit0(struct pmap *pmap)
659 /* kernel_pmap is the same as any other pmap. */
664 * Initialize a preallocated and zeroed pmap structure,
665 * such as one in a vmspace structure.
668 pmap_pinit(struct pmap *pmap)
672 PMAP_LOCK_INIT(pmap);
673 for (i = 0; i < 5; i++)
674 pmap->pm_rid[i] = pmap_allocate_rid();
676 TAILQ_INIT(&pmap->pm_pvlist);
677 bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
681 /***************************************************
682 * Pmap allocation/deallocation routines.
683 ***************************************************/
686 * Release any resources held by the given physical map.
687 * Called when a pmap initialized by pmap_pinit is being released.
688 * Should only be called if the map contains no valid mappings.
691 pmap_release(pmap_t pmap)
695 for (i = 0; i < 5; i++)
697 pmap_free_rid(pmap->pm_rid[i]);
698 PMAP_LOCK_DESTROY(pmap);
702 * grow the number of kernel page table entries, if needed
705 pmap_growkernel(vm_offset_t addr)
707 struct ia64_lpte **dir1;
708 struct ia64_lpte *leaf;
711 while (kernel_vm_end <= addr) {
712 if (nkpt == PAGE_SIZE/8 + PAGE_SIZE*PAGE_SIZE/64)
713 panic("%s: out of kernel address space", __func__);
715 dir1 = ia64_kptdir[KPTE_DIR0_INDEX(kernel_vm_end)];
717 nkpg = vm_page_alloc(NULL, nkpt++,
718 VM_ALLOC_NOOBJ|VM_ALLOC_INTERRUPT|VM_ALLOC_WIRED);
720 panic("%s: cannot add dir. page", __func__);
722 dir1 = (struct ia64_lpte **)
723 IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(nkpg));
724 bzero(dir1, PAGE_SIZE);
725 ia64_kptdir[KPTE_DIR0_INDEX(kernel_vm_end)] = dir1;
728 nkpg = vm_page_alloc(NULL, nkpt++,
729 VM_ALLOC_NOOBJ|VM_ALLOC_INTERRUPT|VM_ALLOC_WIRED);
731 panic("%s: cannot add PTE page", __func__);
733 leaf = (struct ia64_lpte *)
734 IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(nkpg));
735 bzero(leaf, PAGE_SIZE);
736 dir1[KPTE_DIR1_INDEX(kernel_vm_end)] = leaf;
738 kernel_vm_end += PAGE_SIZE * NKPTEPG;
742 /***************************************************
743 * page management routines.
744 ***************************************************/
747 * free the pv_entry back to the free list
749 static PMAP_INLINE void
750 free_pv_entry(pv_entry_t pv)
753 uma_zfree(pvzone, pv);
757 * get a new pv_entry, allocating a block from the system
761 get_pv_entry(pmap_t locked_pmap)
763 static const struct timeval printinterval = { 60, 0 };
764 static struct timeval lastprint;
765 struct vpgqueues *vpq;
766 struct ia64_lpte *pte;
767 pmap_t oldpmap, pmap;
768 pv_entry_t allocated_pv, next_pv, pv;
772 PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED);
773 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
774 allocated_pv = uma_zalloc(pvzone, M_NOWAIT);
775 if (allocated_pv != NULL) {
777 if (pv_entry_count > pv_entry_high_water)
780 return (allocated_pv);
784 * Reclaim pv entries: At first, destroy mappings to inactive
785 * pages. After that, if a pv entry is still needed, destroy
786 * mappings to active pages.
788 if (ratecheck(&lastprint, &printinterval))
789 printf("Approaching the limit on PV entries, "
790 "increase the vm.pmap.shpgperproc tunable.\n");
791 vpq = &vm_page_queues[PQ_INACTIVE];
793 TAILQ_FOREACH(m, &vpq->pl, pageq) {
794 if (m->hold_count || m->busy)
796 TAILQ_FOREACH_SAFE(pv, &m->md.pv_list, pv_list, next_pv) {
799 /* Avoid deadlock and lock recursion. */
800 if (pmap > locked_pmap)
802 else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap))
804 oldpmap = pmap_switch(pmap);
805 pte = pmap_find_vhpt(va);
806 KASSERT(pte != NULL, ("pte"));
807 pmap_remove_pte(pmap, pte, va, pv, 1);
808 pmap_switch(oldpmap);
809 if (pmap != locked_pmap)
811 if (allocated_pv == NULL)
817 if (allocated_pv == NULL) {
818 if (vpq == &vm_page_queues[PQ_INACTIVE]) {
819 vpq = &vm_page_queues[PQ_ACTIVE];
822 panic("get_pv_entry: increase the vm.pmap.shpgperproc tunable");
824 return (allocated_pv);
828 * Conditionally create a pv entry.
831 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
835 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
836 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
837 if (pv_entry_count < pv_entry_high_water &&
838 (pv = uma_zalloc(pvzone, M_NOWAIT)) != NULL) {
842 TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist);
843 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
844 m->md.pv_list_count++;
851 * Add an ia64_lpte to the VHPT.
854 pmap_enter_vhpt(struct ia64_lpte *pte, vm_offset_t va)
856 struct ia64_bucket *bckt;
857 struct ia64_lpte *vhpte;
860 /* Can fault, so get it out of the way. */
861 pte_pa = ia64_tpa((vm_offset_t)pte);
863 vhpte = (struct ia64_lpte *)ia64_thash(va);
864 bckt = (struct ia64_bucket *)vhpte->chain;
866 mtx_lock_spin(&bckt->mutex);
867 pte->chain = bckt->chain;
869 bckt->chain = pte_pa;
873 mtx_unlock_spin(&bckt->mutex);
877 * Remove the ia64_lpte matching va from the VHPT. Return zero if it
878 * worked or an appropriate error code otherwise.
881 pmap_remove_vhpt(vm_offset_t va)
883 struct ia64_bucket *bckt;
884 struct ia64_lpte *pte;
885 struct ia64_lpte *lpte;
886 struct ia64_lpte *vhpte;
890 vhpte = (struct ia64_lpte *)ia64_thash(va);
891 bckt = (struct ia64_bucket *)vhpte->chain;
894 mtx_lock_spin(&bckt->mutex);
896 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
897 while (chain != 0 && pte->tag != tag) {
900 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
903 mtx_unlock_spin(&bckt->mutex);
907 /* Snip this pv_entry out of the collision chain. */
909 bckt->chain = pte->chain;
911 lpte->chain = pte->chain;
915 mtx_unlock_spin(&bckt->mutex);
920 * Find the ia64_lpte for the given va, if any.
922 static struct ia64_lpte *
923 pmap_find_vhpt(vm_offset_t va)
925 struct ia64_bucket *bckt;
926 struct ia64_lpte *pte;
930 pte = (struct ia64_lpte *)ia64_thash(va);
931 bckt = (struct ia64_bucket *)pte->chain;
933 mtx_lock_spin(&bckt->mutex);
935 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
936 while (chain != 0 && pte->tag != tag) {
938 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
940 mtx_unlock_spin(&bckt->mutex);
941 return ((chain != 0) ? pte : NULL);
945 * Remove an entry from the list of managed mappings.
948 pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va, pv_entry_t pv)
951 if (m->md.pv_list_count < pmap->pm_stats.resident_count) {
952 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
953 if (pmap == pv->pv_pmap && va == pv->pv_va)
957 TAILQ_FOREACH(pv, &pmap->pm_pvlist, pv_plist) {
965 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
966 m->md.pv_list_count--;
967 if (TAILQ_FIRST(&m->md.pv_list) == NULL)
968 vm_page_flag_clear(m, PG_WRITEABLE);
970 TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist);
979 * Create a pv entry for page at pa for
983 pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
987 pv = get_pv_entry(pmap);
991 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
992 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
993 TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist);
994 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
995 m->md.pv_list_count++;
999 * Routine: pmap_extract
1001 * Extract the physical page address associated
1002 * with the given map/virtual_address pair.
1005 pmap_extract(pmap_t pmap, vm_offset_t va)
1007 struct ia64_lpte *pte;
1013 oldpmap = pmap_switch(pmap);
1014 pte = pmap_find_vhpt(va);
1015 if (pte != NULL && pmap_present(pte))
1017 pmap_switch(oldpmap);
1023 * Routine: pmap_extract_and_hold
1025 * Atomically extract and hold the physical page
1026 * with the given pmap and virtual address pair
1027 * if that mapping permits the given protection.
1030 pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
1032 struct ia64_lpte *pte;
1037 vm_page_lock_queues();
1039 oldpmap = pmap_switch(pmap);
1040 pte = pmap_find_vhpt(va);
1041 if (pte != NULL && pmap_present(pte) &&
1042 (pmap_prot(pte) & prot) == prot) {
1043 m = PHYS_TO_VM_PAGE(pmap_ppn(pte));
1046 vm_page_unlock_queues();
1047 pmap_switch(oldpmap);
1052 /***************************************************
1053 * Low level mapping routines.....
1054 ***************************************************/
1057 * Find the kernel lpte for mapping the given virtual address, which
1058 * must be in the part of region 5 which we can cover with our kernel
1061 static struct ia64_lpte *
1062 pmap_find_kpte(vm_offset_t va)
1064 struct ia64_lpte **dir1;
1065 struct ia64_lpte *leaf;
1067 KASSERT((va >> 61) == 5,
1068 ("kernel mapping 0x%lx not in region 5", va));
1069 KASSERT(va < kernel_vm_end,
1070 ("kernel mapping 0x%lx out of range", va));
1072 dir1 = ia64_kptdir[KPTE_DIR0_INDEX(va)];
1073 leaf = dir1[KPTE_DIR1_INDEX(va)];
1074 return (&leaf[KPTE_PTE_INDEX(va)]);
1078 * Find a pte suitable for mapping a user-space address. If one exists
1079 * in the VHPT, that one will be returned, otherwise a new pte is
1082 static struct ia64_lpte *
1083 pmap_find_pte(vm_offset_t va)
1085 struct ia64_lpte *pte;
1087 if (va >= VM_MAXUSER_ADDRESS)
1088 return pmap_find_kpte(va);
1090 pte = pmap_find_vhpt(va);
1092 pte = uma_zalloc(ptezone, M_NOWAIT | M_ZERO);
1093 pte->tag = 1UL << 63;
1099 * Free a pte which is now unused. This simply returns it to the zone
1100 * allocator if it is a user mapping. For kernel mappings, clear the
1101 * valid bit to make it clear that the mapping is not currently used.
1104 pmap_free_pte(struct ia64_lpte *pte, vm_offset_t va)
1106 if (va < VM_MAXUSER_ADDRESS)
1107 uma_zfree(ptezone, pte);
1109 pmap_clear_present(pte);
1112 static PMAP_INLINE void
1113 pmap_pte_prot(pmap_t pm, struct ia64_lpte *pte, vm_prot_t prot)
1115 static long prot2ar[4] = {
1116 PTE_AR_R, /* VM_PROT_NONE */
1117 PTE_AR_RW, /* VM_PROT_WRITE */
1118 PTE_AR_RX|PTE_ED, /* VM_PROT_EXECUTE */
1119 PTE_AR_RWX|PTE_ED /* VM_PROT_WRITE|VM_PROT_EXECUTE */
1122 pte->pte &= ~(PTE_PROT_MASK | PTE_PL_MASK | PTE_AR_MASK | PTE_ED);
1123 pte->pte |= (uint64_t)(prot & VM_PROT_ALL) << 56;
1124 pte->pte |= (prot == VM_PROT_NONE || pm == kernel_pmap)
1125 ? PTE_PL_KERN : PTE_PL_USER;
1126 pte->pte |= prot2ar[(prot & VM_PROT_ALL) >> 1];
1130 * Set a pte to contain a valid mapping and enter it in the VHPT. If
1131 * the pte was orginally valid, then its assumed to already be in the
1133 * This functions does not set the protection bits. It's expected
1134 * that those have been set correctly prior to calling this function.
1137 pmap_set_pte(struct ia64_lpte *pte, vm_offset_t va, vm_offset_t pa,
1138 boolean_t wired, boolean_t managed)
1141 pte->pte &= PTE_PROT_MASK | PTE_PL_MASK | PTE_AR_MASK | PTE_ED;
1142 pte->pte |= PTE_PRESENT | PTE_MA_WB;
1143 pte->pte |= (managed) ? PTE_MANAGED : (PTE_DIRTY | PTE_ACCESSED);
1144 pte->pte |= (wired) ? PTE_WIRED : 0;
1145 pte->pte |= pa & PTE_PPN_MASK;
1147 pte->itir = PAGE_SHIFT << 2;
1149 pte->tag = ia64_ttag(va);
1153 * Remove the (possibly managed) mapping represented by pte from the
1157 pmap_remove_pte(pmap_t pmap, struct ia64_lpte *pte, vm_offset_t va,
1158 pv_entry_t pv, int freepte)
1163 KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)),
1164 ("removing pte for non-current pmap"));
1167 * First remove from the VHPT.
1169 error = pmap_remove_vhpt(va);
1173 pmap_invalidate_page(pmap, va);
1175 if (pmap_wired(pte))
1176 pmap->pm_stats.wired_count -= 1;
1178 pmap->pm_stats.resident_count -= 1;
1179 if (pmap_managed(pte)) {
1180 m = PHYS_TO_VM_PAGE(pmap_ppn(pte));
1181 if (pmap_dirty(pte))
1183 if (pmap_accessed(pte))
1184 vm_page_flag_set(m, PG_REFERENCED);
1186 error = pmap_remove_entry(pmap, m, va, pv);
1189 pmap_free_pte(pte, va);
1195 * Extract the physical page address associated with a kernel
1199 pmap_kextract(vm_offset_t va)
1201 struct ia64_lpte *pte;
1204 KASSERT(va >= IA64_RR_BASE(5), ("Must be kernel VA"));
1206 /* Regions 6 and 7 are direct mapped. */
1207 if (va >= IA64_RR_BASE(6))
1208 return (IA64_RR_MASK(va));
1210 /* EPC gateway page? */
1211 gwpage = (vm_offset_t)ia64_get_k5();
1212 if (va >= gwpage && va < gwpage + VM_GATEWAY_SIZE)
1213 return (IA64_RR_MASK((vm_offset_t)ia64_gateway_page));
1215 /* Bail out if the virtual address is beyond our limits. */
1216 if (va >= kernel_vm_end)
1219 pte = pmap_find_kpte(va);
1220 if (!pmap_present(pte))
1222 return (pmap_ppn(pte) | (va & PAGE_MASK));
1226 * Add a list of wired pages to the kva this routine is only used for
1227 * temporary kernel mappings that do not need to have page modification
1228 * or references recorded. Note that old mappings are simply written
1229 * over. The page is effectively wired, but it's customary to not have
1230 * the PTE reflect that, nor update statistics.
1233 pmap_qenter(vm_offset_t va, vm_page_t *m, int count)
1235 struct ia64_lpte *pte;
1238 for (i = 0; i < count; i++) {
1239 pte = pmap_find_kpte(va);
1240 if (pmap_present(pte))
1241 pmap_invalidate_page(kernel_pmap, va);
1243 pmap_enter_vhpt(pte, va);
1244 pmap_pte_prot(kernel_pmap, pte, VM_PROT_ALL);
1245 pmap_set_pte(pte, va, VM_PAGE_TO_PHYS(m[i]), FALSE, FALSE);
1251 * this routine jerks page mappings from the
1252 * kernel -- it is meant only for temporary mappings.
1255 pmap_qremove(vm_offset_t va, int count)
1257 struct ia64_lpte *pte;
1260 for (i = 0; i < count; i++) {
1261 pte = pmap_find_kpte(va);
1262 if (pmap_present(pte)) {
1263 pmap_remove_vhpt(va);
1264 pmap_invalidate_page(kernel_pmap, va);
1265 pmap_clear_present(pte);
1272 * Add a wired page to the kva. As for pmap_qenter(), it's customary
1273 * to not have the PTE reflect that, nor update statistics.
1276 pmap_kenter(vm_offset_t va, vm_offset_t pa)
1278 struct ia64_lpte *pte;
1280 pte = pmap_find_kpte(va);
1281 if (pmap_present(pte))
1282 pmap_invalidate_page(kernel_pmap, va);
1284 pmap_enter_vhpt(pte, va);
1285 pmap_pte_prot(kernel_pmap, pte, VM_PROT_ALL);
1286 pmap_set_pte(pte, va, pa, FALSE, FALSE);
1290 * Remove a page from the kva
1293 pmap_kremove(vm_offset_t va)
1295 struct ia64_lpte *pte;
1297 pte = pmap_find_kpte(va);
1298 if (pmap_present(pte)) {
1299 pmap_remove_vhpt(va);
1300 pmap_invalidate_page(kernel_pmap, va);
1301 pmap_clear_present(pte);
1306 * Used to map a range of physical addresses into kernel
1307 * virtual address space.
1309 * The value passed in '*virt' is a suggested virtual address for
1310 * the mapping. Architectures which can support a direct-mapped
1311 * physical to virtual region can return the appropriate address
1312 * within that region, leaving '*virt' unchanged. Other
1313 * architectures should map the pages starting at '*virt' and
1314 * update '*virt' with the first usable address after the mapped
1318 pmap_map(vm_offset_t *virt, vm_offset_t start, vm_offset_t end, int prot)
1320 return IA64_PHYS_TO_RR7(start);
1324 * Remove a single page from a process address space
1327 pmap_remove_page(pmap_t pmap, vm_offset_t va)
1329 struct ia64_lpte *pte;
1331 KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)),
1332 ("removing page for non-current pmap"));
1334 pte = pmap_find_vhpt(va);
1336 pmap_remove_pte(pmap, pte, va, 0, 1);
1341 * Remove the given range of addresses from the specified map.
1343 * It is assumed that the start and end are properly
1344 * rounded to the page size.
1347 pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
1352 struct ia64_lpte *pte;
1354 if (pmap->pm_stats.resident_count == 0)
1357 vm_page_lock_queues();
1359 oldpmap = pmap_switch(pmap);
1362 * special handling of removing one page. a very
1363 * common operation and easy to short circuit some
1366 if (sva + PAGE_SIZE == eva) {
1367 pmap_remove_page(pmap, sva);
1371 if (pmap->pm_stats.resident_count < ((eva - sva) >> PAGE_SHIFT)) {
1372 TAILQ_FOREACH_SAFE(pv, &pmap->pm_pvlist, pv_plist, npv) {
1374 if (va >= sva && va < eva) {
1375 pte = pmap_find_vhpt(va);
1376 KASSERT(pte != NULL, ("pte"));
1377 pmap_remove_pte(pmap, pte, va, pv, 1);
1381 for (va = sva; va < eva; va += PAGE_SIZE) {
1382 pte = pmap_find_vhpt(va);
1384 pmap_remove_pte(pmap, pte, va, 0, 1);
1389 vm_page_unlock_queues();
1390 pmap_switch(oldpmap);
1395 * Routine: pmap_remove_all
1397 * Removes this physical page from
1398 * all physical maps in which it resides.
1399 * Reflects back modify bits to the pager.
1402 * Original versions of this routine were very
1403 * inefficient because they iteratively called
1404 * pmap_remove (slow...)
1408 pmap_remove_all(vm_page_t m)
1413 #if defined(DIAGNOSTIC)
1415 * XXX this makes pmap_page_protect(NONE) illegal for non-managed
1418 if (m->flags & PG_FICTITIOUS) {
1419 panic("pmap_page_protect: illegal for unmanaged page, va: 0x%lx", VM_PAGE_TO_PHYS(m));
1422 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1423 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
1424 struct ia64_lpte *pte;
1425 pmap_t pmap = pv->pv_pmap;
1426 vm_offset_t va = pv->pv_va;
1429 oldpmap = pmap_switch(pmap);
1430 pte = pmap_find_vhpt(va);
1431 KASSERT(pte != NULL, ("pte"));
1432 if (pmap_ppn(pte) != VM_PAGE_TO_PHYS(m))
1433 panic("pmap_remove_all: pv_table for %lx is inconsistent", VM_PAGE_TO_PHYS(m));
1434 pmap_remove_pte(pmap, pte, va, pv, 1);
1435 pmap_switch(oldpmap);
1438 vm_page_flag_clear(m, PG_WRITEABLE);
1442 * Set the physical protection on the
1443 * specified range of this map as requested.
1446 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
1449 struct ia64_lpte *pte;
1451 if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
1452 pmap_remove(pmap, sva, eva);
1456 if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) ==
1457 (VM_PROT_WRITE|VM_PROT_EXECUTE))
1460 if ((sva & PAGE_MASK) || (eva & PAGE_MASK))
1461 panic("pmap_protect: unaligned addresses");
1463 vm_page_lock_queues();
1465 oldpmap = pmap_switch(pmap);
1466 for ( ; sva < eva; sva += PAGE_SIZE) {
1467 /* If page is invalid, skip this page */
1468 pte = pmap_find_vhpt(sva);
1472 /* If there's no change, skip it too */
1473 if (pmap_prot(pte) == prot)
1476 if (pmap_managed(pte)) {
1477 vm_offset_t pa = pmap_ppn(pte);
1478 vm_page_t m = PHYS_TO_VM_PAGE(pa);
1480 if (pmap_dirty(pte)) {
1482 pmap_clear_dirty(pte);
1485 if (pmap_accessed(pte)) {
1486 vm_page_flag_set(m, PG_REFERENCED);
1487 pmap_clear_accessed(pte);
1491 if (prot & VM_PROT_EXECUTE)
1492 ia64_invalidate_icache(sva, PAGE_SIZE);
1494 pmap_pte_prot(pmap, pte, prot);
1495 pmap_invalidate_page(pmap, sva);
1497 vm_page_unlock_queues();
1498 pmap_switch(oldpmap);
1503 * Insert the given physical page (p) at
1504 * the specified virtual address (v) in the
1505 * target physical map with the protection requested.
1507 * If specified, the page will be wired down, meaning
1508 * that the related pte can not be reclaimed.
1510 * NB: This is the only routine which MAY NOT lazy-evaluate
1511 * or lose information. That is, this routine must actually
1512 * insert this page into the given map NOW.
1515 pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t m,
1516 vm_prot_t prot, boolean_t wired)
1521 struct ia64_lpte origpte;
1522 struct ia64_lpte *pte;
1523 boolean_t icache_inval, managed;
1525 vm_page_lock_queues();
1527 oldpmap = pmap_switch(pmap);
1531 if (va > VM_MAX_KERNEL_ADDRESS)
1532 panic("pmap_enter: toobig");
1536 * Find (or create) a pte for the given mapping.
1538 while ((pte = pmap_find_pte(va)) == NULL) {
1539 pmap_switch(oldpmap);
1541 vm_page_unlock_queues();
1543 vm_page_lock_queues();
1545 oldpmap = pmap_switch(pmap);
1548 if (!pmap_present(pte)) {
1550 pmap_enter_vhpt(pte, va);
1552 opa = pmap_ppn(pte);
1554 pa = VM_PAGE_TO_PHYS(m);
1556 icache_inval = (prot & VM_PROT_EXECUTE) ? TRUE : FALSE;
1559 * Mapping has not changed, must be protection or wiring change.
1563 * Wiring change, just update stats. We don't worry about
1564 * wiring PT pages as they remain resident as long as there
1565 * are valid mappings in them. Hence, if a user page is wired,
1566 * the PT page will be also.
1568 if (wired && !pmap_wired(&origpte))
1569 pmap->pm_stats.wired_count++;
1570 else if (!wired && pmap_wired(&origpte))
1571 pmap->pm_stats.wired_count--;
1573 managed = (pmap_managed(&origpte)) ? TRUE : FALSE;
1576 * We might be turning off write access to the page,
1577 * so we go ahead and sense modify status. Otherwise,
1578 * we can avoid I-cache invalidation if the page
1579 * already allowed execution.
1581 if (managed && pmap_dirty(&origpte))
1583 else if (pmap_exec(&origpte))
1584 icache_inval = FALSE;
1586 pmap_invalidate_page(pmap, va);
1591 * Mapping has changed, invalidate old range and fall
1592 * through to handle validating new mapping.
1595 pmap_remove_pte(pmap, pte, va, 0, 0);
1596 pmap_enter_vhpt(pte, va);
1600 * Enter on the PV list if part of our managed memory.
1602 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) {
1603 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva,
1604 ("pmap_enter: managed mapping within the clean submap"));
1605 pmap_insert_entry(pmap, va, m);
1610 * Increment counters
1612 pmap->pm_stats.resident_count++;
1614 pmap->pm_stats.wired_count++;
1619 * Now validate mapping with desired protection/wiring. This
1620 * adds the pte to the VHPT if necessary.
1622 pmap_pte_prot(pmap, pte, prot);
1623 pmap_set_pte(pte, va, pa, wired, managed);
1625 /* Invalidate the I-cache when needed. */
1627 ia64_invalidate_icache(va, PAGE_SIZE);
1629 if ((prot & VM_PROT_WRITE) != 0)
1630 vm_page_flag_set(m, PG_WRITEABLE);
1631 vm_page_unlock_queues();
1632 pmap_switch(oldpmap);
1637 * Maps a sequence of resident pages belonging to the same object.
1638 * The sequence begins with the given page m_start. This page is
1639 * mapped at the given virtual address start. Each subsequent page is
1640 * mapped at a virtual address that is offset from start by the same
1641 * amount as the page is offset from m_start within the object. The
1642 * last page in the sequence is the page with the largest offset from
1643 * m_start that can be mapped at a virtual address less than the given
1644 * virtual address end. Not every virtual page between start and end
1645 * is mapped; only those for which a resident page exists with the
1646 * corresponding offset from m_start are mapped.
1649 pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,
1650 vm_page_t m_start, vm_prot_t prot)
1654 vm_pindex_t diff, psize;
1656 VM_OBJECT_LOCK_ASSERT(m_start->object, MA_OWNED);
1657 psize = atop(end - start);
1660 oldpmap = pmap_switch(pmap);
1661 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
1662 pmap_enter_quick_locked(pmap, start + ptoa(diff), m, prot);
1663 m = TAILQ_NEXT(m, listq);
1665 pmap_switch(oldpmap);
1670 * this code makes some *MAJOR* assumptions:
1671 * 1. Current pmap & pmap exists.
1674 * 4. No page table pages.
1675 * but is *MUCH* faster than pmap_enter...
1679 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
1684 oldpmap = pmap_switch(pmap);
1685 pmap_enter_quick_locked(pmap, va, m, prot);
1686 pmap_switch(oldpmap);
1691 pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
1694 struct ia64_lpte *pte;
1697 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva ||
1698 (m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0,
1699 ("pmap_enter_quick_locked: managed mapping within the clean submap"));
1700 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1701 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1703 if ((pte = pmap_find_pte(va)) == NULL)
1706 if (!pmap_present(pte)) {
1707 /* Enter on the PV list if the page is managed. */
1708 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) {
1709 if (!pmap_try_insert_pv_entry(pmap, va, m)) {
1710 pmap_free_pte(pte, va);
1717 /* Increment counters. */
1718 pmap->pm_stats.resident_count++;
1720 /* Initialise with R/O protection and enter into VHPT. */
1721 pmap_enter_vhpt(pte, va);
1722 pmap_pte_prot(pmap, pte,
1723 prot & (VM_PROT_READ | VM_PROT_EXECUTE));
1724 pmap_set_pte(pte, va, VM_PAGE_TO_PHYS(m), FALSE, managed);
1729 * pmap_object_init_pt preloads the ptes for a given object
1730 * into the specified pmap. This eliminates the blast of soft
1731 * faults on process startup and immediately after an mmap.
1734 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr,
1735 vm_object_t object, vm_pindex_t pindex,
1739 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1740 KASSERT(object->type == OBJT_DEVICE,
1741 ("pmap_object_init_pt: non-device object"));
1745 * Routine: pmap_change_wiring
1746 * Function: Change the wiring attribute for a map/virtual-address
1748 * In/out conditions:
1749 * The mapping must already exist in the pmap.
1752 pmap_change_wiring(pmap, va, wired)
1753 register pmap_t pmap;
1758 struct ia64_lpte *pte;
1761 oldpmap = pmap_switch(pmap);
1763 pte = pmap_find_vhpt(va);
1764 KASSERT(pte != NULL, ("pte"));
1765 if (wired && !pmap_wired(pte)) {
1766 pmap->pm_stats.wired_count++;
1767 pmap_set_wired(pte);
1768 } else if (!wired && pmap_wired(pte)) {
1769 pmap->pm_stats.wired_count--;
1770 pmap_clear_wired(pte);
1773 pmap_switch(oldpmap);
1780 * Copy the range specified by src_addr/len
1781 * from the source map to the range dst_addr/len
1782 * in the destination map.
1784 * This routine is only advisory and need not do anything.
1788 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
1789 vm_offset_t src_addr)
1795 * pmap_zero_page zeros the specified hardware page by
1796 * mapping it into virtual memory and using bzero to clear
1801 pmap_zero_page(vm_page_t m)
1803 vm_offset_t va = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m));
1804 bzero((caddr_t) va, PAGE_SIZE);
1809 * pmap_zero_page_area zeros the specified hardware page by
1810 * mapping it into virtual memory and using bzero to clear
1813 * off and size must reside within a single page.
1817 pmap_zero_page_area(vm_page_t m, int off, int size)
1819 vm_offset_t va = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m));
1820 bzero((char *)(caddr_t)va + off, size);
1825 * pmap_zero_page_idle zeros the specified hardware page by
1826 * mapping it into virtual memory and using bzero to clear
1827 * its contents. This is for the vm_idlezero process.
1831 pmap_zero_page_idle(vm_page_t m)
1833 vm_offset_t va = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m));
1834 bzero((caddr_t) va, PAGE_SIZE);
1839 * pmap_copy_page copies the specified (machine independent)
1840 * page by mapping the page into virtual memory and using
1841 * bcopy to copy the page, one machine dependent page at a
1845 pmap_copy_page(vm_page_t msrc, vm_page_t mdst)
1847 vm_offset_t src = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(msrc));
1848 vm_offset_t dst = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(mdst));
1849 bcopy((caddr_t) src, (caddr_t) dst, PAGE_SIZE);
1853 * Returns true if the pmap's pv is one of the first
1854 * 16 pvs linked to from this page. This count may
1855 * be changed upwards or downwards in the future; it
1856 * is only necessary that true be returned for a small
1857 * subset of pmaps for proper page aging.
1860 pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
1865 if (m->flags & PG_FICTITIOUS)
1869 * Not found, check current mappings returning immediately if found.
1871 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1872 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
1873 if (pv->pv_pmap == pmap) {
1884 * Remove all pages from specified address space
1885 * this aids process exit speeds. Also, this code
1886 * is special cased for current process only, but
1887 * can have the more generic (and slightly slower)
1888 * mode enabled. This is much faster than pmap_remove
1889 * in the case of running down an entire address space.
1892 pmap_remove_pages(pmap_t pmap)
1897 if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace)) {
1898 printf("warning: pmap_remove_pages called with non-current pmap\n");
1902 vm_page_lock_queues();
1904 oldpmap = pmap_switch(pmap);
1906 for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) {
1907 struct ia64_lpte *pte;
1909 npv = TAILQ_NEXT(pv, pv_plist);
1911 pte = pmap_find_vhpt(pv->pv_va);
1912 KASSERT(pte != NULL, ("pte"));
1913 if (!pmap_wired(pte))
1914 pmap_remove_pte(pmap, pte, pv->pv_va, pv, 1);
1917 pmap_switch(oldpmap);
1919 vm_page_unlock_queues();
1923 * pmap_ts_referenced:
1925 * Return a count of reference bits for a page, clearing those bits.
1926 * It is not necessary for every reference bit to be cleared, but it
1927 * is necessary that 0 only be returned when there are truly no
1928 * reference bits set.
1930 * XXX: The exact number of bits to check and clear is a matter that
1931 * should be tested and standardized at some point in the future for
1932 * optimal aging of shared pages.
1935 pmap_ts_referenced(vm_page_t m)
1937 struct ia64_lpte *pte;
1942 if (m->flags & PG_FICTITIOUS)
1945 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
1946 PMAP_LOCK(pv->pv_pmap);
1947 oldpmap = pmap_switch(pv->pv_pmap);
1948 pte = pmap_find_vhpt(pv->pv_va);
1949 KASSERT(pte != NULL, ("pte"));
1950 if (pmap_accessed(pte)) {
1952 pmap_clear_accessed(pte);
1953 pmap_invalidate_page(pv->pv_pmap, pv->pv_va);
1955 pmap_switch(oldpmap);
1956 PMAP_UNLOCK(pv->pv_pmap);
1965 * Return whether or not the specified physical page was modified
1966 * in any physical maps.
1969 pmap_is_modified(vm_page_t m)
1971 struct ia64_lpte *pte;
1977 if (m->flags & PG_FICTITIOUS)
1980 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
1981 PMAP_LOCK(pv->pv_pmap);
1982 oldpmap = pmap_switch(pv->pv_pmap);
1983 pte = pmap_find_vhpt(pv->pv_va);
1984 pmap_switch(oldpmap);
1985 KASSERT(pte != NULL, ("pte"));
1986 rv = pmap_dirty(pte) ? TRUE : FALSE;
1987 PMAP_UNLOCK(pv->pv_pmap);
1996 * pmap_is_prefaultable:
1998 * Return whether or not the specified virtual address is elgible
2002 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)
2004 struct ia64_lpte *pte;
2006 pte = pmap_find_vhpt(addr);
2007 if (pte != NULL && pmap_present(pte))
2013 * Clear the modify bits on the specified physical page.
2016 pmap_clear_modify(vm_page_t m)
2018 struct ia64_lpte *pte;
2022 if (m->flags & PG_FICTITIOUS)
2025 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2026 PMAP_LOCK(pv->pv_pmap);
2027 oldpmap = pmap_switch(pv->pv_pmap);
2028 pte = pmap_find_vhpt(pv->pv_va);
2029 KASSERT(pte != NULL, ("pte"));
2030 if (pmap_dirty(pte)) {
2031 pmap_clear_dirty(pte);
2032 pmap_invalidate_page(pv->pv_pmap, pv->pv_va);
2034 pmap_switch(oldpmap);
2035 PMAP_UNLOCK(pv->pv_pmap);
2040 * pmap_clear_reference:
2042 * Clear the reference bit on the specified physical page.
2045 pmap_clear_reference(vm_page_t m)
2047 struct ia64_lpte *pte;
2051 if (m->flags & PG_FICTITIOUS)
2054 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2055 PMAP_LOCK(pv->pv_pmap);
2056 oldpmap = pmap_switch(pv->pv_pmap);
2057 pte = pmap_find_vhpt(pv->pv_va);
2058 KASSERT(pte != NULL, ("pte"));
2059 if (pmap_accessed(pte)) {
2060 pmap_clear_accessed(pte);
2061 pmap_invalidate_page(pv->pv_pmap, pv->pv_va);
2063 pmap_switch(oldpmap);
2064 PMAP_UNLOCK(pv->pv_pmap);
2069 * Clear the write and modified bits in each of the given page's mappings.
2072 pmap_remove_write(vm_page_t m)
2074 struct ia64_lpte *pte;
2075 pmap_t oldpmap, pmap;
2079 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
2080 if ((m->flags & PG_FICTITIOUS) != 0 ||
2081 (m->flags & PG_WRITEABLE) == 0)
2083 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2086 oldpmap = pmap_switch(pmap);
2087 pte = pmap_find_vhpt(pv->pv_va);
2088 KASSERT(pte != NULL, ("pte"));
2089 prot = pmap_prot(pte);
2090 if ((prot & VM_PROT_WRITE) != 0) {
2091 if (pmap_dirty(pte)) {
2093 pmap_clear_dirty(pte);
2095 prot &= ~VM_PROT_WRITE;
2096 pmap_pte_prot(pmap, pte, prot);
2097 pmap_invalidate_page(pmap, pv->pv_va);
2099 pmap_switch(oldpmap);
2102 vm_page_flag_clear(m, PG_WRITEABLE);
2106 * Map a set of physical memory pages into the kernel virtual
2107 * address space. Return a pointer to where it is mapped. This
2108 * routine is intended to be used for mapping device memory,
2112 pmap_mapdev(vm_offset_t pa, vm_size_t size)
2114 return (void*) IA64_PHYS_TO_RR6(pa);
2118 * 'Unmap' a range mapped by pmap_mapdev().
2121 pmap_unmapdev(vm_offset_t va, vm_size_t size)
2127 * perform the pmap work for mincore
2130 pmap_mincore(pmap_t pmap, vm_offset_t addr)
2133 struct ia64_lpte *pte, tpte;
2137 oldpmap = pmap_switch(pmap);
2138 pte = pmap_find_vhpt(addr);
2143 pmap_switch(oldpmap);
2149 if (pmap_present(pte)) {
2153 val = MINCORE_INCORE;
2154 if (!pmap_managed(pte))
2159 m = PHYS_TO_VM_PAGE(pa);
2164 if (pmap_dirty(pte))
2165 val |= MINCORE_MODIFIED|MINCORE_MODIFIED_OTHER;
2168 * Modified by someone
2170 vm_page_lock_queues();
2171 if (pmap_is_modified(m))
2172 val |= MINCORE_MODIFIED_OTHER;
2173 vm_page_unlock_queues();
2178 if (pmap_accessed(pte))
2179 val |= MINCORE_REFERENCED|MINCORE_REFERENCED_OTHER;
2182 * Referenced by someone
2184 vm_page_lock_queues();
2185 if (pmap_ts_referenced(m)) {
2186 val |= MINCORE_REFERENCED_OTHER;
2187 vm_page_flag_set(m, PG_REFERENCED);
2189 vm_page_unlock_queues();
2196 pmap_activate(struct thread *td)
2198 pmap_switch(vmspace_pmap(td->td_proc->p_vmspace));
2202 pmap_switch(pmap_t pm)
2208 prevpm = PCPU_GET(current_pmap);
2212 atomic_clear_32(&prevpm->pm_active, PCPU_GET(cpumask));
2214 for (i = 0; i < 5; i++) {
2215 ia64_set_rr(IA64_RR_BASE(i),
2216 (i << 8)|(PAGE_SHIFT << 2)|1);
2219 for (i = 0; i < 5; i++) {
2220 ia64_set_rr(IA64_RR_BASE(i),
2221 (pm->pm_rid[i] << 8)|(PAGE_SHIFT << 2)|1);
2223 atomic_set_32(&pm->pm_active, PCPU_GET(cpumask));
2225 PCPU_SET(current_pmap, pm);
2234 pmap_addr_hint(vm_object_t obj, vm_offset_t addr, vm_size_t size)
2241 * Increase the starting virtual address of the given mapping if a
2242 * different alignment might result in more superpage mappings.
2245 pmap_align_superpage(vm_object_t object, vm_ooffset_t offset,
2246 vm_offset_t *addr, vm_size_t size)
2250 #include "opt_ddb.h"
2254 #include <ddb/ddb.h>
2256 static const char* psnames[] = {
2257 "1B", "2B", "4B", "8B",
2258 "16B", "32B", "64B", "128B",
2259 "256B", "512B", "1K", "2K",
2260 "4K", "8K", "16K", "32K",
2261 "64K", "128K", "256K", "512K",
2262 "1M", "2M", "4M", "8M",
2263 "16M", "32M", "64M", "128M",
2264 "256M", "512M", "1G", "2G"
2270 struct ia64_pal_result res;
2278 static const char *manames[] = {
2279 "WB", "bad", "bad", "bad",
2280 "UC", "UCE", "WC", "NaT",
2283 res = ia64_call_pal_static(PAL_VM_SUMMARY, 0, 0, 0);
2284 if (res.pal_status != 0) {
2285 db_printf("Can't get VM summary\n");
2290 maxtr = (res.pal_result[0] >> 40) & 0xff;
2292 maxtr = (res.pal_result[0] >> 32) & 0xff;
2294 db_printf("V RID Virtual Page Physical Page PgSz ED AR PL D A MA P KEY\n");
2295 for (i = 0; i <= maxtr; i++) {
2296 bzero(&buf, sizeof(buf));
2297 res = ia64_call_pal_stacked_physical
2298 (PAL_VM_TR_READ, i, type, ia64_tpa((uint64_t) &buf));
2299 if (!(res.pal_result[0] & 1))
2300 buf.pte &= ~PTE_AR_MASK;
2301 if (!(res.pal_result[0] & 2))
2302 buf.pte &= ~PTE_PL_MASK;
2303 if (!(res.pal_result[0] & 4))
2304 pmap_clear_dirty(&buf);
2305 if (!(res.pal_result[0] & 8))
2306 buf.pte &= ~PTE_MA_MASK;
2307 db_printf("%d %06x %013lx %013lx %4s %d %d %d %d %d %-3s "
2308 "%d %06x\n", (int)buf.ifa & 1, buf.rr.rr_rid,
2309 buf.ifa >> 12, (buf.pte & PTE_PPN_MASK) >> 12,
2310 psnames[(buf.itir & ITIR_PS_MASK) >> 2],
2311 (buf.pte & PTE_ED) ? 1 : 0,
2312 (int)(buf.pte & PTE_AR_MASK) >> 9,
2313 (int)(buf.pte & PTE_PL_MASK) >> 7,
2314 (pmap_dirty(&buf)) ? 1 : 0,
2315 (pmap_accessed(&buf)) ? 1 : 0,
2316 manames[(buf.pte & PTE_MA_MASK) >> 2],
2317 (pmap_present(&buf)) ? 1 : 0,
2318 (int)((buf.itir & ITIR_KEY_MASK) >> 8));
2322 DB_COMMAND(itr, db_itr)
2327 DB_COMMAND(dtr, db_dtr)
2332 DB_COMMAND(rr, db_rr)
2338 printf("RR RID PgSz VE\n");
2339 for (i = 0; i < 8; i++) {
2340 __asm __volatile ("mov %0=rr[%1]"
2342 : "r"(IA64_RR_BASE(i)));
2343 *(uint64_t *) &rr = t;
2344 printf("%d %06x %4s %d\n",
2345 i, rr.rr_rid, psnames[rr.rr_ps], rr.rr_ve);
2349 DB_COMMAND(thash, db_thash)
2354 db_printf("%p\n", (void *) ia64_thash(addr));
2357 DB_COMMAND(ttag, db_ttag)
2362 db_printf("0x%lx\n", ia64_ttag(addr));
2365 DB_COMMAND(kpte, db_kpte)
2367 struct ia64_lpte *pte;
2370 db_printf("usage: kpte <kva>\n");
2373 if (addr < VM_MIN_KERNEL_ADDRESS) {
2374 db_printf("kpte: error: invalid <kva>\n");
2377 pte = pmap_find_kpte(addr);
2378 db_printf("kpte at %p:\n", pte);
2379 db_printf(" pte =%016lx\n", pte->pte);
2380 db_printf(" itir =%016lx\n", pte->itir);
2381 db_printf(" tag =%016lx\n", pte->tag);
2382 db_printf(" chain=%016lx\n", pte->chain);