2 * Copyright (c) 1991 Regents of the University of California.
4 * Copyright (c) 1994 John S. Dyson
6 * Copyright (c) 1994 David Greenman
8 * Copyright (c) 1998,2000 Doug Rabson
11 * This code is derived from software contributed to Berkeley by
12 * the Systems Programming Group of the University of Utah Computer
13 * Science Department and William Jolitz of UUNET Technologies Inc.
15 * Redistribution and use in source and binary forms, with or without
16 * modification, are permitted provided that the following conditions
18 * 1. Redistributions of source code must retain the above copyright
19 * notice, this list of conditions and the following disclaimer.
20 * 2. Redistributions in binary form must reproduce the above copyright
21 * notice, this list of conditions and the following disclaimer in the
22 * documentation and/or other materials provided with the distribution.
23 * 3. All advertising materials mentioning features or use of this software
24 * must display the following acknowledgement:
25 * This product includes software developed by the University of
26 * California, Berkeley and its contributors.
27 * 4. Neither the name of the University nor the names of its contributors
28 * may be used to endorse or promote products derived from this software
29 * without specific prior written permission.
31 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
32 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
33 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
34 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
35 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
36 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
37 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
38 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
39 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
40 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
43 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91
44 * from: i386 Id: pmap.c,v 1.193 1998/04/19 15:22:48 bde Exp
45 * with some ideas from NetBSD's alpha pmap
48 #include <sys/cdefs.h>
49 __FBSDID("$FreeBSD$");
51 #include <sys/param.h>
52 #include <sys/kernel.h>
55 #include <sys/mutex.h>
58 #include <sys/sysctl.h>
59 #include <sys/systm.h>
62 #include <vm/vm_page.h>
63 #include <vm/vm_map.h>
64 #include <vm/vm_object.h>
65 #include <vm/vm_pageout.h>
68 #include <machine/md_var.h>
69 #include <machine/pal.h>
72 * Manages physical address maps.
74 * In addition to hardware address maps, this
75 * module is called upon to provide software-use-only
76 * maps which may or may not be stored in the same
77 * form as hardware maps. These pseudo-maps are
78 * used to store intermediate results from copy
79 * operations to and from address spaces.
81 * Since the information managed by this module is
82 * also stored by the logical address mapping module,
83 * this module may throw away valid virtual-to-physical
84 * mappings at almost any time. However, invalidations
85 * of virtual-to-physical mappings must be done as
88 * In order to cope with hardware architectures which
89 * make virtual-to-physical map invalidates expensive,
90 * this module may delay invalidate or reduced protection
91 * operations until such time as they are actually
92 * necessary. This module is given full information as
93 * to which processors are currently using which maps,
94 * and to when physical maps must be made correct.
98 * Following the Linux model, region IDs are allocated in groups of
99 * eight so that a single region ID can be used for as many RRs as we
100 * want by encoding the RR number into the low bits of the ID.
102 * We reserve region ID 0 for the kernel and allocate the remaining
103 * IDs for user pmaps.
106 * User virtually mapped
109 * Kernel virtually mapped
112 * Kernel physically mapped uncacheable
115 * Kernel physically mapped cacheable
118 /* XXX move to a header. */
119 extern uint64_t ia64_gateway_page[];
121 /* XXX fc.i kluge (quick fix) */
122 int ia64_icache_sync_kluge;
124 MALLOC_DEFINE(M_PMAP, "PMAP", "PMAP Structures");
126 #ifndef PMAP_SHPGPERPROC
127 #define PMAP_SHPGPERPROC 200
130 #if !defined(DIAGNOSTIC)
131 #define PMAP_INLINE __inline
136 #define pmap_accessed(lpte) ((lpte)->pte & PTE_ACCESSED)
137 #define pmap_dirty(lpte) ((lpte)->pte & PTE_DIRTY)
138 #define pmap_managed(lpte) ((lpte)->pte & PTE_MANAGED)
139 #define pmap_ppn(lpte) ((lpte)->pte & PTE_PPN_MASK)
140 #define pmap_present(lpte) ((lpte)->pte & PTE_PRESENT)
141 #define pmap_prot(lpte) (((lpte)->pte & PTE_PROT_MASK) >> 56)
142 #define pmap_wired(lpte) ((lpte)->pte & PTE_WIRED)
144 #define pmap_clear_accessed(lpte) (lpte)->pte &= ~PTE_ACCESSED
145 #define pmap_clear_dirty(lpte) (lpte)->pte &= ~PTE_DIRTY
146 #define pmap_clear_present(lpte) (lpte)->pte &= ~PTE_PRESENT
147 #define pmap_clear_wired(lpte) (lpte)->pte &= ~PTE_WIRED
149 #define pmap_set_wired(lpte) (lpte)->pte |= PTE_WIRED
152 * The VHPT bucket head structure.
161 * Statically allocated kernel pmap
163 struct pmap kernel_pmap_store;
165 vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */
166 vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */
169 * Kernel virtual memory management.
172 struct ia64_lpte ***ia64_kptdir;
173 #define KPTE_DIR0_INDEX(va) \
174 (((va) >> (3*PAGE_SHIFT-8)) & ((1<<(PAGE_SHIFT-3))-1))
175 #define KPTE_DIR1_INDEX(va) \
176 (((va) >> (2*PAGE_SHIFT-5)) & ((1<<(PAGE_SHIFT-3))-1))
177 #define KPTE_PTE_INDEX(va) \
178 (((va) >> PAGE_SHIFT) & ((1<<(PAGE_SHIFT-5))-1))
179 #define NKPTEPG (PAGE_SIZE / sizeof(struct ia64_lpte))
181 vm_offset_t kernel_vm_end;
183 /* Values for ptc.e. XXX values for SKI. */
184 static uint64_t pmap_ptc_e_base = 0x100000000;
185 static uint64_t pmap_ptc_e_count1 = 3;
186 static uint64_t pmap_ptc_e_count2 = 2;
187 static uint64_t pmap_ptc_e_stride1 = 0x2000;
188 static uint64_t pmap_ptc_e_stride2 = 0x100000000;
189 struct mtx pmap_ptcmutex;
192 * Data for the RID allocator
194 static int pmap_ridcount;
195 static int pmap_rididx;
196 static int pmap_ridmapsz;
197 static int pmap_ridmax;
198 static uint64_t *pmap_ridmap;
199 struct mtx pmap_ridmutex;
202 * Data for the pv entry allocation mechanism
204 static uma_zone_t pvzone;
205 static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0;
208 * Data for allocating PTEs for user processes.
210 static uma_zone_t ptezone;
213 * Virtual Hash Page Table (VHPT) data.
215 /* SYSCTL_DECL(_machdep); */
216 SYSCTL_NODE(_machdep, OID_AUTO, vhpt, CTLFLAG_RD, 0, "");
218 struct ia64_bucket *pmap_vhpt_bucket;
220 int pmap_vhpt_nbuckets;
221 SYSCTL_INT(_machdep_vhpt, OID_AUTO, nbuckets, CTLFLAG_RD,
222 &pmap_vhpt_nbuckets, 0, "");
224 uint64_t pmap_vhpt_base[MAXCPU];
226 int pmap_vhpt_log2size = 0;
227 TUNABLE_INT("machdep.vhpt.log2size", &pmap_vhpt_log2size);
228 SYSCTL_INT(_machdep_vhpt, OID_AUTO, log2size, CTLFLAG_RD,
229 &pmap_vhpt_log2size, 0, "");
231 static int pmap_vhpt_inserts;
232 SYSCTL_INT(_machdep_vhpt, OID_AUTO, inserts, CTLFLAG_RD,
233 &pmap_vhpt_inserts, 0, "");
235 static int pmap_vhpt_population(SYSCTL_HANDLER_ARGS);
236 SYSCTL_PROC(_machdep_vhpt, OID_AUTO, population, CTLTYPE_INT | CTLFLAG_RD,
237 NULL, 0, pmap_vhpt_population, "I", "");
239 static struct ia64_lpte *pmap_find_vhpt(vm_offset_t va);
241 static PMAP_INLINE void free_pv_entry(pv_entry_t pv);
242 static pv_entry_t get_pv_entry(pmap_t locked_pmap);
244 static void pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
245 vm_page_t m, vm_prot_t prot);
246 static void pmap_invalidate_all(pmap_t pmap);
247 static int pmap_remove_pte(pmap_t pmap, struct ia64_lpte *pte,
248 vm_offset_t va, pv_entry_t pv, int freepte);
249 static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,
253 pmap_steal_memory(vm_size_t size)
258 size = round_page(size);
260 bank_size = phys_avail[1] - phys_avail[0];
261 while (size > bank_size) {
263 for (i = 0; phys_avail[i+2]; i+= 2) {
264 phys_avail[i] = phys_avail[i+2];
265 phys_avail[i+1] = phys_avail[i+3];
270 panic("pmap_steal_memory: out of memory");
271 bank_size = phys_avail[1] - phys_avail[0];
275 phys_avail[0] += size;
277 va = IA64_PHYS_TO_RR7(pa);
278 bzero((caddr_t) va, size);
283 * Bootstrap the system enough to run with virtual memory.
288 struct ia64_pal_result res;
289 struct ia64_lpte *pte;
290 vm_offset_t base, limit;
292 int i, j, count, ridbits;
295 * Query the PAL Code to find the loop parameters for the
298 res = ia64_call_pal_static(PAL_PTCE_INFO, 0, 0, 0);
299 if (res.pal_status != 0)
300 panic("Can't configure ptc.e parameters");
301 pmap_ptc_e_base = res.pal_result[0];
302 pmap_ptc_e_count1 = res.pal_result[1] >> 32;
303 pmap_ptc_e_count2 = res.pal_result[1] & ((1L<<32) - 1);
304 pmap_ptc_e_stride1 = res.pal_result[2] >> 32;
305 pmap_ptc_e_stride2 = res.pal_result[2] & ((1L<<32) - 1);
307 printf("ptc.e base=0x%lx, count1=%ld, count2=%ld, "
308 "stride1=0x%lx, stride2=0x%lx\n",
314 mtx_init(&pmap_ptcmutex, "Global PTC lock", NULL, MTX_SPIN);
317 * Setup RIDs. RIDs 0..7 are reserved for the kernel.
319 * We currently need at least 19 bits in the RID because PID_MAX
320 * can only be encoded in 17 bits and we need RIDs for 5 regions
321 * per process. With PID_MAX equalling 99999 this means that we
322 * need to be able to encode 499995 (=5*PID_MAX).
323 * The Itanium processor only has 18 bits and the architected
324 * minimum is exactly that. So, we cannot use a PID based scheme
325 * in those cases. Enter pmap_ridmap...
326 * We should avoid the map when running on a processor that has
327 * implemented enough bits. This means that we should pass the
328 * process/thread ID to pmap. This we currently don't do, so we
329 * use the map anyway. However, we don't want to allocate a map
330 * that is large enough to cover the range dictated by the number
331 * of bits in the RID, because that may result in a RID map of
332 * 2MB in size for a 24-bit RID. A 64KB map is enough.
333 * The bottomline: we create a 32KB map when the processor only
334 * implements 18 bits (or when we can't figure it out). Otherwise
335 * we create a 64KB map.
337 res = ia64_call_pal_static(PAL_VM_SUMMARY, 0, 0, 0);
338 if (res.pal_status != 0) {
340 printf("Can't read VM Summary - assuming 18 Region ID bits\n");
341 ridbits = 18; /* guaranteed minimum */
343 ridbits = (res.pal_result[1] >> 8) & 0xff;
345 printf("Processor supports %d Region ID bits\n",
351 pmap_ridmax = (1 << ridbits);
352 pmap_ridmapsz = pmap_ridmax / 64;
353 pmap_ridmap = (uint64_t *)pmap_steal_memory(pmap_ridmax / 8);
354 pmap_ridmap[0] |= 0xff;
357 mtx_init(&pmap_ridmutex, "RID allocator lock", NULL, MTX_DEF);
360 * Allocate some memory for initial kernel 'page tables'.
362 ia64_kptdir = (void *)pmap_steal_memory(PAGE_SIZE);
364 kernel_vm_end = VM_MIN_KERNEL_ADDRESS - VM_GATEWAY_SIZE;
366 for (i = 0; phys_avail[i+2]; i+= 2)
371 * Figure out a useful size for the VHPT, based on the size of
372 * physical memory and try to locate a region which is large
373 * enough to contain the VHPT (which must be a power of two in
374 * size and aligned to a natural boundary).
375 * We silently bump up the VHPT size to the minimum size if the
376 * user has set the tunable too small. Likewise, the VHPT size
377 * is silently capped to the maximum allowed.
379 TUNABLE_INT_FETCH("machdep.vhpt.log2size", &pmap_vhpt_log2size);
380 if (pmap_vhpt_log2size == 0) {
381 pmap_vhpt_log2size = 15;
382 size = 1UL << pmap_vhpt_log2size;
383 while (size < Maxmem * 32) {
384 pmap_vhpt_log2size++;
387 } else if (pmap_vhpt_log2size < 15)
388 pmap_vhpt_log2size = 15;
389 if (pmap_vhpt_log2size > 61)
390 pmap_vhpt_log2size = 61;
392 pmap_vhpt_base[0] = 0;
394 size = 1UL << pmap_vhpt_log2size;
395 while (pmap_vhpt_base[0] == 0) {
397 printf("Trying VHPT size 0x%lx\n", size);
398 for (i = 0; i < count; i += 2) {
399 base = (phys_avail[i] + size - 1) & ~(size - 1);
400 limit = base + MAXCPU * size;
401 if (limit <= phys_avail[i+1])
403 * VHPT can fit in this region
407 if (!phys_avail[i]) {
408 /* Can't fit, try next smaller size. */
409 pmap_vhpt_log2size--;
412 pmap_vhpt_base[0] = IA64_PHYS_TO_RR7(base);
414 if (pmap_vhpt_log2size < 15)
415 panic("Can't find space for VHPT");
418 printf("Putting VHPT at 0x%lx\n", base);
420 if (base != phys_avail[i]) {
421 /* Split this region. */
423 printf("Splitting [%p-%p]\n", (void *)phys_avail[i],
424 (void *)phys_avail[i+1]);
425 for (j = count; j > i; j -= 2) {
426 phys_avail[j] = phys_avail[j-2];
427 phys_avail[j+1] = phys_avail[j-2+1];
429 phys_avail[i+1] = base;
430 phys_avail[i+2] = limit;
432 phys_avail[i] = limit;
434 pmap_vhpt_nbuckets = size / sizeof(struct ia64_lpte);
436 pmap_vhpt_bucket = (void *)pmap_steal_memory(pmap_vhpt_nbuckets *
437 sizeof(struct ia64_bucket));
438 pte = (struct ia64_lpte *)pmap_vhpt_base[0];
439 for (i = 0; i < pmap_vhpt_nbuckets; i++) {
442 pte[i].tag = 1UL << 63; /* Invalid tag */
443 pte[i].chain = (uintptr_t)(pmap_vhpt_bucket + i);
444 /* Stolen memory is zeroed! */
445 mtx_init(&pmap_vhpt_bucket[i].mutex, "VHPT bucket lock", NULL,
449 for (i = 1; i < MAXCPU; i++) {
450 pmap_vhpt_base[i] = pmap_vhpt_base[i - 1] + size;
451 bcopy((void *)pmap_vhpt_base[i - 1], (void *)pmap_vhpt_base[i],
455 map_vhpt(pmap_vhpt_base[0]);
456 ia64_set_pta(pmap_vhpt_base[0] + (1 << 8) +
457 (pmap_vhpt_log2size << 2) + 1);
460 virtual_avail = VM_MIN_KERNEL_ADDRESS;
461 virtual_end = VM_MAX_KERNEL_ADDRESS;
464 * Initialize the kernel pmap (which is statically allocated).
466 PMAP_LOCK_INIT(kernel_pmap);
467 for (i = 0; i < 5; i++)
468 kernel_pmap->pm_rid[i] = 0;
469 kernel_pmap->pm_active = 1;
470 TAILQ_INIT(&kernel_pmap->pm_pvlist);
471 PCPU_SET(current_pmap, kernel_pmap);
474 * Region 5 is mapped via the vhpt.
476 ia64_set_rr(IA64_RR_BASE(5),
477 (5 << 8) | (PAGE_SHIFT << 2) | 1);
480 * Region 6 is direct mapped UC and region 7 is direct mapped
481 * WC. The details of this is controlled by the Alt {I,D}TLB
482 * handlers. Here we just make sure that they have the largest
483 * possible page size to minimise TLB usage.
485 ia64_set_rr(IA64_RR_BASE(6), (6 << 8) | (IA64_ID_PAGE_SHIFT << 2));
486 ia64_set_rr(IA64_RR_BASE(7), (7 << 8) | (IA64_ID_PAGE_SHIFT << 2));
490 * Clear out any random TLB entries left over from booting.
492 pmap_invalidate_all(kernel_pmap);
498 pmap_vhpt_population(SYSCTL_HANDLER_ARGS)
503 for (i = 0; i < pmap_vhpt_nbuckets; i++)
504 count += pmap_vhpt_bucket[i].length;
506 error = SYSCTL_OUT(req, &count, sizeof(count));
511 * Initialize a vm_page's machine-dependent fields.
514 pmap_page_init(vm_page_t m)
517 TAILQ_INIT(&m->md.pv_list);
518 m->md.pv_list_count = 0;
522 * Initialize the pmap module.
523 * Called by vm_init, to initialize any structures that the pmap
524 * system needs to map virtual memory.
529 int shpgperproc = PMAP_SHPGPERPROC;
532 * Initialize the address space (zone) for the pv entries. Set a
533 * high water mark so that the system can recover from excessive
534 * numbers of pv entries.
536 pvzone = uma_zcreate("PV ENTRY", sizeof(struct pv_entry), NULL, NULL,
537 NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE);
538 TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc);
539 pv_entry_max = shpgperproc * maxproc + cnt.v_page_count;
540 TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max);
541 pv_entry_high_water = 9 * (pv_entry_max / 10);
543 ptezone = uma_zcreate("PT ENTRY", sizeof (struct ia64_lpte),
544 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM|UMA_ZONE_NOFREE);
548 /***************************************************
549 * Manipulate TLBs for a pmap
550 ***************************************************/
554 pmap_invalidate_page_locally(void *arg)
556 vm_offset_t va = (uintptr_t)arg;
557 struct ia64_lpte *pte;
559 pte = (struct ia64_lpte *)ia64_thash(va);
560 if (pte->tag == ia64_ttag(va))
561 pte->tag = 1UL << 63;
562 ia64_ptc_l(va, PAGE_SHIFT << 2);
567 pmap_invalidate_page_1(void *arg)
573 oldpmap = pmap_switch(args[0]);
574 pmap_invalidate_page_locally(args[1]);
575 pmap_switch(oldpmap);
581 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
584 KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)),
585 ("invalidating TLB for non-current pmap"));
591 args[1] = (void *)va;
592 smp_rendezvous(NULL, pmap_invalidate_page_1, NULL, args);
595 pmap_invalidate_page_locally((void *)va);
600 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
602 struct ia64_lpte *pte;
605 KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)),
606 ("invalidating TLB for non-current pmap"));
608 vhpt_ofs = ia64_thash(va) - pmap_vhpt_base[PCPU_GET(cpuid)];
610 for (i = 0; i < MAXCPU; i++) {
611 pte = (struct ia64_lpte *)(pmap_vhpt_base[i] + vhpt_ofs);
612 if (pte->tag == ia64_ttag(va))
613 pte->tag = 1UL << 63;
616 mtx_lock_spin(&pmap_ptcmutex);
617 ia64_ptc_ga(va, PAGE_SHIFT << 2);
618 mtx_unlock_spin(&pmap_ptcmutex);
622 pmap_invalidate_all_1(void *arg)
628 addr = pmap_ptc_e_base;
629 for (i = 0; i < pmap_ptc_e_count1; i++) {
630 for (j = 0; j < pmap_ptc_e_count2; j++) {
632 addr += pmap_ptc_e_stride2;
634 addr += pmap_ptc_e_stride1;
640 pmap_invalidate_all(pmap_t pmap)
643 KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)),
644 ("invalidating TLB for non-current pmap"));
648 smp_rendezvous(NULL, pmap_invalidate_all_1, NULL, NULL);
651 pmap_invalidate_all_1(NULL);
655 pmap_allocate_rid(void)
660 mtx_lock(&pmap_ridmutex);
661 if (pmap_ridcount == pmap_ridmax)
662 panic("pmap_allocate_rid: All Region IDs used");
664 /* Find an index with a free bit. */
665 while ((bits = pmap_ridmap[pmap_rididx]) == ~0UL) {
667 if (pmap_rididx == pmap_ridmapsz)
670 rid = pmap_rididx * 64;
672 /* Find a free bit. */
679 pmap_ridmap[pmap_rididx] |= bit;
681 mtx_unlock(&pmap_ridmutex);
687 pmap_free_rid(uint32_t rid)
693 bit = ~(1UL << (rid & 63));
695 mtx_lock(&pmap_ridmutex);
696 pmap_ridmap[idx] &= bit;
698 mtx_unlock(&pmap_ridmutex);
701 /***************************************************
702 * Page table page management routines.....
703 ***************************************************/
706 pmap_pinit0(struct pmap *pmap)
708 /* kernel_pmap is the same as any other pmap. */
713 * Initialize a preallocated and zeroed pmap structure,
714 * such as one in a vmspace structure.
717 pmap_pinit(struct pmap *pmap)
721 PMAP_LOCK_INIT(pmap);
722 for (i = 0; i < 5; i++)
723 pmap->pm_rid[i] = pmap_allocate_rid();
725 TAILQ_INIT(&pmap->pm_pvlist);
726 bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
730 /***************************************************
731 * Pmap allocation/deallocation routines.
732 ***************************************************/
735 * Release any resources held by the given physical map.
736 * Called when a pmap initialized by pmap_pinit is being released.
737 * Should only be called if the map contains no valid mappings.
740 pmap_release(pmap_t pmap)
744 for (i = 0; i < 5; i++)
746 pmap_free_rid(pmap->pm_rid[i]);
747 PMAP_LOCK_DESTROY(pmap);
751 * grow the number of kernel page table entries, if needed
754 pmap_growkernel(vm_offset_t addr)
756 struct ia64_lpte **dir1;
757 struct ia64_lpte *leaf;
760 while (kernel_vm_end <= addr) {
761 if (nkpt == PAGE_SIZE/8 + PAGE_SIZE*PAGE_SIZE/64)
762 panic("%s: out of kernel address space", __func__);
764 dir1 = ia64_kptdir[KPTE_DIR0_INDEX(kernel_vm_end)];
766 nkpg = vm_page_alloc(NULL, nkpt++,
767 VM_ALLOC_NOOBJ|VM_ALLOC_INTERRUPT|VM_ALLOC_WIRED);
769 panic("%s: cannot add dir. page", __func__);
771 dir1 = (struct ia64_lpte **)
772 IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(nkpg));
773 bzero(dir1, PAGE_SIZE);
774 ia64_kptdir[KPTE_DIR0_INDEX(kernel_vm_end)] = dir1;
777 nkpg = vm_page_alloc(NULL, nkpt++,
778 VM_ALLOC_NOOBJ|VM_ALLOC_INTERRUPT|VM_ALLOC_WIRED);
780 panic("%s: cannot add PTE page", __func__);
782 leaf = (struct ia64_lpte *)
783 IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(nkpg));
784 bzero(leaf, PAGE_SIZE);
785 dir1[KPTE_DIR1_INDEX(kernel_vm_end)] = leaf;
787 kernel_vm_end += PAGE_SIZE * NKPTEPG;
791 /***************************************************
792 * page management routines.
793 ***************************************************/
796 * free the pv_entry back to the free list
798 static PMAP_INLINE void
799 free_pv_entry(pv_entry_t pv)
802 uma_zfree(pvzone, pv);
806 * get a new pv_entry, allocating a block from the system
810 get_pv_entry(pmap_t locked_pmap)
812 static const struct timeval printinterval = { 60, 0 };
813 static struct timeval lastprint;
814 struct vpgqueues *vpq;
815 struct ia64_lpte *pte;
816 pmap_t oldpmap, pmap;
817 pv_entry_t allocated_pv, next_pv, pv;
821 PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED);
822 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
823 allocated_pv = uma_zalloc(pvzone, M_NOWAIT);
824 if (allocated_pv != NULL) {
826 if (pv_entry_count > pv_entry_high_water)
829 return (allocated_pv);
833 * Reclaim pv entries: At first, destroy mappings to inactive
834 * pages. After that, if a pv entry is still needed, destroy
835 * mappings to active pages.
837 if (ratecheck(&lastprint, &printinterval))
838 printf("Approaching the limit on PV entries, "
839 "increase the vm.pmap.shpgperproc tunable.\n");
840 vpq = &vm_page_queues[PQ_INACTIVE];
842 TAILQ_FOREACH(m, &vpq->pl, pageq) {
843 if (m->hold_count || m->busy)
845 TAILQ_FOREACH_SAFE(pv, &m->md.pv_list, pv_list, next_pv) {
848 /* Avoid deadlock and lock recursion. */
849 if (pmap > locked_pmap)
851 else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap))
853 oldpmap = pmap_switch(pmap);
854 pte = pmap_find_vhpt(va);
855 KASSERT(pte != NULL, ("pte"));
856 pmap_remove_pte(pmap, pte, va, pv, 1);
857 pmap_switch(oldpmap);
858 if (pmap != locked_pmap)
860 if (allocated_pv == NULL)
866 if (allocated_pv == NULL) {
867 if (vpq == &vm_page_queues[PQ_INACTIVE]) {
868 vpq = &vm_page_queues[PQ_ACTIVE];
871 panic("get_pv_entry: increase the vm.pmap.shpgperproc tunable");
873 return (allocated_pv);
877 * Conditionally create a pv entry.
880 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
884 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
885 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
886 if (pv_entry_count < pv_entry_high_water &&
887 (pv = uma_zalloc(pvzone, M_NOWAIT)) != NULL) {
891 TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist);
892 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
893 m->md.pv_list_count++;
900 * Add an ia64_lpte to the VHPT.
903 pmap_enter_vhpt(struct ia64_lpte *pte, vm_offset_t va)
905 struct ia64_bucket *bckt;
906 struct ia64_lpte *vhpte;
909 /* Can fault, so get it out of the way. */
910 pte_pa = ia64_tpa((vm_offset_t)pte);
912 vhpte = (struct ia64_lpte *)ia64_thash(va);
913 bckt = (struct ia64_bucket *)vhpte->chain;
915 mtx_lock_spin(&bckt->mutex);
916 pte->chain = bckt->chain;
918 bckt->chain = pte_pa;
922 mtx_unlock_spin(&bckt->mutex);
926 * Remove the ia64_lpte matching va from the VHPT. Return zero if it
927 * worked or an appropriate error code otherwise.
930 pmap_remove_vhpt(vm_offset_t va)
932 struct ia64_bucket *bckt;
933 struct ia64_lpte *pte;
934 struct ia64_lpte *lpte;
935 struct ia64_lpte *vhpte;
939 vhpte = (struct ia64_lpte *)ia64_thash(va);
940 bckt = (struct ia64_bucket *)vhpte->chain;
943 mtx_lock_spin(&bckt->mutex);
945 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
946 while (chain != 0 && pte->tag != tag) {
949 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
952 mtx_unlock_spin(&bckt->mutex);
956 /* Snip this pv_entry out of the collision chain. */
958 bckt->chain = pte->chain;
960 lpte->chain = pte->chain;
964 mtx_unlock_spin(&bckt->mutex);
969 * Find the ia64_lpte for the given va, if any.
971 static struct ia64_lpte *
972 pmap_find_vhpt(vm_offset_t va)
974 struct ia64_bucket *bckt;
975 struct ia64_lpte *pte;
979 pte = (struct ia64_lpte *)ia64_thash(va);
980 bckt = (struct ia64_bucket *)pte->chain;
982 mtx_lock_spin(&bckt->mutex);
984 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
985 while (chain != 0 && pte->tag != tag) {
987 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
989 mtx_unlock_spin(&bckt->mutex);
990 return ((chain != 0) ? pte : NULL);
994 * Remove an entry from the list of managed mappings.
997 pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va, pv_entry_t pv)
1000 if (m->md.pv_list_count < pmap->pm_stats.resident_count) {
1001 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
1002 if (pmap == pv->pv_pmap && va == pv->pv_va)
1006 TAILQ_FOREACH(pv, &pmap->pm_pvlist, pv_plist) {
1007 if (va == pv->pv_va)
1014 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
1015 m->md.pv_list_count--;
1016 if (TAILQ_FIRST(&m->md.pv_list) == NULL)
1017 vm_page_flag_clear(m, PG_WRITEABLE);
1019 TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist);
1028 * Create a pv entry for page at pa for
1032 pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
1036 pv = get_pv_entry(pmap);
1040 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1041 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1042 TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist);
1043 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
1044 m->md.pv_list_count++;
1048 * Routine: pmap_extract
1050 * Extract the physical page address associated
1051 * with the given map/virtual_address pair.
1054 pmap_extract(pmap_t pmap, vm_offset_t va)
1056 struct ia64_lpte *pte;
1062 oldpmap = pmap_switch(pmap);
1063 pte = pmap_find_vhpt(va);
1064 if (pte != NULL && pmap_present(pte))
1066 pmap_switch(oldpmap);
1072 * Routine: pmap_extract_and_hold
1074 * Atomically extract and hold the physical page
1075 * with the given pmap and virtual address pair
1076 * if that mapping permits the given protection.
1079 pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
1081 struct ia64_lpte *pte;
1086 vm_page_lock_queues();
1088 oldpmap = pmap_switch(pmap);
1089 pte = pmap_find_vhpt(va);
1090 if (pte != NULL && pmap_present(pte) &&
1091 (pmap_prot(pte) & prot) == prot) {
1092 m = PHYS_TO_VM_PAGE(pmap_ppn(pte));
1095 vm_page_unlock_queues();
1096 pmap_switch(oldpmap);
1101 /***************************************************
1102 * Low level mapping routines.....
1103 ***************************************************/
1106 * Find the kernel lpte for mapping the given virtual address, which
1107 * must be in the part of region 5 which we can cover with our kernel
1110 static struct ia64_lpte *
1111 pmap_find_kpte(vm_offset_t va)
1113 struct ia64_lpte **dir1;
1114 struct ia64_lpte *leaf;
1116 KASSERT((va >> 61) == 5,
1117 ("kernel mapping 0x%lx not in region 5", va));
1118 KASSERT(va < kernel_vm_end,
1119 ("kernel mapping 0x%lx out of range", va));
1121 dir1 = ia64_kptdir[KPTE_DIR0_INDEX(va)];
1122 leaf = dir1[KPTE_DIR1_INDEX(va)];
1123 return (&leaf[KPTE_PTE_INDEX(va)]);
1127 * Find a pte suitable for mapping a user-space address. If one exists
1128 * in the VHPT, that one will be returned, otherwise a new pte is
1131 static struct ia64_lpte *
1132 pmap_find_pte(vm_offset_t va)
1134 struct ia64_lpte *pte;
1136 if (va >= VM_MAXUSER_ADDRESS)
1137 return pmap_find_kpte(va);
1139 pte = pmap_find_vhpt(va);
1141 pte = uma_zalloc(ptezone, M_NOWAIT | M_ZERO);
1142 pte->tag = 1UL << 63;
1148 * Free a pte which is now unused. This simply returns it to the zone
1149 * allocator if it is a user mapping. For kernel mappings, clear the
1150 * valid bit to make it clear that the mapping is not currently used.
1153 pmap_free_pte(struct ia64_lpte *pte, vm_offset_t va)
1155 if (va < VM_MAXUSER_ADDRESS)
1156 uma_zfree(ptezone, pte);
1158 pmap_clear_present(pte);
1161 static PMAP_INLINE void
1162 pmap_pte_prot(pmap_t pm, struct ia64_lpte *pte, vm_prot_t prot)
1164 static long prot2ar[4] = {
1165 PTE_AR_R, /* VM_PROT_NONE */
1166 PTE_AR_RW, /* VM_PROT_WRITE */
1167 PTE_AR_RX|PTE_ED, /* VM_PROT_EXECUTE */
1168 PTE_AR_RWX|PTE_ED /* VM_PROT_WRITE|VM_PROT_EXECUTE */
1171 pte->pte &= ~(PTE_PROT_MASK | PTE_PL_MASK | PTE_AR_MASK | PTE_ED);
1172 pte->pte |= (uint64_t)(prot & VM_PROT_ALL) << 56;
1173 pte->pte |= (prot == VM_PROT_NONE || pm == kernel_pmap)
1174 ? PTE_PL_KERN : PTE_PL_USER;
1175 pte->pte |= prot2ar[(prot & VM_PROT_ALL) >> 1];
1179 * Set a pte to contain a valid mapping and enter it in the VHPT. If
1180 * the pte was orginally valid, then its assumed to already be in the
1182 * This functions does not set the protection bits. It's expected
1183 * that those have been set correctly prior to calling this function.
1186 pmap_set_pte(struct ia64_lpte *pte, vm_offset_t va, vm_offset_t pa,
1187 boolean_t wired, boolean_t managed)
1191 pte->pte &= PTE_PROT_MASK | PTE_PL_MASK | PTE_AR_MASK | PTE_ED;
1192 pte->pte |= PTE_PRESENT | PTE_MA_WB;
1193 pte->pte |= (managed) ? PTE_MANAGED : (PTE_DIRTY | PTE_ACCESSED);
1194 pte->pte |= (wired) ? PTE_WIRED : 0;
1195 pte->pte |= pa & PTE_PPN_MASK;
1197 pte->itir = PAGE_SHIFT << 2;
1199 pte->tag = ia64_ttag(va);
1201 /* XXX fc.i kluge (quick fix) */
1202 if (ia64_icache_sync_kluge) {
1203 lim = va + PAGE_SIZE;
1205 __asm __volatile("fc.i %0" :: "r"(va));
1212 * Remove the (possibly managed) mapping represented by pte from the
1216 pmap_remove_pte(pmap_t pmap, struct ia64_lpte *pte, vm_offset_t va,
1217 pv_entry_t pv, int freepte)
1222 KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)),
1223 ("removing pte for non-current pmap"));
1226 * First remove from the VHPT.
1228 error = pmap_remove_vhpt(va);
1232 pmap_invalidate_page(pmap, va);
1234 if (pmap_wired(pte))
1235 pmap->pm_stats.wired_count -= 1;
1237 pmap->pm_stats.resident_count -= 1;
1238 if (pmap_managed(pte)) {
1239 m = PHYS_TO_VM_PAGE(pmap_ppn(pte));
1240 if (pmap_dirty(pte))
1242 if (pmap_accessed(pte))
1243 vm_page_flag_set(m, PG_REFERENCED);
1245 error = pmap_remove_entry(pmap, m, va, pv);
1248 pmap_free_pte(pte, va);
1254 * Extract the physical page address associated with a kernel
1258 pmap_kextract(vm_offset_t va)
1260 struct ia64_lpte *pte;
1263 KASSERT(va >= IA64_RR_BASE(5), ("Must be kernel VA"));
1265 /* Regions 6 and 7 are direct mapped. */
1266 if (va >= IA64_RR_BASE(6))
1267 return (IA64_RR_MASK(va));
1269 /* EPC gateway page? */
1270 gwpage = (vm_offset_t)ia64_get_k5();
1271 if (va >= gwpage && va < gwpage + VM_GATEWAY_SIZE)
1272 return (IA64_RR_MASK((vm_offset_t)ia64_gateway_page));
1274 /* Bail out if the virtual address is beyond our limits. */
1275 if (va >= kernel_vm_end)
1278 pte = pmap_find_kpte(va);
1279 if (!pmap_present(pte))
1281 return (pmap_ppn(pte) | (va & PAGE_MASK));
1285 * Add a list of wired pages to the kva this routine is only used for
1286 * temporary kernel mappings that do not need to have page modification
1287 * or references recorded. Note that old mappings are simply written
1288 * over. The page is effectively wired, but it's customary to not have
1289 * the PTE reflect that, nor update statistics.
1292 pmap_qenter(vm_offset_t va, vm_page_t *m, int count)
1294 struct ia64_lpte *pte;
1297 for (i = 0; i < count; i++) {
1298 pte = pmap_find_kpte(va);
1299 if (pmap_present(pte))
1300 pmap_invalidate_page(kernel_pmap, va);
1302 pmap_enter_vhpt(pte, va);
1303 pmap_pte_prot(kernel_pmap, pte, VM_PROT_ALL);
1304 pmap_set_pte(pte, va, VM_PAGE_TO_PHYS(m[i]), FALSE, FALSE);
1310 * this routine jerks page mappings from the
1311 * kernel -- it is meant only for temporary mappings.
1314 pmap_qremove(vm_offset_t va, int count)
1316 struct ia64_lpte *pte;
1319 for (i = 0; i < count; i++) {
1320 pte = pmap_find_kpte(va);
1321 if (pmap_present(pte)) {
1322 pmap_remove_vhpt(va);
1323 pmap_invalidate_page(kernel_pmap, va);
1324 pmap_clear_present(pte);
1331 * Add a wired page to the kva. As for pmap_qenter(), it's customary
1332 * to not have the PTE reflect that, nor update statistics.
1335 pmap_kenter(vm_offset_t va, vm_offset_t pa)
1337 struct ia64_lpte *pte;
1339 pte = pmap_find_kpte(va);
1340 if (pmap_present(pte))
1341 pmap_invalidate_page(kernel_pmap, va);
1343 pmap_enter_vhpt(pte, va);
1344 pmap_pte_prot(kernel_pmap, pte, VM_PROT_ALL);
1345 pmap_set_pte(pte, va, pa, FALSE, FALSE);
1349 * Remove a page from the kva
1352 pmap_kremove(vm_offset_t va)
1354 struct ia64_lpte *pte;
1356 pte = pmap_find_kpte(va);
1357 if (pmap_present(pte)) {
1358 pmap_remove_vhpt(va);
1359 pmap_invalidate_page(kernel_pmap, va);
1360 pmap_clear_present(pte);
1365 * Used to map a range of physical addresses into kernel
1366 * virtual address space.
1368 * The value passed in '*virt' is a suggested virtual address for
1369 * the mapping. Architectures which can support a direct-mapped
1370 * physical to virtual region can return the appropriate address
1371 * within that region, leaving '*virt' unchanged. Other
1372 * architectures should map the pages starting at '*virt' and
1373 * update '*virt' with the first usable address after the mapped
1377 pmap_map(vm_offset_t *virt, vm_offset_t start, vm_offset_t end, int prot)
1379 return IA64_PHYS_TO_RR7(start);
1383 * Remove a single page from a process address space
1386 pmap_remove_page(pmap_t pmap, vm_offset_t va)
1388 struct ia64_lpte *pte;
1390 KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)),
1391 ("removing page for non-current pmap"));
1393 pte = pmap_find_vhpt(va);
1395 pmap_remove_pte(pmap, pte, va, 0, 1);
1400 * Remove the given range of addresses from the specified map.
1402 * It is assumed that the start and end are properly
1403 * rounded to the page size.
1406 pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
1411 struct ia64_lpte *pte;
1413 if (pmap->pm_stats.resident_count == 0)
1416 vm_page_lock_queues();
1418 oldpmap = pmap_switch(pmap);
1421 * special handling of removing one page. a very
1422 * common operation and easy to short circuit some
1425 if (sva + PAGE_SIZE == eva) {
1426 pmap_remove_page(pmap, sva);
1430 if (pmap->pm_stats.resident_count < ((eva - sva) >> PAGE_SHIFT)) {
1431 TAILQ_FOREACH_SAFE(pv, &pmap->pm_pvlist, pv_plist, npv) {
1433 if (va >= sva && va < eva) {
1434 pte = pmap_find_vhpt(va);
1435 KASSERT(pte != NULL, ("pte"));
1436 pmap_remove_pte(pmap, pte, va, pv, 1);
1440 for (va = sva; va < eva; va += PAGE_SIZE) {
1441 pte = pmap_find_vhpt(va);
1443 pmap_remove_pte(pmap, pte, va, 0, 1);
1448 vm_page_unlock_queues();
1449 pmap_switch(oldpmap);
1454 * Routine: pmap_remove_all
1456 * Removes this physical page from
1457 * all physical maps in which it resides.
1458 * Reflects back modify bits to the pager.
1461 * Original versions of this routine were very
1462 * inefficient because they iteratively called
1463 * pmap_remove (slow...)
1467 pmap_remove_all(vm_page_t m)
1472 #if defined(DIAGNOSTIC)
1474 * XXX This makes pmap_remove_all() illegal for non-managed pages!
1476 if (m->flags & PG_FICTITIOUS) {
1477 panic("pmap_remove_all: illegal for unmanaged page, va: 0x%lx", VM_PAGE_TO_PHYS(m));
1480 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1481 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
1482 struct ia64_lpte *pte;
1483 pmap_t pmap = pv->pv_pmap;
1484 vm_offset_t va = pv->pv_va;
1487 oldpmap = pmap_switch(pmap);
1488 pte = pmap_find_vhpt(va);
1489 KASSERT(pte != NULL, ("pte"));
1490 if (pmap_ppn(pte) != VM_PAGE_TO_PHYS(m))
1491 panic("pmap_remove_all: pv_table for %lx is inconsistent", VM_PAGE_TO_PHYS(m));
1492 pmap_remove_pte(pmap, pte, va, pv, 1);
1493 pmap_switch(oldpmap);
1496 vm_page_flag_clear(m, PG_WRITEABLE);
1500 * Set the physical protection on the
1501 * specified range of this map as requested.
1504 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
1507 struct ia64_lpte *pte;
1509 if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
1510 pmap_remove(pmap, sva, eva);
1514 if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) ==
1515 (VM_PROT_WRITE|VM_PROT_EXECUTE))
1518 if ((sva & PAGE_MASK) || (eva & PAGE_MASK))
1519 panic("pmap_protect: unaligned addresses");
1521 vm_page_lock_queues();
1523 oldpmap = pmap_switch(pmap);
1526 * If page is invalid, skip this page
1528 pte = pmap_find_vhpt(sva);
1534 if (pmap_prot(pte) != prot) {
1535 if (pmap_managed(pte)) {
1536 vm_offset_t pa = pmap_ppn(pte);
1537 vm_page_t m = PHYS_TO_VM_PAGE(pa);
1538 if (pmap_dirty(pte)) {
1540 pmap_clear_dirty(pte);
1542 if (pmap_accessed(pte)) {
1543 vm_page_flag_set(m, PG_REFERENCED);
1544 pmap_clear_accessed(pte);
1547 pmap_pte_prot(pmap, pte, prot);
1548 pmap_invalidate_page(pmap, sva);
1553 vm_page_unlock_queues();
1554 pmap_switch(oldpmap);
1559 * Insert the given physical page (p) at
1560 * the specified virtual address (v) in the
1561 * target physical map with the protection requested.
1563 * If specified, the page will be wired down, meaning
1564 * that the related pte can not be reclaimed.
1566 * NB: This is the only routine which MAY NOT lazy-evaluate
1567 * or lose information. That is, this routine must actually
1568 * insert this page into the given map NOW.
1571 pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t m,
1572 vm_prot_t prot, boolean_t wired)
1577 struct ia64_lpte origpte;
1578 struct ia64_lpte *pte;
1581 vm_page_lock_queues();
1583 oldpmap = pmap_switch(pmap);
1587 if (va > VM_MAX_KERNEL_ADDRESS)
1588 panic("pmap_enter: toobig");
1592 * Find (or create) a pte for the given mapping.
1594 while ((pte = pmap_find_pte(va)) == NULL) {
1595 pmap_switch(oldpmap);
1597 vm_page_unlock_queues();
1599 vm_page_lock_queues();
1601 oldpmap = pmap_switch(pmap);
1604 if (!pmap_present(pte)) {
1606 pmap_enter_vhpt(pte, va);
1608 opa = pmap_ppn(pte);
1610 pa = VM_PAGE_TO_PHYS(m);
1613 * Mapping has not changed, must be protection or wiring change.
1617 * Wiring change, just update stats. We don't worry about
1618 * wiring PT pages as they remain resident as long as there
1619 * are valid mappings in them. Hence, if a user page is wired,
1620 * the PT page will be also.
1622 if (wired && !pmap_wired(&origpte))
1623 pmap->pm_stats.wired_count++;
1624 else if (!wired && pmap_wired(&origpte))
1625 pmap->pm_stats.wired_count--;
1627 managed = (pmap_managed(&origpte)) ? TRUE : FALSE;
1630 * We might be turning off write access to the page,
1631 * so we go ahead and sense modify status.
1633 if (managed && pmap_dirty(&origpte))
1636 pmap_invalidate_page(pmap, va);
1641 * Mapping has changed, invalidate old range and fall
1642 * through to handle validating new mapping.
1645 pmap_remove_pte(pmap, pte, va, 0, 0);
1646 pmap_enter_vhpt(pte, va);
1650 * Enter on the PV list if part of our managed memory.
1652 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) {
1653 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva,
1654 ("pmap_enter: managed mapping within the clean submap"));
1655 pmap_insert_entry(pmap, va, m);
1660 * Increment counters
1662 pmap->pm_stats.resident_count++;
1664 pmap->pm_stats.wired_count++;
1669 * Now validate mapping with desired protection/wiring. This
1670 * adds the pte to the VHPT if necessary.
1672 pmap_pte_prot(pmap, pte, prot);
1673 pmap_set_pte(pte, va, pa, wired, managed);
1675 if ((prot & VM_PROT_WRITE) != 0)
1676 vm_page_flag_set(m, PG_WRITEABLE);
1677 vm_page_unlock_queues();
1678 pmap_switch(oldpmap);
1683 * Maps a sequence of resident pages belonging to the same object.
1684 * The sequence begins with the given page m_start. This page is
1685 * mapped at the given virtual address start. Each subsequent page is
1686 * mapped at a virtual address that is offset from start by the same
1687 * amount as the page is offset from m_start within the object. The
1688 * last page in the sequence is the page with the largest offset from
1689 * m_start that can be mapped at a virtual address less than the given
1690 * virtual address end. Not every virtual page between start and end
1691 * is mapped; only those for which a resident page exists with the
1692 * corresponding offset from m_start are mapped.
1695 pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,
1696 vm_page_t m_start, vm_prot_t prot)
1700 vm_pindex_t diff, psize;
1702 VM_OBJECT_LOCK_ASSERT(m_start->object, MA_OWNED);
1703 psize = atop(end - start);
1706 oldpmap = pmap_switch(pmap);
1707 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
1708 pmap_enter_quick_locked(pmap, start + ptoa(diff), m, prot);
1709 m = TAILQ_NEXT(m, listq);
1711 pmap_switch(oldpmap);
1716 * this code makes some *MAJOR* assumptions:
1717 * 1. Current pmap & pmap exists.
1720 * 4. No page table pages.
1721 * but is *MUCH* faster than pmap_enter...
1725 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
1730 oldpmap = pmap_switch(pmap);
1731 pmap_enter_quick_locked(pmap, va, m, prot);
1732 pmap_switch(oldpmap);
1737 pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
1740 struct ia64_lpte *pte;
1743 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva ||
1744 (m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0,
1745 ("pmap_enter_quick_locked: managed mapping within the clean submap"));
1746 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1747 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1749 if ((pte = pmap_find_pte(va)) == NULL)
1752 if (!pmap_present(pte)) {
1753 /* Enter on the PV list if the page is managed. */
1754 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) {
1755 if (!pmap_try_insert_pv_entry(pmap, va, m)) {
1756 pmap_free_pte(pte, va);
1763 /* Increment counters. */
1764 pmap->pm_stats.resident_count++;
1766 /* Initialise with R/O protection and enter into VHPT. */
1767 pmap_enter_vhpt(pte, va);
1768 pmap_pte_prot(pmap, pte,
1769 prot & (VM_PROT_READ | VM_PROT_EXECUTE));
1770 pmap_set_pte(pte, va, VM_PAGE_TO_PHYS(m), FALSE, managed);
1775 * pmap_object_init_pt preloads the ptes for a given object
1776 * into the specified pmap. This eliminates the blast of soft
1777 * faults on process startup and immediately after an mmap.
1780 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr,
1781 vm_object_t object, vm_pindex_t pindex,
1785 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1786 KASSERT(object->type == OBJT_DEVICE,
1787 ("pmap_object_init_pt: non-device object"));
1791 * Routine: pmap_change_wiring
1792 * Function: Change the wiring attribute for a map/virtual-address
1794 * In/out conditions:
1795 * The mapping must already exist in the pmap.
1798 pmap_change_wiring(pmap, va, wired)
1799 register pmap_t pmap;
1804 struct ia64_lpte *pte;
1807 oldpmap = pmap_switch(pmap);
1809 pte = pmap_find_vhpt(va);
1810 KASSERT(pte != NULL, ("pte"));
1811 if (wired && !pmap_wired(pte)) {
1812 pmap->pm_stats.wired_count++;
1813 pmap_set_wired(pte);
1814 } else if (!wired && pmap_wired(pte)) {
1815 pmap->pm_stats.wired_count--;
1816 pmap_clear_wired(pte);
1819 pmap_switch(oldpmap);
1826 * Copy the range specified by src_addr/len
1827 * from the source map to the range dst_addr/len
1828 * in the destination map.
1830 * This routine is only advisory and need not do anything.
1834 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
1835 vm_offset_t src_addr)
1841 * pmap_zero_page zeros the specified hardware page by
1842 * mapping it into virtual memory and using bzero to clear
1847 pmap_zero_page(vm_page_t m)
1849 vm_offset_t va = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m));
1850 bzero((caddr_t) va, PAGE_SIZE);
1855 * pmap_zero_page_area zeros the specified hardware page by
1856 * mapping it into virtual memory and using bzero to clear
1859 * off and size must reside within a single page.
1863 pmap_zero_page_area(vm_page_t m, int off, int size)
1865 vm_offset_t va = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m));
1866 bzero((char *)(caddr_t)va + off, size);
1871 * pmap_zero_page_idle zeros the specified hardware page by
1872 * mapping it into virtual memory and using bzero to clear
1873 * its contents. This is for the vm_idlezero process.
1877 pmap_zero_page_idle(vm_page_t m)
1879 vm_offset_t va = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m));
1880 bzero((caddr_t) va, PAGE_SIZE);
1885 * pmap_copy_page copies the specified (machine independent)
1886 * page by mapping the page into virtual memory and using
1887 * bcopy to copy the page, one machine dependent page at a
1891 pmap_copy_page(vm_page_t msrc, vm_page_t mdst)
1893 vm_offset_t src = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(msrc));
1894 vm_offset_t dst = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(mdst));
1895 bcopy((caddr_t) src, (caddr_t) dst, PAGE_SIZE);
1899 * Returns true if the pmap's pv is one of the first
1900 * 16 pvs linked to from this page. This count may
1901 * be changed upwards or downwards in the future; it
1902 * is only necessary that true be returned for a small
1903 * subset of pmaps for proper page aging.
1906 pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
1911 if (m->flags & PG_FICTITIOUS)
1915 * Not found, check current mappings returning immediately if found.
1917 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1918 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
1919 if (pv->pv_pmap == pmap) {
1930 * pmap_page_wired_mappings:
1932 * Return the number of managed mappings to the given physical page
1936 pmap_page_wired_mappings(vm_page_t m)
1938 struct ia64_lpte *pte;
1939 pmap_t oldpmap, pmap;
1944 if ((m->flags & PG_FICTITIOUS) != 0)
1946 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1947 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
1950 oldpmap = pmap_switch(pmap);
1951 pte = pmap_find_vhpt(pv->pv_va);
1952 KASSERT(pte != NULL, ("pte"));
1953 if (pmap_wired(pte))
1955 pmap_switch(oldpmap);
1962 * Remove all pages from specified address space
1963 * this aids process exit speeds. Also, this code
1964 * is special cased for current process only, but
1965 * can have the more generic (and slightly slower)
1966 * mode enabled. This is much faster than pmap_remove
1967 * in the case of running down an entire address space.
1970 pmap_remove_pages(pmap_t pmap)
1975 if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace)) {
1976 printf("warning: pmap_remove_pages called with non-current pmap\n");
1980 vm_page_lock_queues();
1982 oldpmap = pmap_switch(pmap);
1984 for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) {
1985 struct ia64_lpte *pte;
1987 npv = TAILQ_NEXT(pv, pv_plist);
1989 pte = pmap_find_vhpt(pv->pv_va);
1990 KASSERT(pte != NULL, ("pte"));
1991 if (!pmap_wired(pte))
1992 pmap_remove_pte(pmap, pte, pv->pv_va, pv, 1);
1995 pmap_switch(oldpmap);
1997 vm_page_unlock_queues();
2001 * pmap_ts_referenced:
2003 * Return a count of reference bits for a page, clearing those bits.
2004 * It is not necessary for every reference bit to be cleared, but it
2005 * is necessary that 0 only be returned when there are truly no
2006 * reference bits set.
2008 * XXX: The exact number of bits to check and clear is a matter that
2009 * should be tested and standardized at some point in the future for
2010 * optimal aging of shared pages.
2013 pmap_ts_referenced(vm_page_t m)
2015 struct ia64_lpte *pte;
2020 if (m->flags & PG_FICTITIOUS)
2023 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2024 PMAP_LOCK(pv->pv_pmap);
2025 oldpmap = pmap_switch(pv->pv_pmap);
2026 pte = pmap_find_vhpt(pv->pv_va);
2027 KASSERT(pte != NULL, ("pte"));
2028 if (pmap_accessed(pte)) {
2030 pmap_clear_accessed(pte);
2031 pmap_invalidate_page(pv->pv_pmap, pv->pv_va);
2033 pmap_switch(oldpmap);
2034 PMAP_UNLOCK(pv->pv_pmap);
2043 * Return whether or not the specified physical page was modified
2044 * in any physical maps.
2047 pmap_is_modified(vm_page_t m)
2049 struct ia64_lpte *pte;
2055 if (m->flags & PG_FICTITIOUS)
2058 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2059 PMAP_LOCK(pv->pv_pmap);
2060 oldpmap = pmap_switch(pv->pv_pmap);
2061 pte = pmap_find_vhpt(pv->pv_va);
2062 pmap_switch(oldpmap);
2063 KASSERT(pte != NULL, ("pte"));
2064 rv = pmap_dirty(pte) ? TRUE : FALSE;
2065 PMAP_UNLOCK(pv->pv_pmap);
2074 * pmap_is_prefaultable:
2076 * Return whether or not the specified virtual address is elgible
2080 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)
2082 struct ia64_lpte *pte;
2084 pte = pmap_find_vhpt(addr);
2085 if (pte != NULL && pmap_present(pte))
2091 * Clear the modify bits on the specified physical page.
2094 pmap_clear_modify(vm_page_t m)
2096 struct ia64_lpte *pte;
2100 if (m->flags & PG_FICTITIOUS)
2103 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2104 PMAP_LOCK(pv->pv_pmap);
2105 oldpmap = pmap_switch(pv->pv_pmap);
2106 pte = pmap_find_vhpt(pv->pv_va);
2107 KASSERT(pte != NULL, ("pte"));
2108 if (pmap_dirty(pte)) {
2109 pmap_clear_dirty(pte);
2110 pmap_invalidate_page(pv->pv_pmap, pv->pv_va);
2112 pmap_switch(oldpmap);
2113 PMAP_UNLOCK(pv->pv_pmap);
2118 * pmap_clear_reference:
2120 * Clear the reference bit on the specified physical page.
2123 pmap_clear_reference(vm_page_t m)
2125 struct ia64_lpte *pte;
2129 if (m->flags & PG_FICTITIOUS)
2132 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2133 PMAP_LOCK(pv->pv_pmap);
2134 oldpmap = pmap_switch(pv->pv_pmap);
2135 pte = pmap_find_vhpt(pv->pv_va);
2136 KASSERT(pte != NULL, ("pte"));
2137 if (pmap_accessed(pte)) {
2138 pmap_clear_accessed(pte);
2139 pmap_invalidate_page(pv->pv_pmap, pv->pv_va);
2141 pmap_switch(oldpmap);
2142 PMAP_UNLOCK(pv->pv_pmap);
2147 * Clear the write and modified bits in each of the given page's mappings.
2150 pmap_remove_write(vm_page_t m)
2152 struct ia64_lpte *pte;
2153 pmap_t oldpmap, pmap;
2157 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
2158 if ((m->flags & PG_FICTITIOUS) != 0 ||
2159 (m->flags & PG_WRITEABLE) == 0)
2161 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2164 oldpmap = pmap_switch(pmap);
2165 pte = pmap_find_vhpt(pv->pv_va);
2166 KASSERT(pte != NULL, ("pte"));
2167 prot = pmap_prot(pte);
2168 if ((prot & VM_PROT_WRITE) != 0) {
2169 if (pmap_dirty(pte)) {
2171 pmap_clear_dirty(pte);
2173 prot &= ~VM_PROT_WRITE;
2174 pmap_pte_prot(pmap, pte, prot);
2175 pmap_invalidate_page(pmap, pv->pv_va);
2177 pmap_switch(oldpmap);
2180 vm_page_flag_clear(m, PG_WRITEABLE);
2184 * Map a set of physical memory pages into the kernel virtual
2185 * address space. Return a pointer to where it is mapped. This
2186 * routine is intended to be used for mapping device memory,
2190 pmap_mapdev(vm_offset_t pa, vm_size_t size)
2192 return (void*) IA64_PHYS_TO_RR6(pa);
2196 * 'Unmap' a range mapped by pmap_mapdev().
2199 pmap_unmapdev(vm_offset_t va, vm_size_t size)
2205 * perform the pmap work for mincore
2208 pmap_mincore(pmap_t pmap, vm_offset_t addr)
2211 struct ia64_lpte *pte, tpte;
2215 oldpmap = pmap_switch(pmap);
2216 pte = pmap_find_vhpt(addr);
2221 pmap_switch(oldpmap);
2227 if (pmap_present(pte)) {
2231 val = MINCORE_INCORE;
2232 if (!pmap_managed(pte))
2237 m = PHYS_TO_VM_PAGE(pa);
2242 if (pmap_dirty(pte))
2243 val |= MINCORE_MODIFIED|MINCORE_MODIFIED_OTHER;
2246 * Modified by someone
2248 vm_page_lock_queues();
2249 if (pmap_is_modified(m))
2250 val |= MINCORE_MODIFIED_OTHER;
2251 vm_page_unlock_queues();
2256 if (pmap_accessed(pte))
2257 val |= MINCORE_REFERENCED|MINCORE_REFERENCED_OTHER;
2260 * Referenced by someone
2262 vm_page_lock_queues();
2263 if (pmap_ts_referenced(m)) {
2264 val |= MINCORE_REFERENCED_OTHER;
2265 vm_page_flag_set(m, PG_REFERENCED);
2267 vm_page_unlock_queues();
2274 pmap_activate(struct thread *td)
2276 pmap_switch(vmspace_pmap(td->td_proc->p_vmspace));
2280 pmap_switch(pmap_t pm)
2286 prevpm = PCPU_GET(current_pmap);
2290 atomic_clear_32(&prevpm->pm_active, PCPU_GET(cpumask));
2292 for (i = 0; i < 5; i++) {
2293 ia64_set_rr(IA64_RR_BASE(i),
2294 (i << 8)|(PAGE_SHIFT << 2)|1);
2297 for (i = 0; i < 5; i++) {
2298 ia64_set_rr(IA64_RR_BASE(i),
2299 (pm->pm_rid[i] << 8)|(PAGE_SHIFT << 2)|1);
2301 atomic_set_32(&pm->pm_active, PCPU_GET(cpumask));
2303 PCPU_SET(current_pmap, pm);
2312 pmap_addr_hint(vm_object_t obj, vm_offset_t addr, vm_size_t size)
2318 #include "opt_ddb.h"
2322 #include <ddb/ddb.h>
2324 static const char* psnames[] = {
2325 "1B", "2B", "4B", "8B",
2326 "16B", "32B", "64B", "128B",
2327 "256B", "512B", "1K", "2K",
2328 "4K", "8K", "16K", "32K",
2329 "64K", "128K", "256K", "512K",
2330 "1M", "2M", "4M", "8M",
2331 "16M", "32M", "64M", "128M",
2332 "256M", "512M", "1G", "2G"
2338 struct ia64_pal_result res;
2346 static const char *manames[] = {
2347 "WB", "bad", "bad", "bad",
2348 "UC", "UCE", "WC", "NaT",
2351 res = ia64_call_pal_static(PAL_VM_SUMMARY, 0, 0, 0);
2352 if (res.pal_status != 0) {
2353 db_printf("Can't get VM summary\n");
2358 maxtr = (res.pal_result[0] >> 40) & 0xff;
2360 maxtr = (res.pal_result[0] >> 32) & 0xff;
2362 db_printf("V RID Virtual Page Physical Page PgSz ED AR PL D A MA P KEY\n");
2363 for (i = 0; i <= maxtr; i++) {
2364 bzero(&buf, sizeof(buf));
2365 res = ia64_call_pal_stacked_physical
2366 (PAL_VM_TR_READ, i, type, ia64_tpa((uint64_t) &buf));
2367 if (!(res.pal_result[0] & 1))
2368 buf.pte &= ~PTE_AR_MASK;
2369 if (!(res.pal_result[0] & 2))
2370 buf.pte &= ~PTE_PL_MASK;
2371 if (!(res.pal_result[0] & 4))
2372 pmap_clear_dirty(&buf);
2373 if (!(res.pal_result[0] & 8))
2374 buf.pte &= ~PTE_MA_MASK;
2375 db_printf("%d %06x %013lx %013lx %4s %d %d %d %d %d %-3s "
2376 "%d %06x\n", (int)buf.ifa & 1, buf.rr.rr_rid,
2377 buf.ifa >> 12, (buf.pte & PTE_PPN_MASK) >> 12,
2378 psnames[(buf.itir & ITIR_PS_MASK) >> 2],
2379 (buf.pte & PTE_ED) ? 1 : 0,
2380 (int)(buf.pte & PTE_AR_MASK) >> 9,
2381 (int)(buf.pte & PTE_PL_MASK) >> 7,
2382 (pmap_dirty(&buf)) ? 1 : 0,
2383 (pmap_accessed(&buf)) ? 1 : 0,
2384 manames[(buf.pte & PTE_MA_MASK) >> 2],
2385 (pmap_present(&buf)) ? 1 : 0,
2386 (int)((buf.itir & ITIR_KEY_MASK) >> 8));
2390 DB_COMMAND(itr, db_itr)
2395 DB_COMMAND(dtr, db_dtr)
2400 DB_COMMAND(rr, db_rr)
2406 printf("RR RID PgSz VE\n");
2407 for (i = 0; i < 8; i++) {
2408 __asm __volatile ("mov %0=rr[%1]"
2410 : "r"(IA64_RR_BASE(i)));
2411 *(uint64_t *) &rr = t;
2412 printf("%d %06x %4s %d\n",
2413 i, rr.rr_rid, psnames[rr.rr_ps], rr.rr_ve);
2417 DB_COMMAND(thash, db_thash)
2422 db_printf("%p\n", (void *) ia64_thash(addr));
2425 DB_COMMAND(ttag, db_ttag)
2430 db_printf("0x%lx\n", ia64_ttag(addr));
2433 DB_COMMAND(kpte, db_kpte)
2435 struct ia64_lpte *pte;
2438 db_printf("usage: kpte <kva>\n");
2441 if (addr < VM_MIN_KERNEL_ADDRESS) {
2442 db_printf("kpte: error: invalid <kva>\n");
2445 pte = pmap_find_kpte(addr);
2446 db_printf("kpte at %p:\n", pte);
2447 db_printf(" pte =%016lx\n", pte->pte);
2448 db_printf(" itir =%016lx\n", pte->itir);
2449 db_printf(" tag =%016lx\n", pte->tag);
2450 db_printf(" chain=%016lx\n", pte->chain);