2 * Copyright (c) 1991 Regents of the University of California.
4 * Copyright (c) 1994 John S. Dyson
6 * Copyright (c) 1994 David Greenman
9 * This code is derived from software contributed to Berkeley by
10 * the Systems Programming Group of the University of Utah Computer
11 * Science Department and William Jolitz of UUNET Technologies Inc.
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 * 3. All advertising materials mentioning features or use of this software
22 * must display the following acknowledgement:
23 * This product includes software developed by the University of
24 * California, Berkeley and its contributors.
25 * 4. Neither the name of the University nor the names of its contributors
26 * may be used to endorse or promote products derived from this software
27 * without specific prior written permission.
29 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
30 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
31 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
32 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
33 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
34 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
35 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
36 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
37 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
38 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
41 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91
46 * Manages physical address maps.
48 * In addition to hardware address maps, this module is called upon to
49 * provide software-use-only maps which may or may not be stored in the
50 * same form as hardware maps. These pseudo-maps are used to store
51 * intermediate results from copy operations to and from address spaces.
53 * Since the information managed by this module is also stored by the
54 * logical address mapping module, this module may throw away valid virtual
55 * to physical mappings at almost any time. However, invalidations of
56 * mappings must be done as requested.
58 * In order to cope with hardware architectures which make virtual to
59 * physical map invalidates expensive, this module may delay invalidate
60 * reduced protection operations until such time as they are actually
61 * necessary. This module is given full information as to which processors
62 * are currently using which maps, and to when physical maps must be made
66 #include "opt_kstack_pages.h"
67 #include "opt_msgbuf.h"
70 #include <sys/param.h>
71 #include <sys/kernel.h>
74 #include <sys/msgbuf.h>
75 #include <sys/mutex.h>
78 #include <sys/sysctl.h>
79 #include <sys/systm.h>
80 #include <sys/vmmeter.h>
82 #include <dev/ofw/openfirm.h>
85 #include <vm/vm_param.h>
86 #include <vm/vm_kern.h>
87 #include <vm/vm_page.h>
88 #include <vm/vm_map.h>
89 #include <vm/vm_object.h>
90 #include <vm/vm_extern.h>
91 #include <vm/vm_pageout.h>
92 #include <vm/vm_pager.h>
94 #include <machine/cache.h>
95 #include <machine/frame.h>
96 #include <machine/instr.h>
97 #include <machine/md_var.h>
98 #include <machine/metadata.h>
99 #include <machine/ofw_mem.h>
100 #include <machine/smp.h>
101 #include <machine/tlb.h>
102 #include <machine/tte.h>
103 #include <machine/tsb.h>
107 #ifndef PMAP_SHPGPERPROC
108 #define PMAP_SHPGPERPROC 200
112 #include "opt_sched.h"
114 #error "sparc64 only works with SCHED_4BSD which uses a global scheduler lock."
116 extern struct mtx sched_lock;
119 * Virtual and physical address of message buffer.
121 struct msgbuf *msgbufp;
122 vm_paddr_t msgbuf_phys;
125 * Map of physical memory reagions.
127 vm_paddr_t phys_avail[128];
128 static struct ofw_mem_region mra[128];
129 struct ofw_mem_region sparc64_memreg[128];
131 static struct ofw_map translations[128];
132 static int translations_size;
134 static vm_offset_t pmap_idle_map;
135 static vm_offset_t pmap_temp_map_1;
136 static vm_offset_t pmap_temp_map_2;
139 * First and last available kernel virtual addresses.
141 vm_offset_t virtual_avail;
142 vm_offset_t virtual_end;
143 vm_offset_t kernel_vm_end;
145 vm_offset_t vm_max_kernel_address;
150 struct pmap kernel_pmap_store;
153 * Allocate physical memory for use in pmap_bootstrap.
155 static vm_paddr_t pmap_bootstrap_alloc(vm_size_t size);
158 * Map the given physical page at the specified virtual address in the
159 * target pmap with the protection requested. If specified the page
160 * will be wired down.
162 * The page queues and pmap must be locked.
164 static void pmap_enter_locked(pmap_t pm, vm_offset_t va, vm_page_t m,
165 vm_prot_t prot, boolean_t wired);
167 extern int tl1_immu_miss_patch_1[];
168 extern int tl1_immu_miss_patch_2[];
169 extern int tl1_dmmu_miss_patch_1[];
170 extern int tl1_dmmu_miss_patch_2[];
171 extern int tl1_dmmu_prot_patch_1[];
172 extern int tl1_dmmu_prot_patch_2[];
175 * If user pmap is processed with pmap_remove and with pmap_remove and the
176 * resident count drops to 0, there are no more pages to remove, so we
179 #define PMAP_REMOVE_DONE(pm) \
180 ((pm) != kernel_pmap && (pm)->pm_stats.resident_count == 0)
183 * The threshold (in bytes) above which tsb_foreach() is used in pmap_remove()
184 * and pmap_protect() instead of trying each virtual address.
186 #define PMAP_TSB_THRESH ((TSB_SIZE / 2) * PAGE_SIZE)
188 SYSCTL_NODE(_debug, OID_AUTO, pmap_stats, CTLFLAG_RD, 0, "");
190 PMAP_STATS_VAR(pmap_nenter);
191 PMAP_STATS_VAR(pmap_nenter_update);
192 PMAP_STATS_VAR(pmap_nenter_replace);
193 PMAP_STATS_VAR(pmap_nenter_new);
194 PMAP_STATS_VAR(pmap_nkenter);
195 PMAP_STATS_VAR(pmap_nkenter_oc);
196 PMAP_STATS_VAR(pmap_nkenter_stupid);
197 PMAP_STATS_VAR(pmap_nkremove);
198 PMAP_STATS_VAR(pmap_nqenter);
199 PMAP_STATS_VAR(pmap_nqremove);
200 PMAP_STATS_VAR(pmap_ncache_enter);
201 PMAP_STATS_VAR(pmap_ncache_enter_c);
202 PMAP_STATS_VAR(pmap_ncache_enter_oc);
203 PMAP_STATS_VAR(pmap_ncache_enter_cc);
204 PMAP_STATS_VAR(pmap_ncache_enter_coc);
205 PMAP_STATS_VAR(pmap_ncache_enter_nc);
206 PMAP_STATS_VAR(pmap_ncache_enter_cnc);
207 PMAP_STATS_VAR(pmap_ncache_remove);
208 PMAP_STATS_VAR(pmap_ncache_remove_c);
209 PMAP_STATS_VAR(pmap_ncache_remove_oc);
210 PMAP_STATS_VAR(pmap_ncache_remove_cc);
211 PMAP_STATS_VAR(pmap_ncache_remove_coc);
212 PMAP_STATS_VAR(pmap_ncache_remove_nc);
213 PMAP_STATS_VAR(pmap_nzero_page);
214 PMAP_STATS_VAR(pmap_nzero_page_c);
215 PMAP_STATS_VAR(pmap_nzero_page_oc);
216 PMAP_STATS_VAR(pmap_nzero_page_nc);
217 PMAP_STATS_VAR(pmap_nzero_page_area);
218 PMAP_STATS_VAR(pmap_nzero_page_area_c);
219 PMAP_STATS_VAR(pmap_nzero_page_area_oc);
220 PMAP_STATS_VAR(pmap_nzero_page_area_nc);
221 PMAP_STATS_VAR(pmap_nzero_page_idle);
222 PMAP_STATS_VAR(pmap_nzero_page_idle_c);
223 PMAP_STATS_VAR(pmap_nzero_page_idle_oc);
224 PMAP_STATS_VAR(pmap_nzero_page_idle_nc);
225 PMAP_STATS_VAR(pmap_ncopy_page);
226 PMAP_STATS_VAR(pmap_ncopy_page_c);
227 PMAP_STATS_VAR(pmap_ncopy_page_oc);
228 PMAP_STATS_VAR(pmap_ncopy_page_nc);
229 PMAP_STATS_VAR(pmap_ncopy_page_dc);
230 PMAP_STATS_VAR(pmap_ncopy_page_doc);
231 PMAP_STATS_VAR(pmap_ncopy_page_sc);
232 PMAP_STATS_VAR(pmap_ncopy_page_soc);
234 PMAP_STATS_VAR(pmap_nnew_thread);
235 PMAP_STATS_VAR(pmap_nnew_thread_oc);
238 * Quick sort callout for comparing memory regions.
240 static int mr_cmp(const void *a, const void *b);
241 static int om_cmp(const void *a, const void *b);
243 mr_cmp(const void *a, const void *b)
245 const struct ofw_mem_region *mra;
246 const struct ofw_mem_region *mrb;
250 if (mra->mr_start < mrb->mr_start)
252 else if (mra->mr_start > mrb->mr_start)
258 om_cmp(const void *a, const void *b)
260 const struct ofw_map *oma;
261 const struct ofw_map *omb;
265 if (oma->om_start < omb->om_start)
267 else if (oma->om_start > omb->om_start)
274 * Bootstrap the system enough to run with virtual memory.
277 pmap_bootstrap(vm_offset_t ekva)
293 * Find out what physical memory is available from the prom and
294 * initialize the phys_avail array. This must be done before
295 * pmap_bootstrap_alloc is called.
297 if ((pmem = OF_finddevice("/memory")) == -1)
298 panic("pmap_bootstrap: finddevice /memory");
299 if ((sz = OF_getproplen(pmem, "available")) == -1)
300 panic("pmap_bootstrap: getproplen /memory/available");
301 if (sizeof(phys_avail) < sz)
302 panic("pmap_bootstrap: phys_avail too small");
303 if (sizeof(mra) < sz)
304 panic("pmap_bootstrap: mra too small");
306 if (OF_getprop(pmem, "available", mra, sz) == -1)
307 panic("pmap_bootstrap: getprop /memory/available");
309 CTR0(KTR_PMAP, "pmap_bootstrap: physical memory");
310 qsort(mra, sz, sizeof (*mra), mr_cmp);
312 getenv_quad("hw.physmem", &physmem);
313 physmem = btoc(physmem);
314 for (i = 0, j = 0; i < sz; i++, j += 2) {
315 CTR2(KTR_PMAP, "start=%#lx size=%#lx", mra[i].mr_start,
317 if (physmem != 0 && btoc(physsz + mra[i].mr_size) >= physmem) {
318 if (btoc(physsz) < physmem) {
319 phys_avail[j] = mra[i].mr_start;
320 phys_avail[j + 1] = mra[i].mr_start +
321 (ctob(physmem) - physsz);
322 physsz = ctob(physmem);
326 phys_avail[j] = mra[i].mr_start;
327 phys_avail[j + 1] = mra[i].mr_start + mra[i].mr_size;
328 physsz += mra[i].mr_size;
330 physmem = btoc(physsz);
333 * Calculate the size of kernel virtual memory, and the size and mask
334 * for the kernel tsb.
336 virtsz = roundup(physsz, PAGE_SIZE_4M << (PAGE_SHIFT - TTE_SHIFT));
337 vm_max_kernel_address = VM_MIN_KERNEL_ADDRESS + virtsz;
338 tsb_kernel_size = virtsz >> (PAGE_SHIFT - TTE_SHIFT);
339 tsb_kernel_mask = (tsb_kernel_size >> TTE_SHIFT) - 1;
342 * Allocate the kernel tsb and lock it in the tlb.
344 pa = pmap_bootstrap_alloc(tsb_kernel_size);
345 if (pa & PAGE_MASK_4M)
346 panic("pmap_bootstrap: tsb unaligned\n");
347 tsb_kernel_phys = pa;
348 tsb_kernel = (struct tte *)(VM_MIN_KERNEL_ADDRESS - tsb_kernel_size);
350 bzero(tsb_kernel, tsb_kernel_size);
353 * Allocate and map the message buffer.
355 msgbuf_phys = pmap_bootstrap_alloc(MSGBUF_SIZE);
356 msgbufp = (struct msgbuf *)TLB_PHYS_TO_DIRECT(msgbuf_phys);
359 * Patch the virtual address and the tsb mask into the trap table.
362 #define SETHI(rd, imm22) \
363 (EIF_OP(IOP_FORM2) | EIF_F2_RD(rd) | EIF_F2_OP2(INS0_SETHI) | \
364 EIF_IMM((imm22) >> 10, 22))
365 #define OR_R_I_R(rd, imm13, rs1) \
366 (EIF_OP(IOP_MISC) | EIF_F3_RD(rd) | EIF_F3_OP3(INS2_OR) | \
367 EIF_F3_RS1(rs1) | EIF_F3_I(1) | EIF_IMM(imm13, 13))
369 #define PATCH(addr) do { \
370 if (addr[0] != SETHI(IF_F2_RD(addr[0]), 0x0) || \
371 addr[1] != OR_R_I_R(IF_F3_RD(addr[1]), 0x0, IF_F3_RS1(addr[1])) || \
372 addr[2] != SETHI(IF_F2_RD(addr[2]), 0x0)) \
373 panic("pmap_boostrap: patched instructions have changed"); \
374 addr[0] |= EIF_IMM((tsb_kernel_mask) >> 10, 22); \
375 addr[1] |= EIF_IMM(tsb_kernel_mask, 10); \
376 addr[2] |= EIF_IMM(((vm_offset_t)tsb_kernel) >> 10, 22); \
382 PATCH(tl1_immu_miss_patch_1);
383 PATCH(tl1_immu_miss_patch_2);
384 PATCH(tl1_dmmu_miss_patch_1);
385 PATCH(tl1_dmmu_miss_patch_2);
386 PATCH(tl1_dmmu_prot_patch_1);
387 PATCH(tl1_dmmu_prot_patch_2);
390 * Enter fake 8k pages for the 4MB kernel pages, so that
391 * pmap_kextract() will work for them.
393 for (i = 0; i < kernel_tlb_slots; i++) {
394 pa = kernel_tlbs[i].te_pa;
395 va = kernel_tlbs[i].te_va;
396 for (off = 0; off < PAGE_SIZE_4M; off += PAGE_SIZE) {
397 tp = tsb_kvtotte(va + off);
398 tp->tte_vpn = TV_VPN(va + off, TS_8K);
399 tp->tte_data = TD_V | TD_8K | TD_PA(pa + off) |
400 TD_REF | TD_SW | TD_CP | TD_CV | TD_P | TD_W;
405 * Set the start and end of kva. The kernel is loaded at the first
406 * available 4 meg super page, so round up to the end of the page.
408 virtual_avail = roundup2(ekva, PAGE_SIZE_4M);
409 virtual_end = vm_max_kernel_address;
410 kernel_vm_end = vm_max_kernel_address;
413 * Allocate kva space for temporary mappings.
415 pmap_idle_map = virtual_avail;
416 virtual_avail += PAGE_SIZE * DCACHE_COLORS;
417 pmap_temp_map_1 = virtual_avail;
418 virtual_avail += PAGE_SIZE * DCACHE_COLORS;
419 pmap_temp_map_2 = virtual_avail;
420 virtual_avail += PAGE_SIZE * DCACHE_COLORS;
423 * Allocate a kernel stack with guard page for thread0 and map it into
424 * the kernel tsb. We must ensure that the virtual address is coloured
425 * properly, since we're allocating from phys_avail so the memory won't
426 * have an associated vm_page_t.
428 pa = pmap_bootstrap_alloc(roundup(KSTACK_PAGES, DCACHE_COLORS) *
431 virtual_avail += roundup(KSTACK_GUARD_PAGES, DCACHE_COLORS) *
433 kstack0 = virtual_avail;
434 virtual_avail += roundup(KSTACK_PAGES, DCACHE_COLORS) * PAGE_SIZE;
435 KASSERT(DCACHE_COLOR(kstack0) == DCACHE_COLOR(kstack0_phys),
436 ("pmap_bootstrap: kstack0 miscoloured"));
437 for (i = 0; i < KSTACK_PAGES; i++) {
438 pa = kstack0_phys + i * PAGE_SIZE;
439 va = kstack0 + i * PAGE_SIZE;
440 tp = tsb_kvtotte(va);
441 tp->tte_vpn = TV_VPN(va, TS_8K);
442 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_SW |
443 TD_CP | TD_CV | TD_P | TD_W;
447 * Calculate the last available physical address.
449 for (i = 0; phys_avail[i + 2] != 0; i += 2)
451 Maxmem = sparc64_btop(phys_avail[i + 1]);
454 * Add the prom mappings to the kernel tsb.
456 if ((vmem = OF_finddevice("/virtual-memory")) == -1)
457 panic("pmap_bootstrap: finddevice /virtual-memory");
458 if ((sz = OF_getproplen(vmem, "translations")) == -1)
459 panic("pmap_bootstrap: getproplen translations");
460 if (sizeof(translations) < sz)
461 panic("pmap_bootstrap: translations too small");
462 bzero(translations, sz);
463 if (OF_getprop(vmem, "translations", translations, sz) == -1)
464 panic("pmap_bootstrap: getprop /virtual-memory/translations");
465 sz /= sizeof(*translations);
466 translations_size = sz;
467 CTR0(KTR_PMAP, "pmap_bootstrap: translations");
468 qsort(translations, sz, sizeof (*translations), om_cmp);
469 for (i = 0; i < sz; i++) {
471 "translation: start=%#lx size=%#lx tte=%#lx",
472 translations[i].om_start, translations[i].om_size,
473 translations[i].om_tte);
474 if (translations[i].om_start < VM_MIN_PROM_ADDRESS ||
475 translations[i].om_start > VM_MAX_PROM_ADDRESS)
477 for (off = 0; off < translations[i].om_size;
479 va = translations[i].om_start + off;
480 tp = tsb_kvtotte(va);
481 tp->tte_vpn = TV_VPN(va, TS_8K);
483 ((translations[i].om_tte &
484 ~(TD_SOFT_MASK << TD_SOFT_SHIFT)) | TD_EXEC) +
490 * Get the available physical memory ranges from /memory/reg. These
491 * are only used for kernel dumps, but it may not be wise to do prom
492 * calls in that situation.
494 if ((sz = OF_getproplen(pmem, "reg")) == -1)
495 panic("pmap_bootstrap: getproplen /memory/reg");
496 if (sizeof(sparc64_memreg) < sz)
497 panic("pmap_bootstrap: sparc64_memreg too small");
498 if (OF_getprop(pmem, "reg", sparc64_memreg, sz) == -1)
499 panic("pmap_bootstrap: getprop /memory/reg");
500 sparc64_nmemreg = sz / sizeof(*sparc64_memreg);
503 * Initialize the kernel pmap (which is statically allocated).
504 * NOTE: PMAP_LOCK_INIT() is needed as part of the initialization
505 * but sparc64 start up is not ready to initialize mutexes yet.
506 * It is called in machdep.c.
509 for (i = 0; i < MAXCPU; i++)
510 pm->pm_context[i] = TLB_CTX_KERNEL;
513 /* XXX flush all non-locked tlb entries */
528 * Map the 4mb tsb pages.
530 for (i = 0; i < tsb_kernel_size; i += PAGE_SIZE_4M) {
531 va = (vm_offset_t)tsb_kernel + i;
532 pa = tsb_kernel_phys + i;
533 data = TD_V | TD_4M | TD_PA(pa) | TD_L | TD_CP | TD_CV |
536 stxa(AA_DMMU_TAR, ASI_DMMU, TLB_TAR_VA(va) |
537 TLB_TAR_CTX(TLB_CTX_KERNEL));
538 stxa_sync(0, ASI_DTLB_DATA_IN_REG, data);
542 * Set the secondary context to be the kernel context (needed for
543 * fp block operations in the kernel and the cache code).
545 stxa(AA_DMMU_SCXR, ASI_DMMU, TLB_CTX_KERNEL);
552 * Allocate a physical page of memory directly from the phys_avail map.
553 * Can only be called from pmap_bootstrap before avail start and end are
557 pmap_bootstrap_alloc(vm_size_t size)
562 size = round_page(size);
563 for (i = 0; phys_avail[i + 1] != 0; i += 2) {
564 if (phys_avail[i + 1] - phys_avail[i] < size)
567 phys_avail[i] += size;
570 panic("pmap_bootstrap_alloc");
574 * Initialize a vm_page's machine-dependent fields.
577 pmap_page_init(vm_page_t m)
580 TAILQ_INIT(&m->md.tte_list);
581 m->md.color = DCACHE_COLOR(VM_PAGE_TO_PHYS(m));
587 * Initialize the pmap module.
597 for (i = 0; i < translations_size; i++) {
598 addr = translations[i].om_start;
599 size = translations[i].om_size;
600 if (addr < VM_MIN_PROM_ADDRESS || addr > VM_MAX_PROM_ADDRESS)
602 result = vm_map_find(kernel_map, NULL, 0, &addr, size, FALSE,
603 VM_PROT_ALL, VM_PROT_ALL, 0);
604 if (result != KERN_SUCCESS || addr != translations[i].om_start)
605 panic("pmap_init: vm_map_find");
610 * Extract the physical page address associated with the given
611 * map/virtual_address pair.
614 pmap_extract(pmap_t pm, vm_offset_t va)
619 if (pm == kernel_pmap)
620 return (pmap_kextract(va));
622 tp = tsb_tte_lookup(pm, va);
626 pa = TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp));
632 * Atomically extract and hold the physical page with the given
633 * pmap and virtual address pair if that mapping permits the given
637 pmap_extract_and_hold(pmap_t pm, vm_offset_t va, vm_prot_t prot)
643 vm_page_lock_queues();
644 if (pm == kernel_pmap) {
645 if (va >= VM_MIN_DIRECT_ADDRESS) {
647 m = PHYS_TO_VM_PAGE(TLB_DIRECT_TO_PHYS(va));
650 tp = tsb_kvtotte(va);
651 if ((tp->tte_data & TD_V) == 0)
656 tp = tsb_tte_lookup(pm, va);
658 if (tp != NULL && ((tp->tte_data & TD_SW) ||
659 (prot & VM_PROT_WRITE) == 0)) {
660 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp));
663 vm_page_unlock_queues();
664 if (pm != kernel_pmap)
670 * Extract the physical page address associated with the given kernel virtual
674 pmap_kextract(vm_offset_t va)
678 if (va >= VM_MIN_DIRECT_ADDRESS)
679 return (TLB_DIRECT_TO_PHYS(va));
680 tp = tsb_kvtotte(va);
681 if ((tp->tte_data & TD_V) == 0)
683 return (TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp)));
687 pmap_cache_enter(vm_page_t m, vm_offset_t va)
692 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
693 KASSERT((m->flags & PG_FICTITIOUS) == 0,
694 ("pmap_cache_enter: fake page"));
695 PMAP_STATS_INC(pmap_ncache_enter);
698 * Find the color for this virtual address and note the added mapping.
700 color = DCACHE_COLOR(va);
701 m->md.colors[color]++;
704 * If all existing mappings have the same color, the mapping is
707 if (m->md.color == color) {
708 KASSERT(m->md.colors[DCACHE_OTHER_COLOR(color)] == 0,
709 ("pmap_cache_enter: cacheable, mappings of other color"));
710 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m)))
711 PMAP_STATS_INC(pmap_ncache_enter_c);
713 PMAP_STATS_INC(pmap_ncache_enter_oc);
718 * If there are no mappings of the other color, and the page still has
719 * the wrong color, this must be a new mapping. Change the color to
720 * match the new mapping, which is cacheable. We must flush the page
721 * from the cache now.
723 if (m->md.colors[DCACHE_OTHER_COLOR(color)] == 0) {
724 KASSERT(m->md.colors[color] == 1,
725 ("pmap_cache_enter: changing color, not new mapping"));
726 dcache_page_inval(VM_PAGE_TO_PHYS(m));
728 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m)))
729 PMAP_STATS_INC(pmap_ncache_enter_cc);
731 PMAP_STATS_INC(pmap_ncache_enter_coc);
736 * If the mapping is already non-cacheable, just return.
738 if (m->md.color == -1) {
739 PMAP_STATS_INC(pmap_ncache_enter_nc);
743 PMAP_STATS_INC(pmap_ncache_enter_cnc);
746 * Mark all mappings as uncacheable, flush any lines with the other
747 * color out of the dcache, and set the color to none (-1).
749 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) {
750 atomic_clear_long(&tp->tte_data, TD_CV);
751 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp));
753 dcache_page_inval(VM_PAGE_TO_PHYS(m));
759 pmap_cache_remove(vm_page_t m, vm_offset_t va)
764 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
765 CTR3(KTR_PMAP, "pmap_cache_remove: m=%p va=%#lx c=%d", m, va,
766 m->md.colors[DCACHE_COLOR(va)]);
767 KASSERT((m->flags & PG_FICTITIOUS) == 0,
768 ("pmap_cache_remove: fake page"));
769 KASSERT(m->md.colors[DCACHE_COLOR(va)] > 0,
770 ("pmap_cache_remove: no mappings %d <= 0",
771 m->md.colors[DCACHE_COLOR(va)]));
772 PMAP_STATS_INC(pmap_ncache_remove);
775 * Find the color for this virtual address and note the removal of
778 color = DCACHE_COLOR(va);
779 m->md.colors[color]--;
782 * If the page is cacheable, just return and keep the same color, even
783 * if there are no longer any mappings.
785 if (m->md.color != -1) {
786 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m)))
787 PMAP_STATS_INC(pmap_ncache_remove_c);
789 PMAP_STATS_INC(pmap_ncache_remove_oc);
793 KASSERT(m->md.colors[DCACHE_OTHER_COLOR(color)] != 0,
794 ("pmap_cache_remove: uncacheable, no mappings of other color"));
797 * If the page is not cacheable (color is -1), and the number of
798 * mappings for this color is not zero, just return. There are
799 * mappings of the other color still, so remain non-cacheable.
801 if (m->md.colors[color] != 0) {
802 PMAP_STATS_INC(pmap_ncache_remove_nc);
807 * The number of mappings for this color is now zero. Recache the
808 * other colored mappings, and change the page color to the other
809 * color. There should be no lines in the data cache for this page,
810 * so flushing should not be needed.
812 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) {
813 atomic_set_long(&tp->tte_data, TD_CV);
814 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp));
816 m->md.color = DCACHE_OTHER_COLOR(color);
818 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m)))
819 PMAP_STATS_INC(pmap_ncache_remove_cc);
821 PMAP_STATS_INC(pmap_ncache_remove_coc);
825 * Map a wired page into kernel virtual address space.
828 pmap_kenter(vm_offset_t va, vm_page_t m)
835 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
836 PMAP_STATS_INC(pmap_nkenter);
837 tp = tsb_kvtotte(va);
838 CTR4(KTR_PMAP, "pmap_kenter: va=%#lx pa=%#lx tp=%p data=%#lx",
839 va, VM_PAGE_TO_PHYS(m), tp, tp->tte_data);
840 if (DCACHE_COLOR(VM_PAGE_TO_PHYS(m)) != DCACHE_COLOR(va)) {
842 "pmap_kenter: off colour va=%#lx pa=%#lx o=%p oc=%#lx ot=%d pi=%#lx",
843 va, VM_PAGE_TO_PHYS(m), m->object,
844 m->object ? m->object->pg_color : -1,
845 m->object ? m->object->type : -1,
847 PMAP_STATS_INC(pmap_nkenter_oc);
849 if ((tp->tte_data & TD_V) != 0) {
850 om = PHYS_TO_VM_PAGE(TTE_GET_PA(tp));
851 ova = TTE_GET_VA(tp);
852 if (m == om && va == ova) {
853 PMAP_STATS_INC(pmap_nkenter_stupid);
856 TAILQ_REMOVE(&om->md.tte_list, tp, tte_link);
857 pmap_cache_remove(om, ova);
859 tlb_page_demap(kernel_pmap, ova);
861 data = TD_V | TD_8K | VM_PAGE_TO_PHYS(m) | TD_REF | TD_SW | TD_CP |
863 if (pmap_cache_enter(m, va) != 0)
865 tp->tte_vpn = TV_VPN(va, TS_8K);
867 TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link);
871 * Map a wired page into kernel virtual address space. This additionally
872 * takes a flag argument wich is or'ed to the TTE data. This is used by
874 * NOTE: if the mapping is non-cacheable, it's the caller's responsibility
875 * to flush entries that might still be in the cache, if applicable.
878 pmap_kenter_flags(vm_offset_t va, vm_paddr_t pa, u_long flags)
882 tp = tsb_kvtotte(va);
883 CTR4(KTR_PMAP, "pmap_kenter_flags: va=%#lx pa=%#lx tp=%p data=%#lx",
884 va, pa, tp, tp->tte_data);
885 tp->tte_vpn = TV_VPN(va, TS_8K);
886 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_P | flags;
890 * Remove a wired page from kernel virtual address space.
893 pmap_kremove(vm_offset_t va)
898 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
899 PMAP_STATS_INC(pmap_nkremove);
900 tp = tsb_kvtotte(va);
901 CTR3(KTR_PMAP, "pmap_kremove: va=%#lx tp=%p data=%#lx", va, tp,
903 if ((tp->tte_data & TD_V) == 0)
905 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp));
906 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link);
907 pmap_cache_remove(m, va);
912 * Inverse of pmap_kenter_flags, used by bus_space_unmap().
915 pmap_kremove_flags(vm_offset_t va)
919 tp = tsb_kvtotte(va);
920 CTR3(KTR_PMAP, "pmap_kremove: va=%#lx tp=%p data=%#lx", va, tp,
926 * Map a range of physical addresses into kernel virtual address space.
928 * The value passed in *virt is a suggested virtual address for the mapping.
929 * Architectures which can support a direct-mapped physical to virtual region
930 * can return the appropriate address within that region, leaving '*virt'
934 pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot)
937 return (TLB_PHYS_TO_DIRECT(start));
941 * Map a list of wired pages into kernel virtual address space. This is
942 * intended for temporary mappings which do not need page modification or
943 * references recorded. Existing mappings in the region are overwritten.
946 pmap_qenter(vm_offset_t sva, vm_page_t *m, int count)
951 PMAP_STATS_INC(pmap_nqenter);
953 if (!(locked = mtx_owned(&vm_page_queue_mtx)))
954 vm_page_lock_queues();
955 while (count-- > 0) {
961 vm_page_unlock_queues();
962 tlb_range_demap(kernel_pmap, sva, va);
966 * Remove page mappings from kernel virtual address space. Intended for
967 * temporary mappings entered by pmap_qenter.
970 pmap_qremove(vm_offset_t sva, int count)
975 PMAP_STATS_INC(pmap_nqremove);
977 if (!(locked = mtx_owned(&vm_page_queue_mtx)))
978 vm_page_lock_queues();
979 while (count-- > 0) {
984 vm_page_unlock_queues();
985 tlb_range_demap(kernel_pmap, sva, va);
989 * Initialize the pmap associated with process 0.
992 pmap_pinit0(pmap_t pm)
997 for (i = 0; i < MAXCPU; i++)
998 pm->pm_context[i] = 0;
1001 pm->pm_tsb_obj = NULL;
1002 bzero(&pm->pm_stats, sizeof(pm->pm_stats));
1006 * Initialize a preallocated and zeroed pmap structure, such as one in a
1007 * vmspace structure.
1010 pmap_pinit(pmap_t pm)
1012 vm_page_t ma[TSB_PAGES];
1019 * Allocate kva space for the tsb.
1021 if (pm->pm_tsb == NULL) {
1022 pm->pm_tsb = (struct tte *)kmem_alloc_nofault(kernel_map,
1027 * Allocate an object for it.
1029 if (pm->pm_tsb_obj == NULL)
1030 pm->pm_tsb_obj = vm_object_allocate(OBJT_DEFAULT, TSB_PAGES);
1032 VM_OBJECT_LOCK(pm->pm_tsb_obj);
1033 for (i = 0; i < TSB_PAGES; i++) {
1034 m = vm_page_grab(pm->pm_tsb_obj, i, VM_ALLOC_NOBUSY |
1035 VM_ALLOC_RETRY | VM_ALLOC_WIRED | VM_ALLOC_ZERO);
1036 m->valid = VM_PAGE_BITS_ALL;
1040 VM_OBJECT_UNLOCK(pm->pm_tsb_obj);
1041 pmap_qenter((vm_offset_t)pm->pm_tsb, ma, TSB_PAGES);
1043 for (i = 0; i < MAXCPU; i++)
1044 pm->pm_context[i] = -1;
1046 bzero(&pm->pm_stats, sizeof(pm->pm_stats));
1050 * Release any resources held by the given physical map.
1051 * Called when a pmap initialized by pmap_pinit is being released.
1052 * Should only be called if the map contains no valid mappings.
1055 pmap_release(pmap_t pm)
1061 CTR2(KTR_PMAP, "pmap_release: ctx=%#x tsb=%p",
1062 pm->pm_context[PCPU_GET(cpuid)], pm->pm_tsb);
1063 KASSERT(pmap_resident_count(pm) == 0,
1064 ("pmap_release: resident pages %ld != 0",
1065 pmap_resident_count(pm)));
1068 * After the pmap was freed, it might be reallocated to a new process.
1069 * When switching, this might lead us to wrongly assume that we need
1070 * not switch contexts because old and new pmap pointer are equal.
1071 * Therefore, make sure that this pmap is not referenced by any PCPU
1072 * pointer any more. This could happen in two cases:
1073 * - A process that referenced the pmap is currently exiting on a CPU.
1074 * However, it is guaranteed to not switch in any more after setting
1075 * its state to PRS_ZOMBIE.
1076 * - A process that referenced this pmap ran on a CPU, but we switched
1077 * to a kernel thread, leaving the pmap pointer unchanged.
1079 mtx_lock_spin(&sched_lock);
1080 SLIST_FOREACH(pc, &cpuhead, pc_allcpu) {
1081 if (pc->pc_pmap == pm)
1084 mtx_unlock_spin(&sched_lock);
1086 obj = pm->pm_tsb_obj;
1087 VM_OBJECT_LOCK(obj);
1088 KASSERT(obj->ref_count == 1, ("pmap_release: tsbobj ref count != 1"));
1089 while (!TAILQ_EMPTY(&obj->memq)) {
1090 m = TAILQ_FIRST(&obj->memq);
1091 vm_page_lock_queues();
1092 if (vm_page_sleep_if_busy(m, FALSE, "pmaprl"))
1094 KASSERT(m->hold_count == 0,
1095 ("pmap_release: freeing held tsb page"));
1098 atomic_subtract_int(&cnt.v_wire_count, 1);
1099 vm_page_free_zero(m);
1100 vm_page_unlock_queues();
1102 VM_OBJECT_UNLOCK(obj);
1103 pmap_qremove((vm_offset_t)pm->pm_tsb, TSB_PAGES);
1104 PMAP_LOCK_DESTROY(pm);
1108 * Grow the number of kernel page table entries. Unneeded.
1111 pmap_growkernel(vm_offset_t addr)
1114 panic("pmap_growkernel: can't grow kernel");
1118 pmap_remove_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp,
1124 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1125 data = atomic_readandclear_long(&tp->tte_data);
1126 if ((data & TD_FAKE) == 0) {
1127 m = PHYS_TO_VM_PAGE(TD_PA(data));
1128 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link);
1129 if ((data & TD_WIRED) != 0)
1130 pm->pm_stats.wired_count--;
1131 if ((data & TD_PV) != 0) {
1132 if ((data & TD_W) != 0)
1134 if ((data & TD_REF) != 0)
1135 vm_page_flag_set(m, PG_REFERENCED);
1136 if (TAILQ_EMPTY(&m->md.tte_list))
1137 vm_page_flag_clear(m, PG_WRITEABLE);
1138 pm->pm_stats.resident_count--;
1140 pmap_cache_remove(m, va);
1143 if (PMAP_REMOVE_DONE(pm))
1149 * Remove the given range of addresses from the specified map.
1152 pmap_remove(pmap_t pm, vm_offset_t start, vm_offset_t end)
1157 CTR3(KTR_PMAP, "pmap_remove: ctx=%#lx start=%#lx end=%#lx",
1158 pm->pm_context[PCPU_GET(cpuid)], start, end);
1159 if (PMAP_REMOVE_DONE(pm))
1161 vm_page_lock_queues();
1163 if (end - start > PMAP_TSB_THRESH) {
1164 tsb_foreach(pm, NULL, start, end, pmap_remove_tte);
1165 tlb_context_demap(pm);
1167 for (va = start; va < end; va += PAGE_SIZE) {
1168 if ((tp = tsb_tte_lookup(pm, va)) != NULL) {
1169 if (!pmap_remove_tte(pm, NULL, tp, va))
1173 tlb_range_demap(pm, start, end - 1);
1176 vm_page_unlock_queues();
1180 pmap_remove_all(vm_page_t m)
1187 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1188 for (tp = TAILQ_FIRST(&m->md.tte_list); tp != NULL; tp = tpn) {
1189 tpn = TAILQ_NEXT(tp, tte_link);
1190 if ((tp->tte_data & TD_PV) == 0)
1192 pm = TTE_GET_PMAP(tp);
1193 va = TTE_GET_VA(tp);
1195 if ((tp->tte_data & TD_WIRED) != 0)
1196 pm->pm_stats.wired_count--;
1197 if ((tp->tte_data & TD_REF) != 0)
1198 vm_page_flag_set(m, PG_REFERENCED);
1199 if ((tp->tte_data & TD_W) != 0)
1201 tp->tte_data &= ~TD_V;
1202 tlb_page_demap(pm, va);
1203 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link);
1204 pm->pm_stats.resident_count--;
1205 pmap_cache_remove(m, va);
1209 vm_page_flag_clear(m, PG_WRITEABLE);
1213 pmap_protect_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp,
1219 data = atomic_clear_long(&tp->tte_data, TD_REF | TD_SW | TD_W);
1220 if ((data & TD_PV) != 0) {
1221 m = PHYS_TO_VM_PAGE(TD_PA(data));
1222 if ((data & TD_REF) != 0)
1223 vm_page_flag_set(m, PG_REFERENCED);
1224 if ((data & TD_W) != 0)
1231 * Set the physical protection on the specified range of this map as requested.
1234 pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
1239 CTR4(KTR_PMAP, "pmap_protect: ctx=%#lx sva=%#lx eva=%#lx prot=%#lx",
1240 pm->pm_context[PCPU_GET(cpuid)], sva, eva, prot);
1242 if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
1243 pmap_remove(pm, sva, eva);
1247 if (prot & VM_PROT_WRITE)
1250 vm_page_lock_queues();
1252 if (eva - sva > PMAP_TSB_THRESH) {
1253 tsb_foreach(pm, NULL, sva, eva, pmap_protect_tte);
1254 tlb_context_demap(pm);
1256 for (va = sva; va < eva; va += PAGE_SIZE) {
1257 if ((tp = tsb_tte_lookup(pm, va)) != NULL)
1258 pmap_protect_tte(pm, NULL, tp, va);
1260 tlb_range_demap(pm, sva, eva - 1);
1263 vm_page_unlock_queues();
1267 * Map the given physical page at the specified virtual address in the
1268 * target pmap with the protection requested. If specified the page
1269 * will be wired down.
1272 pmap_enter(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot,
1276 vm_page_lock_queues();
1278 pmap_enter_locked(pm, va, m, prot, wired);
1279 vm_page_unlock_queues();
1284 * Map the given physical page at the specified virtual address in the
1285 * target pmap with the protection requested. If specified the page
1286 * will be wired down.
1288 * The page queues and pmap must be locked.
1291 pmap_enter_locked(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot,
1299 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1300 PMAP_LOCK_ASSERT(pm, MA_OWNED);
1301 PMAP_STATS_INC(pmap_nenter);
1302 pa = VM_PAGE_TO_PHYS(m);
1305 * If this is a fake page from the device_pager, but it covers actual
1306 * physical memory, convert to the real backing page.
1308 if ((m->flags & PG_FICTITIOUS) != 0) {
1309 for (i = 0; phys_avail[i + 1] != 0; i += 2) {
1310 if (pa >= phys_avail[i] && pa <= phys_avail[i + 1]) {
1311 m = PHYS_TO_VM_PAGE(pa);
1318 "pmap_enter: ctx=%p m=%p va=%#lx pa=%#lx prot=%#x wired=%d",
1319 pm->pm_context[PCPU_GET(cpuid)], m, va, pa, prot, wired);
1322 * If there is an existing mapping, and the physical address has not
1323 * changed, must be protection or wiring change.
1325 if ((tp = tsb_tte_lookup(pm, va)) != NULL && TTE_GET_PA(tp) == pa) {
1326 CTR0(KTR_PMAP, "pmap_enter: update");
1327 PMAP_STATS_INC(pmap_nenter_update);
1330 * Wiring change, just update stats.
1333 if ((tp->tte_data & TD_WIRED) == 0) {
1334 tp->tte_data |= TD_WIRED;
1335 pm->pm_stats.wired_count++;
1338 if ((tp->tte_data & TD_WIRED) != 0) {
1339 tp->tte_data &= ~TD_WIRED;
1340 pm->pm_stats.wired_count--;
1345 * Save the old bits and clear the ones we're interested in.
1347 data = tp->tte_data;
1348 tp->tte_data &= ~(TD_EXEC | TD_SW | TD_W);
1351 * If we're turning off write permissions, sense modify status.
1353 if ((prot & VM_PROT_WRITE) != 0) {
1354 tp->tte_data |= TD_SW;
1356 tp->tte_data |= TD_W;
1358 vm_page_flag_set(m, PG_WRITEABLE);
1359 } else if ((data & TD_W) != 0) {
1364 * If we're turning on execute permissions, flush the icache.
1366 if ((prot & VM_PROT_EXECUTE) != 0) {
1367 if ((data & TD_EXEC) == 0) {
1368 icache_page_inval(pa);
1370 tp->tte_data |= TD_EXEC;
1374 * Delete the old mapping.
1376 tlb_page_demap(pm, TTE_GET_VA(tp));
1379 * If there is an existing mapping, but its for a different
1380 * phsyical address, delete the old mapping.
1383 CTR0(KTR_PMAP, "pmap_enter: replace");
1384 PMAP_STATS_INC(pmap_nenter_replace);
1385 pmap_remove_tte(pm, NULL, tp, va);
1386 tlb_page_demap(pm, va);
1388 CTR0(KTR_PMAP, "pmap_enter: new");
1389 PMAP_STATS_INC(pmap_nenter_new);
1393 * Now set up the data and install the new mapping.
1395 data = TD_V | TD_8K | TD_PA(pa);
1396 if (pm == kernel_pmap)
1398 if ((prot & VM_PROT_WRITE) != 0) {
1400 vm_page_flag_set(m, PG_WRITEABLE);
1402 if (prot & VM_PROT_EXECUTE) {
1404 icache_page_inval(pa);
1408 * If its wired update stats. We also don't need reference or
1409 * modify tracking for wired mappings, so set the bits now.
1412 pm->pm_stats.wired_count++;
1413 data |= TD_REF | TD_WIRED;
1414 if ((prot & VM_PROT_WRITE) != 0)
1418 tsb_tte_enter(pm, m, va, TS_8K, data);
1423 * Maps a sequence of resident pages belonging to the same object.
1424 * The sequence begins with the given page m_start. This page is
1425 * mapped at the given virtual address start. Each subsequent page is
1426 * mapped at a virtual address that is offset from start by the same
1427 * amount as the page is offset from m_start within the object. The
1428 * last page in the sequence is the page with the largest offset from
1429 * m_start that can be mapped at a virtual address less than the given
1430 * virtual address end. Not every virtual page between start and end
1431 * is mapped; only those for which a resident page exists with the
1432 * corresponding offset from m_start are mapped.
1435 pmap_enter_object(pmap_t pm, vm_offset_t start, vm_offset_t end,
1436 vm_page_t m_start, vm_prot_t prot)
1439 vm_pindex_t diff, psize;
1441 psize = atop(end - start);
1444 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
1445 pmap_enter_locked(pm, start + ptoa(diff), m, prot &
1446 (VM_PROT_READ | VM_PROT_EXECUTE), FALSE);
1447 m = TAILQ_NEXT(m, listq);
1453 pmap_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot)
1457 pmap_enter_locked(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE),
1463 pmap_object_init_pt(pmap_t pm, vm_offset_t addr, vm_object_t object,
1464 vm_pindex_t pindex, vm_size_t size)
1467 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1468 KASSERT(object->type == OBJT_DEVICE,
1469 ("pmap_object_init_pt: non-device object"));
1473 * Change the wiring attribute for a map/virtual-address pair.
1474 * The mapping must already exist in the pmap.
1477 pmap_change_wiring(pmap_t pm, vm_offset_t va, boolean_t wired)
1483 if ((tp = tsb_tte_lookup(pm, va)) != NULL) {
1485 data = atomic_set_long(&tp->tte_data, TD_WIRED);
1486 if ((data & TD_WIRED) == 0)
1487 pm->pm_stats.wired_count++;
1489 data = atomic_clear_long(&tp->tte_data, TD_WIRED);
1490 if ((data & TD_WIRED) != 0)
1491 pm->pm_stats.wired_count--;
1498 pmap_copy_tte(pmap_t src_pmap, pmap_t dst_pmap, struct tte *tp, vm_offset_t va)
1503 if ((tp->tte_data & TD_FAKE) != 0)
1505 if (tsb_tte_lookup(dst_pmap, va) == NULL) {
1506 data = tp->tte_data &
1507 ~(TD_PV | TD_REF | TD_SW | TD_CV | TD_W);
1508 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp));
1509 tsb_tte_enter(dst_pmap, m, va, TS_8K, data);
1515 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr,
1516 vm_size_t len, vm_offset_t src_addr)
1521 if (dst_addr != src_addr)
1523 vm_page_lock_queues();
1524 if (dst_pmap < src_pmap) {
1525 PMAP_LOCK(dst_pmap);
1526 PMAP_LOCK(src_pmap);
1528 PMAP_LOCK(src_pmap);
1529 PMAP_LOCK(dst_pmap);
1531 if (len > PMAP_TSB_THRESH) {
1532 tsb_foreach(src_pmap, dst_pmap, src_addr, src_addr + len,
1534 tlb_context_demap(dst_pmap);
1536 for (va = src_addr; va < src_addr + len; va += PAGE_SIZE) {
1537 if ((tp = tsb_tte_lookup(src_pmap, va)) != NULL)
1538 pmap_copy_tte(src_pmap, dst_pmap, tp, va);
1540 tlb_range_demap(dst_pmap, src_addr, src_addr + len - 1);
1542 vm_page_unlock_queues();
1543 PMAP_UNLOCK(src_pmap);
1544 PMAP_UNLOCK(dst_pmap);
1548 pmap_zero_page(vm_page_t m)
1554 KASSERT((m->flags & PG_FICTITIOUS) == 0,
1555 ("pmap_zero_page: fake page"));
1556 PMAP_STATS_INC(pmap_nzero_page);
1557 pa = VM_PAGE_TO_PHYS(m);
1558 if (m->md.color == -1) {
1559 PMAP_STATS_INC(pmap_nzero_page_nc);
1560 aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE);
1561 } else if (m->md.color == DCACHE_COLOR(pa)) {
1562 PMAP_STATS_INC(pmap_nzero_page_c);
1563 va = TLB_PHYS_TO_DIRECT(pa);
1564 cpu_block_zero((void *)va, PAGE_SIZE);
1566 PMAP_STATS_INC(pmap_nzero_page_oc);
1567 PMAP_LOCK(kernel_pmap);
1568 va = pmap_temp_map_1 + (m->md.color * PAGE_SIZE);
1569 tp = tsb_kvtotte(va);
1570 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W;
1571 tp->tte_vpn = TV_VPN(va, TS_8K);
1572 cpu_block_zero((void *)va, PAGE_SIZE);
1573 tlb_page_demap(kernel_pmap, va);
1574 PMAP_UNLOCK(kernel_pmap);
1579 pmap_zero_page_area(vm_page_t m, int off, int size)
1585 KASSERT((m->flags & PG_FICTITIOUS) == 0,
1586 ("pmap_zero_page_area: fake page"));
1587 KASSERT(off + size <= PAGE_SIZE, ("pmap_zero_page_area: bad off/size"));
1588 PMAP_STATS_INC(pmap_nzero_page_area);
1589 pa = VM_PAGE_TO_PHYS(m);
1590 if (m->md.color == -1) {
1591 PMAP_STATS_INC(pmap_nzero_page_area_nc);
1592 aszero(ASI_PHYS_USE_EC, pa + off, size);
1593 } else if (m->md.color == DCACHE_COLOR(pa)) {
1594 PMAP_STATS_INC(pmap_nzero_page_area_c);
1595 va = TLB_PHYS_TO_DIRECT(pa);
1596 bzero((void *)(va + off), size);
1598 PMAP_STATS_INC(pmap_nzero_page_area_oc);
1599 PMAP_LOCK(kernel_pmap);
1600 va = pmap_temp_map_1 + (m->md.color * PAGE_SIZE);
1601 tp = tsb_kvtotte(va);
1602 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W;
1603 tp->tte_vpn = TV_VPN(va, TS_8K);
1604 bzero((void *)(va + off), size);
1605 tlb_page_demap(kernel_pmap, va);
1606 PMAP_UNLOCK(kernel_pmap);
1611 pmap_zero_page_idle(vm_page_t m)
1617 KASSERT((m->flags & PG_FICTITIOUS) == 0,
1618 ("pmap_zero_page_idle: fake page"));
1619 PMAP_STATS_INC(pmap_nzero_page_idle);
1620 pa = VM_PAGE_TO_PHYS(m);
1621 if (m->md.color == -1) {
1622 PMAP_STATS_INC(pmap_nzero_page_idle_nc);
1623 aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE);
1624 } else if (m->md.color == DCACHE_COLOR(pa)) {
1625 PMAP_STATS_INC(pmap_nzero_page_idle_c);
1626 va = TLB_PHYS_TO_DIRECT(pa);
1627 cpu_block_zero((void *)va, PAGE_SIZE);
1629 PMAP_STATS_INC(pmap_nzero_page_idle_oc);
1630 va = pmap_idle_map + (m->md.color * PAGE_SIZE);
1631 tp = tsb_kvtotte(va);
1632 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W;
1633 tp->tte_vpn = TV_VPN(va, TS_8K);
1634 cpu_block_zero((void *)va, PAGE_SIZE);
1635 tlb_page_demap(kernel_pmap, va);
1640 pmap_copy_page(vm_page_t msrc, vm_page_t mdst)
1648 KASSERT((mdst->flags & PG_FICTITIOUS) == 0,
1649 ("pmap_copy_page: fake dst page"));
1650 KASSERT((msrc->flags & PG_FICTITIOUS) == 0,
1651 ("pmap_copy_page: fake src page"));
1652 PMAP_STATS_INC(pmap_ncopy_page);
1653 pdst = VM_PAGE_TO_PHYS(mdst);
1654 psrc = VM_PAGE_TO_PHYS(msrc);
1655 if (msrc->md.color == -1 && mdst->md.color == -1) {
1656 PMAP_STATS_INC(pmap_ncopy_page_nc);
1657 ascopy(ASI_PHYS_USE_EC, psrc, pdst, PAGE_SIZE);
1658 } else if (msrc->md.color == DCACHE_COLOR(psrc) &&
1659 mdst->md.color == DCACHE_COLOR(pdst)) {
1660 PMAP_STATS_INC(pmap_ncopy_page_c);
1661 vdst = TLB_PHYS_TO_DIRECT(pdst);
1662 vsrc = TLB_PHYS_TO_DIRECT(psrc);
1663 cpu_block_copy((void *)vsrc, (void *)vdst, PAGE_SIZE);
1664 } else if (msrc->md.color == -1) {
1665 if (mdst->md.color == DCACHE_COLOR(pdst)) {
1666 PMAP_STATS_INC(pmap_ncopy_page_dc);
1667 vdst = TLB_PHYS_TO_DIRECT(pdst);
1668 ascopyfrom(ASI_PHYS_USE_EC, psrc, (void *)vdst,
1671 PMAP_STATS_INC(pmap_ncopy_page_doc);
1672 PMAP_LOCK(kernel_pmap);
1673 vdst = pmap_temp_map_1 + (mdst->md.color * PAGE_SIZE);
1674 tp = tsb_kvtotte(vdst);
1676 TD_V | TD_8K | TD_PA(pdst) | TD_CP | TD_CV | TD_W;
1677 tp->tte_vpn = TV_VPN(vdst, TS_8K);
1678 ascopyfrom(ASI_PHYS_USE_EC, psrc, (void *)vdst,
1680 tlb_page_demap(kernel_pmap, vdst);
1681 PMAP_UNLOCK(kernel_pmap);
1683 } else if (mdst->md.color == -1) {
1684 if (msrc->md.color == DCACHE_COLOR(psrc)) {
1685 PMAP_STATS_INC(pmap_ncopy_page_sc);
1686 vsrc = TLB_PHYS_TO_DIRECT(psrc);
1687 ascopyto((void *)vsrc, ASI_PHYS_USE_EC, pdst,
1690 PMAP_STATS_INC(pmap_ncopy_page_soc);
1691 PMAP_LOCK(kernel_pmap);
1692 vsrc = pmap_temp_map_1 + (msrc->md.color * PAGE_SIZE);
1693 tp = tsb_kvtotte(vsrc);
1695 TD_V | TD_8K | TD_PA(psrc) | TD_CP | TD_CV | TD_W;
1696 tp->tte_vpn = TV_VPN(vsrc, TS_8K);
1697 ascopyto((void *)vsrc, ASI_PHYS_USE_EC, pdst,
1699 tlb_page_demap(kernel_pmap, vsrc);
1700 PMAP_UNLOCK(kernel_pmap);
1703 PMAP_STATS_INC(pmap_ncopy_page_oc);
1704 PMAP_LOCK(kernel_pmap);
1705 vdst = pmap_temp_map_1 + (mdst->md.color * PAGE_SIZE);
1706 tp = tsb_kvtotte(vdst);
1708 TD_V | TD_8K | TD_PA(pdst) | TD_CP | TD_CV | TD_W;
1709 tp->tte_vpn = TV_VPN(vdst, TS_8K);
1710 vsrc = pmap_temp_map_2 + (msrc->md.color * PAGE_SIZE);
1711 tp = tsb_kvtotte(vsrc);
1713 TD_V | TD_8K | TD_PA(psrc) | TD_CP | TD_CV | TD_W;
1714 tp->tte_vpn = TV_VPN(vsrc, TS_8K);
1715 cpu_block_copy((void *)vsrc, (void *)vdst, PAGE_SIZE);
1716 tlb_page_demap(kernel_pmap, vdst);
1717 tlb_page_demap(kernel_pmap, vsrc);
1718 PMAP_UNLOCK(kernel_pmap);
1723 * Returns true if the pmap's pv is one of the first
1724 * 16 pvs linked to from this page. This count may
1725 * be changed upwards or downwards in the future; it
1726 * is only necessary that true be returned for a small
1727 * subset of pmaps for proper page aging.
1730 pmap_page_exists_quick(pmap_t pm, vm_page_t m)
1735 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1736 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
1739 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) {
1740 if ((tp->tte_data & TD_PV) == 0)
1742 if (TTE_GET_PMAP(tp) == pm)
1751 * Remove all pages from specified address space, this aids process exit
1752 * speeds. This is much faster than pmap_remove n the case of running down
1753 * an entire address space. Only works for the current pmap.
1756 pmap_remove_pages(pmap_t pm)
1761 * Returns TRUE if the given page has a managed mapping.
1764 pmap_page_is_mapped(vm_page_t m)
1768 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1769 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
1771 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) {
1772 if ((tp->tte_data & TD_PV) != 0)
1779 * pmap_ts_referenced:
1781 * Return a count of reference bits for a page, clearing those bits.
1782 * It is not necessary for every reference bit to be cleared, but it
1783 * is necessary that 0 only be returned when there are truly no
1784 * reference bits set.
1786 * XXX: The exact number of bits to check and clear is a matter that
1787 * should be tested and standardized at some point in the future for
1788 * optimal aging of shared pages.
1792 pmap_ts_referenced(vm_page_t m)
1800 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1801 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
1804 if ((tp = TAILQ_FIRST(&m->md.tte_list)) != NULL) {
1807 tpn = TAILQ_NEXT(tp, tte_link);
1808 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link);
1809 TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link);
1810 if ((tp->tte_data & TD_PV) == 0)
1812 data = atomic_clear_long(&tp->tte_data, TD_REF);
1813 if ((data & TD_REF) != 0 && ++count > 4)
1815 } while ((tp = tpn) != NULL && tp != tpf);
1821 pmap_is_modified(vm_page_t m)
1825 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1826 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
1828 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) {
1829 if ((tp->tte_data & TD_PV) == 0)
1831 if ((tp->tte_data & TD_W) != 0)
1838 * pmap_is_prefaultable:
1840 * Return whether or not the specified virtual address is elgible
1844 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)
1851 pmap_clear_modify(vm_page_t m)
1856 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1857 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
1859 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) {
1860 if ((tp->tte_data & TD_PV) == 0)
1862 data = atomic_clear_long(&tp->tte_data, TD_W);
1863 if ((data & TD_W) != 0)
1864 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp));
1869 pmap_clear_reference(vm_page_t m)
1874 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1875 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
1877 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) {
1878 if ((tp->tte_data & TD_PV) == 0)
1880 data = atomic_clear_long(&tp->tte_data, TD_REF);
1881 if ((data & TD_REF) != 0)
1882 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp));
1887 pmap_remove_write(vm_page_t m)
1892 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1893 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0 ||
1894 (m->flags & PG_WRITEABLE) == 0)
1896 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) {
1897 if ((tp->tte_data & TD_PV) == 0)
1899 data = atomic_clear_long(&tp->tte_data, TD_SW | TD_W);
1900 if ((data & TD_W) != 0) {
1902 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp));
1905 vm_page_flag_clear(m, PG_WRITEABLE);
1909 pmap_mincore(pmap_t pm, vm_offset_t addr)
1916 * Activate a user pmap. The pmap must be activated before its address space
1917 * can be accessed in any way.
1920 pmap_activate(struct thread *td)
1926 vm = td->td_proc->p_vmspace;
1927 pm = vmspace_pmap(vm);
1929 mtx_lock_spin(&sched_lock);
1931 context = PCPU_GET(tlb_ctx);
1932 if (context == PCPU_GET(tlb_ctx_max)) {
1934 context = PCPU_GET(tlb_ctx_min);
1936 PCPU_SET(tlb_ctx, context + 1);
1938 pm->pm_context[PCPU_GET(cpuid)] = context;
1939 pm->pm_active |= PCPU_GET(cpumask);
1942 stxa(AA_DMMU_TSB, ASI_DMMU, pm->pm_tsb);
1943 stxa(AA_IMMU_TSB, ASI_IMMU, pm->pm_tsb);
1944 stxa(AA_DMMU_PCXR, ASI_DMMU, context);
1947 mtx_unlock_spin(&sched_lock);
1951 pmap_addr_hint(vm_object_t object, vm_offset_t va, vm_size_t size)