2 * Copyright (c) 2005 Peter Grehan
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
32 * Dispatch MI pmap calls to the appropriate MMU implementation
33 * through a previously registered kernel object.
35 * Before pmap_bootstrap() can be called, a CPU module must have
36 * called pmap_mmu_install(). This may be called multiple times:
37 * the highest priority call will be installed as the default
38 * MMU handler when pmap_bootstrap() is called.
40 * It is required that mutex_init() be called before pmap_bootstrap(),
41 * as the PMAP layer makes extensive use of mutexes.
44 #include <sys/param.h>
45 #include <sys/kernel.h>
48 #include <sys/mutex.h>
49 #include <sys/systm.h>
52 #include <vm/vm_page.h>
54 #include <machine/mmuvar.h>
55 #include <machine/smp.h>
59 static mmu_def_t *mmu_def_impl;
61 static struct mmu_kobj mmu_kernel_obj;
62 static struct kobj_ops mmu_kernel_kops;
67 struct pmap kernel_pmap_store;
69 struct msgbuf *msgbufp;
70 vm_offset_t msgbuf_phys;
72 vm_offset_t kernel_vm_end;
73 vm_offset_t phys_avail[PHYS_AVAIL_SZ];
74 vm_offset_t virtual_avail;
75 vm_offset_t virtual_end;
77 int pmap_bootstrapped;
81 pvo_vaddr_compare(struct pvo_entry *a, struct pvo_entry *b)
83 if (PVO_VADDR(a) < PVO_VADDR(b))
85 else if (PVO_VADDR(a) > PVO_VADDR(b))
89 RB_GENERATE(pvo_tree, pvo_entry, pvo_plink, pvo_vaddr_compare);
94 pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired)
97 CTR4(KTR_PMAP, "%s(%p, %#x, %u)", __func__, pmap, va, wired);
98 MMU_CHANGE_WIRING(mmu_obj, pmap, va, wired);
102 pmap_clear_modify(vm_page_t m)
105 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
106 MMU_CLEAR_MODIFY(mmu_obj, m);
110 pmap_clear_reference(vm_page_t m)
113 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
114 MMU_CLEAR_REFERENCE(mmu_obj, m);
118 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr,
119 vm_size_t len, vm_offset_t src_addr)
122 CTR6(KTR_PMAP, "%s(%p, %p, %#x, %#x, %#x)", __func__, dst_pmap,
123 src_pmap, dst_addr, len, src_addr);
124 MMU_COPY(mmu_obj, dst_pmap, src_pmap, dst_addr, len, src_addr);
128 pmap_copy_page(vm_page_t src, vm_page_t dst)
131 CTR3(KTR_PMAP, "%s(%p, %p)", __func__, src, dst);
132 MMU_COPY_PAGE(mmu_obj, src, dst);
136 pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[],
137 vm_offset_t b_offset, int xfersize)
140 CTR6(KTR_PMAP, "%s(%p, %#x, %p, %#x, %#x)", __func__, ma,
141 a_offset, mb, b_offset, xfersize);
142 MMU_COPY_PAGES(mmu_obj, ma, a_offset, mb, b_offset, xfersize);
146 pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t p,
147 vm_prot_t prot, boolean_t wired)
150 CTR6(KTR_PMAP, "pmap_enter(%p, %#x, %#x, %p, %#x, %u)", pmap, va,
151 access, p, prot, wired);
152 MMU_ENTER(mmu_obj, pmap, va, p, prot, wired);
156 pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,
157 vm_page_t m_start, vm_prot_t prot)
160 CTR6(KTR_PMAP, "%s(%p, %#x, %#x, %p, %#x)", __func__, pmap, start,
162 MMU_ENTER_OBJECT(mmu_obj, pmap, start, end, m_start, prot);
166 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
169 CTR5(KTR_PMAP, "%s(%p, %#x, %p, %#x)", __func__, pmap, va, m, prot);
170 MMU_ENTER_QUICK(mmu_obj, pmap, va, m, prot);
174 pmap_extract(pmap_t pmap, vm_offset_t va)
177 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, va);
178 return (MMU_EXTRACT(mmu_obj, pmap, va));
182 pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
185 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, va, prot);
186 return (MMU_EXTRACT_AND_HOLD(mmu_obj, pmap, va, prot));
190 pmap_growkernel(vm_offset_t va)
193 CTR2(KTR_PMAP, "%s(%#x)", __func__, va);
194 MMU_GROWKERNEL(mmu_obj, va);
201 CTR1(KTR_PMAP, "%s()", __func__);
206 pmap_is_modified(vm_page_t m)
209 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
210 return (MMU_IS_MODIFIED(mmu_obj, m));
214 pmap_is_prefaultable(pmap_t pmap, vm_offset_t va)
217 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, va);
218 return (MMU_IS_PREFAULTABLE(mmu_obj, pmap, va));
222 pmap_is_referenced(vm_page_t m)
225 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
226 return (MMU_IS_REFERENCED(mmu_obj, m));
230 pmap_ts_referenced(vm_page_t m)
233 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
234 return (MMU_TS_REFERENCED(mmu_obj, m));
238 pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot)
241 CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, virt, start, end,
243 return (MMU_MAP(mmu_obj, virt, start, end, prot));
247 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,
248 vm_pindex_t pindex, vm_size_t size)
251 CTR6(KTR_PMAP, "%s(%p, %#x, %p, %u, %#x)", __func__, pmap, addr,
252 object, pindex, size);
253 MMU_OBJECT_INIT_PT(mmu_obj, pmap, addr, object, pindex, size);
257 pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
260 CTR3(KTR_PMAP, "%s(%p, %p)", __func__, pmap, m);
261 return (MMU_PAGE_EXISTS_QUICK(mmu_obj, pmap, m));
265 pmap_page_init(vm_page_t m)
268 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
269 MMU_PAGE_INIT(mmu_obj, m);
273 pmap_page_wired_mappings(vm_page_t m)
276 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
277 return (MMU_PAGE_WIRED_MAPPINGS(mmu_obj, m));
281 pmap_pinit(pmap_t pmap)
284 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);
285 MMU_PINIT(mmu_obj, pmap);
290 pmap_pinit0(pmap_t pmap)
293 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);
294 MMU_PINIT0(mmu_obj, pmap);
298 pmap_protect(pmap_t pmap, vm_offset_t start, vm_offset_t end, vm_prot_t prot)
301 CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, pmap, start, end,
303 MMU_PROTECT(mmu_obj, pmap, start, end, prot);
307 pmap_qenter(vm_offset_t start, vm_page_t *m, int count)
310 CTR4(KTR_PMAP, "%s(%#x, %p, %d)", __func__, start, m, count);
311 MMU_QENTER(mmu_obj, start, m, count);
315 pmap_qremove(vm_offset_t start, int count)
318 CTR3(KTR_PMAP, "%s(%#x, %d)", __func__, start, count);
319 MMU_QREMOVE(mmu_obj, start, count);
323 pmap_release(pmap_t pmap)
326 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);
327 MMU_RELEASE(mmu_obj, pmap);
331 pmap_remove(pmap_t pmap, vm_offset_t start, vm_offset_t end)
334 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, start, end);
335 MMU_REMOVE(mmu_obj, pmap, start, end);
339 pmap_remove_all(vm_page_t m)
342 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
343 MMU_REMOVE_ALL(mmu_obj, m);
347 pmap_remove_pages(pmap_t pmap)
350 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);
351 MMU_REMOVE_PAGES(mmu_obj, pmap);
355 pmap_remove_write(vm_page_t m)
358 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
359 MMU_REMOVE_WRITE(mmu_obj, m);
363 pmap_zero_page(vm_page_t m)
366 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
367 MMU_ZERO_PAGE(mmu_obj, m);
371 pmap_zero_page_area(vm_page_t m, int off, int size)
374 CTR4(KTR_PMAP, "%s(%p, %d, %d)", __func__, m, off, size);
375 MMU_ZERO_PAGE_AREA(mmu_obj, m, off, size);
379 pmap_zero_page_idle(vm_page_t m)
382 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
383 MMU_ZERO_PAGE_IDLE(mmu_obj, m);
387 pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa)
390 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr);
391 return (MMU_MINCORE(mmu_obj, pmap, addr, locked_pa));
395 pmap_activate(struct thread *td)
398 CTR2(KTR_PMAP, "%s(%p)", __func__, td);
399 MMU_ACTIVATE(mmu_obj, td);
403 pmap_deactivate(struct thread *td)
406 CTR2(KTR_PMAP, "%s(%p)", __func__, td);
407 MMU_DEACTIVATE(mmu_obj, td);
411 * Increase the starting virtual address of the given mapping if a
412 * different alignment might result in more superpage mappings.
415 pmap_align_superpage(vm_object_t object, vm_ooffset_t offset,
416 vm_offset_t *addr, vm_size_t size)
419 CTR5(KTR_PMAP, "%s(%p, %#x, %p, %#x)", __func__, object, offset, addr,
421 MMU_ALIGN_SUPERPAGE(mmu_obj, object, offset, addr, size);
425 * Routines used in machine-dependent code
428 pmap_bootstrap(vm_offset_t start, vm_offset_t end)
430 mmu_obj = &mmu_kernel_obj;
433 * Take care of compiling the selected class, and
434 * then statically initialise the MMU object
436 kobj_class_compile_static(mmu_def_impl, &mmu_kernel_kops);
437 kobj_init_static((kobj_t)mmu_obj, mmu_def_impl);
439 MMU_BOOTSTRAP(mmu_obj, start, end);
443 pmap_cpu_bootstrap(int ap)
446 * No KTR here because our console probably doesn't work yet
449 return (MMU_CPU_BOOTSTRAP(mmu_obj, ap));
453 pmap_mapdev(vm_paddr_t pa, vm_size_t size)
456 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size);
457 return (MMU_MAPDEV(mmu_obj, pa, size));
461 pmap_mapdev_attr(vm_offset_t pa, vm_size_t size, vm_memattr_t attr)
464 CTR4(KTR_PMAP, "%s(%#x, %#x, %#x)", __func__, pa, size, attr);
465 return (MMU_MAPDEV_ATTR(mmu_obj, pa, size, attr));
469 pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma)
472 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, m, ma);
473 return (MMU_PAGE_SET_MEMATTR(mmu_obj, m, ma));
477 pmap_unmapdev(vm_offset_t va, vm_size_t size)
480 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, va, size);
481 MMU_UNMAPDEV(mmu_obj, va, size);
485 pmap_kextract(vm_offset_t va)
488 CTR2(KTR_PMAP, "%s(%#x)", __func__, va);
489 return (MMU_KEXTRACT(mmu_obj, va));
493 pmap_kenter(vm_offset_t va, vm_paddr_t pa)
496 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, va, pa);
497 MMU_KENTER(mmu_obj, va, pa);
501 pmap_kenter_attr(vm_offset_t va, vm_offset_t pa, vm_memattr_t ma)
504 CTR4(KTR_PMAP, "%s(%#x, %#x, %#x)", __func__, va, pa, ma);
505 MMU_KENTER_ATTR(mmu_obj, va, pa, ma);
509 pmap_dev_direct_mapped(vm_paddr_t pa, vm_size_t size)
512 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size);
513 return (MMU_DEV_DIRECT_MAPPED(mmu_obj, pa, size));
517 pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz)
520 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pm, va, sz);
521 return (MMU_SYNC_ICACHE(mmu_obj, pm, va, sz));
525 pmap_dumpsys_map(struct pmap_md *md, vm_size_t ofs, vm_size_t *sz)
528 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, md, ofs, *sz);
529 return (MMU_DUMPSYS_MAP(mmu_obj, md, ofs, sz));
533 pmap_dumpsys_unmap(struct pmap_md *md, vm_size_t ofs, vm_offset_t va)
536 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, md, ofs, va);
537 return (MMU_DUMPSYS_UNMAP(mmu_obj, md, ofs, va));
541 pmap_scan_md(struct pmap_md *prev)
544 CTR2(KTR_PMAP, "%s(%p)", __func__, prev);
545 return (MMU_SCAN_MD(mmu_obj, prev));
549 * MMU install routines. Highest priority wins, equal priority also
550 * overrides allowing last-set to win.
552 SET_DECLARE(mmu_set, mmu_def_t);
555 pmap_mmu_install(char *name, int prio)
557 mmu_def_t **mmupp, *mmup;
558 static int curr_prio = 0;
561 * Try and locate the MMU kobj corresponding to the name
563 SET_FOREACH(mmupp, mmu_set) {
567 !strcmp(mmup->name, name) &&
568 (prio >= curr_prio || mmu_def_impl == NULL)) {
578 int unmapped_buf_allowed;