sys/powerpc/powerpc/pmap_dispatch.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2005 Peter Grehan
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  *
  28  */
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 /*
  34  * Dispatch MI pmap calls to the appropriate MMU implementation
  35  * through a previously registered kernel object.
  36  *
  37  * Before pmap_bootstrap() can be called, a CPU module must have
  38  * called pmap_mmu_install(). This may be called multiple times:
  39  * the highest priority call will be installed as the default
  40  * MMU handler when pmap_bootstrap() is called.
  41  *
  42  * It is required that mutex_init() be called before pmap_bootstrap(),
  43  * as the PMAP layer makes extensive use of mutexes.
  44  */
  45
  46 #include <sys/param.h>
  47 #include <sys/kernel.h>
  48 #include <sys/conf.h>
  49 #include <sys/lock.h>
  50 #include <sys/kerneldump.h>
  51 #include <sys/ktr.h>
  52 #include <sys/mutex.h>
  53 #include <sys/sysctl.h>
  54 #include <sys/systm.h>
  55
  56 #include <vm/vm.h>
  57 #include <vm/vm_extern.h>
  58 #include <vm/vm_page.h>
  59
  60 #include <machine/dump.h>
  61 #include <machine/ifunc.h>
  62 #include <machine/md_var.h>
  63 #include <machine/mmuvar.h>
  64 #include <machine/smp.h>
  65
  66 mmu_t           mmu_obj;
  67
  68 /*
  69  * pmap globals
  70  */
  71 struct pmap kernel_pmap_store;
  72
  73 vm_offset_t    msgbuf_phys;
  74
  75 vm_offset_t kernel_vm_end;
  76 vm_offset_t virtual_avail;
  77 vm_offset_t virtual_end;
  78 caddr_t crashdumpmap;
  79
  80 int pmap_bootstrapped;
  81 /* Default level 0 reservations consist of 512 pages (2MB superpage). */
  82 int vm_level_0_order = 9;
  83
  84 SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");
  85
  86 int superpages_enabled = 0;
  87 SYSCTL_INT(_vm_pmap, OID_AUTO, superpages_enabled, CTLFLAG_RDTUN,
  88     &superpages_enabled, 0, "Enable support for transparent superpages");
  89
  90 #ifdef AIM
  91 int
  92 pvo_vaddr_compare(struct pvo_entry *a, struct pvo_entry *b)
  93 {
  94         if (PVO_VADDR(a) < PVO_VADDR(b))
  95                 return (-1);
  96         else if (PVO_VADDR(a) > PVO_VADDR(b))
  97                 return (1);
  98         return (0);
  99 }
 100 RB_GENERATE(pvo_tree, pvo_entry, pvo_plink, pvo_vaddr_compare);
 101 #endif
 102
 103 static int
 104 pmap_nomethod(void)
 105 {
 106         return (0);
 107 }
 108
 109 #define DEFINE_PMAP_IFUNC(ret, func, args)                              \
 110         DEFINE_IFUNC(, ret, pmap_##func, args) {                        \
 111                 pmap_##func##_t f;                                      \
 112                 f = PMAP_RESOLVE_FUNC(func);                            \
 113                 return (f != NULL ? f : (pmap_##func##_t)pmap_nomethod);\
 114         }
 115 #define DEFINE_DUMPSYS_IFUNC(ret, func, args)                           \
 116         DEFINE_IFUNC(, ret, dumpsys_##func, args) {                     \
 117                 pmap_dumpsys_##func##_t f;                              \
 118                 f = PMAP_RESOLVE_FUNC(dumpsys_##func);                  \
 119                 return (f != NULL ? f : (pmap_dumpsys_##func##_t)pmap_nomethod);\
 120         }
 121
 122 DEFINE_PMAP_IFUNC(void, activate, (struct thread *));
 123 DEFINE_PMAP_IFUNC(void, advise, (pmap_t, vm_offset_t, vm_offset_t, int));
 124 DEFINE_PMAP_IFUNC(void, align_superpage, (vm_object_t, vm_ooffset_t,
 125         vm_offset_t *, vm_size_t));
 126 DEFINE_PMAP_IFUNC(void, clear_modify, (vm_page_t));
 127 DEFINE_PMAP_IFUNC(void, copy, (pmap_t, pmap_t, vm_offset_t, vm_size_t, vm_offset_t));
 128 DEFINE_PMAP_IFUNC(int, enter, (pmap_t, vm_offset_t, vm_page_t, vm_prot_t, u_int, int8_t));
 129 DEFINE_PMAP_IFUNC(void, enter_quick, (pmap_t, vm_offset_t, vm_page_t, vm_prot_t));
 130 DEFINE_PMAP_IFUNC(void, enter_object, (pmap_t, vm_offset_t, vm_offset_t, vm_page_t,
 131         vm_prot_t));
 132 DEFINE_PMAP_IFUNC(vm_paddr_t, extract, (pmap_t, vm_offset_t));
 133 DEFINE_PMAP_IFUNC(vm_page_t, extract_and_hold, (pmap_t, vm_offset_t, vm_prot_t));
 134 DEFINE_PMAP_IFUNC(void, kenter, (vm_offset_t, vm_paddr_t));
 135 DEFINE_PMAP_IFUNC(void, kenter_attr, (vm_offset_t, vm_paddr_t, vm_memattr_t));
 136 DEFINE_PMAP_IFUNC(vm_paddr_t, kextract, (vm_offset_t));
 137 DEFINE_PMAP_IFUNC(void, kremove, (vm_offset_t));
 138 DEFINE_PMAP_IFUNC(void, object_init_pt, (pmap_t, vm_offset_t, vm_object_t, vm_pindex_t,
 139         vm_size_t));
 140 DEFINE_PMAP_IFUNC(boolean_t, is_modified, (vm_page_t));
 141 DEFINE_PMAP_IFUNC(boolean_t, is_prefaultable, (pmap_t, vm_offset_t));
 142 DEFINE_PMAP_IFUNC(boolean_t, is_referenced, (vm_page_t));
 143 DEFINE_PMAP_IFUNC(boolean_t, page_exists_quick, (pmap_t, vm_page_t));
 144 DEFINE_PMAP_IFUNC(void, page_init, (vm_page_t));
 145 DEFINE_PMAP_IFUNC(boolean_t, page_is_mapped, (vm_page_t));
 146 DEFINE_PMAP_IFUNC(int, page_wired_mappings, (vm_page_t));
 147 DEFINE_PMAP_IFUNC(void, protect, (pmap_t, vm_offset_t, vm_offset_t, vm_prot_t));
 148 DEFINE_PMAP_IFUNC(bool, ps_enabled, (pmap_t));
 149 DEFINE_PMAP_IFUNC(void, qenter, (vm_offset_t, vm_page_t *, int));
 150 DEFINE_PMAP_IFUNC(void, qremove, (vm_offset_t, int));
 151 DEFINE_PMAP_IFUNC(vm_offset_t, quick_enter_page, (vm_page_t));
 152 DEFINE_PMAP_IFUNC(void, quick_remove_page, (vm_offset_t));
 153 DEFINE_PMAP_IFUNC(boolean_t, ts_referenced, (vm_page_t));
 154 DEFINE_PMAP_IFUNC(void, release, (pmap_t));
 155 DEFINE_PMAP_IFUNC(void, remove, (pmap_t, vm_offset_t, vm_offset_t));
 156 DEFINE_PMAP_IFUNC(void, remove_all, (vm_page_t));
 157 DEFINE_PMAP_IFUNC(void, remove_pages, (pmap_t));
 158 DEFINE_PMAP_IFUNC(void, remove_write, (vm_page_t));
 159 DEFINE_PMAP_IFUNC(void, unwire, (pmap_t, vm_offset_t, vm_offset_t));
 160 DEFINE_PMAP_IFUNC(void, zero_page, (vm_page_t));
 161 DEFINE_PMAP_IFUNC(void, zero_page_area, (vm_page_t, int, int));
 162 DEFINE_PMAP_IFUNC(void, copy_page, (vm_page_t, vm_page_t));
 163 DEFINE_PMAP_IFUNC(void, copy_pages,
 164     (vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[],
 165     vm_offset_t b_offset, int xfersize));
 166 DEFINE_PMAP_IFUNC(void, growkernel, (vm_offset_t));
 167 DEFINE_PMAP_IFUNC(void, init, (void));
 168 DEFINE_PMAP_IFUNC(vm_offset_t, map, (vm_offset_t *, vm_paddr_t, vm_paddr_t, int));
 169 DEFINE_PMAP_IFUNC(int, pinit, (pmap_t));
 170 DEFINE_PMAP_IFUNC(void, pinit0, (pmap_t));
 171 DEFINE_PMAP_IFUNC(int, mincore, (pmap_t, vm_offset_t, vm_paddr_t *));
 172 DEFINE_PMAP_IFUNC(void, deactivate, (struct thread *));
 173 DEFINE_PMAP_IFUNC(void, bootstrap, (vm_offset_t, vm_offset_t));
 174 DEFINE_PMAP_IFUNC(void, cpu_bootstrap, (int));
 175 DEFINE_PMAP_IFUNC(void *, mapdev, (vm_paddr_t, vm_size_t));
 176 DEFINE_PMAP_IFUNC(void *, mapdev_attr, (vm_paddr_t, vm_size_t, vm_memattr_t));
 177 DEFINE_PMAP_IFUNC(void, page_set_memattr, (vm_page_t, vm_memattr_t));
 178 DEFINE_PMAP_IFUNC(void, unmapdev, (vm_offset_t, vm_size_t));
 179 DEFINE_PMAP_IFUNC(int, map_user_ptr,
 180     (pmap_t, volatile const void *, void **, size_t, size_t *));
 181 DEFINE_PMAP_IFUNC(int, decode_kernel_ptr, (vm_offset_t, int *, vm_offset_t *));
 182 DEFINE_PMAP_IFUNC(boolean_t, dev_direct_mapped, (vm_paddr_t, vm_size_t));
 183 DEFINE_PMAP_IFUNC(void, sync_icache, (pmap_t, vm_offset_t, vm_size_t));
 184 DEFINE_PMAP_IFUNC(int, change_attr, (vm_offset_t, vm_size_t, vm_memattr_t));
 185 DEFINE_PMAP_IFUNC(void, page_array_startup, (long));
 186 DEFINE_PMAP_IFUNC(void, tlbie_all, (void));
 187
 188 DEFINE_DUMPSYS_IFUNC(void, map_chunk, (vm_paddr_t, size_t, void **));
 189 DEFINE_DUMPSYS_IFUNC(void, unmap_chunk, (vm_paddr_t, size_t, void *));
 190 DEFINE_DUMPSYS_IFUNC(void, pa_init, (void));
 191 DEFINE_DUMPSYS_IFUNC(size_t, scan_pmap, (void));
 192 DEFINE_DUMPSYS_IFUNC(void *, dump_pmap_init, (unsigned));
 193 DEFINE_DUMPSYS_IFUNC(void *, dump_pmap, (void *, void *, u_long *));
 194
 195 /*
 196  * MMU install routines. Highest priority wins, equal priority also
 197  * overrides allowing last-set to win.
 198  */
 199 SET_DECLARE(mmu_set, struct mmu_kobj);
 200
 201 boolean_t
 202 pmap_mmu_install(char *name, int prio)
 203 {
 204         mmu_t   *mmupp, mmup;
 205         static int      curr_prio = 0;
 206
 207         /*
 208          * Try and locate the MMU kobj corresponding to the name
 209          */
 210         SET_FOREACH(mmupp, mmu_set) {
 211                 mmup = *mmupp;
 212
 213                 if (mmup->name &&
 214                     !strcmp(mmup->name, name) &&
 215                     (prio >= curr_prio || mmu_obj == NULL)) {
 216                         curr_prio = prio;
 217                         mmu_obj = mmup;
 218                         return (TRUE);
 219                 }
 220         }
 221
 222         return (FALSE);
 223 }
 224
 225 /* MMU "pre-bootstrap" init, used to install extra resolvers, etc. */
 226 void
 227 pmap_mmu_init()
 228 {
 229         if (mmu_obj->funcs->install != NULL)
 230                 (mmu_obj->funcs->install)();
 231 }
 232
 233 const char *
 234 pmap_mmu_name(void)
 235 {
 236         return (mmu_obj->name);
 237 }
 238
 239 int unmapped_buf_allowed;
 240
 241 boolean_t
 242 pmap_is_valid_memattr(pmap_t pmap __unused, vm_memattr_t mode)
 243 {
 244
 245         switch (mode) {
 246         case VM_MEMATTR_DEFAULT:
 247         case VM_MEMATTR_UNCACHEABLE:
 248         case VM_MEMATTR_CACHEABLE:
 249         case VM_MEMATTR_WRITE_COMBINING:
 250         case VM_MEMATTR_WRITE_BACK:
 251         case VM_MEMATTR_WRITE_THROUGH:
 252         case VM_MEMATTR_PREFETCHABLE:
 253                 return (TRUE);
 254         default:
 255                 return (FALSE);
 256         }
 257 }