2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2010 Nathan Whitehorn
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 #include <sys/param.h>
32 #include <sys/kernel.h>
34 #include <sys/malloc.h>
35 #include <sys/mutex.h>
37 #include <sys/systm.h>
43 #include <vm/vm_map.h>
44 #include <vm/vm_page.h>
45 #include <vm/vm_pageout.h>
47 #include <machine/md_var.h>
48 #include <machine/platform.h>
49 #include <machine/vmparam.h>
50 #include <machine/trap.h>
52 #include "mmu_oea64.h"
54 uintptr_t moea64_get_unique_vsid(void);
55 void moea64_release_vsid(uint64_t vsid);
56 static void slb_zone_init(void *);
58 static uma_zone_t slbt_zone;
59 static uma_zone_t slb_cache_zone;
62 SYSINIT(slb_zone_init, SI_SUB_KMEM, SI_ORDER_ANY, slb_zone_init, NULL);
67 /* Only 36 bits needed for full 64-bit address space. */
70 struct slbtnode *ua_child[16];
71 struct slb slb_entries[16];
76 * For a full 64-bit address space, there are 36 bits in play in an
77 * esid, so 8 levels, with the leaf being at level 0.
79 * |3333|3322|2222|2222|1111|1111|11 | | | esid
80 * |5432|1098|7654|3210|9876|5432|1098|7654|3210| bits
81 * +----+----+----+----+----+----+----+----+----+--------
82 * | 8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 | level
84 #define UAD_ROOT_LEVEL 8
85 #define UAD_LEAF_LEVEL 0
88 esid2idx(uint64_t esid, int level)
93 return ((esid >> shift) & 0xF);
97 * The ua_base field should have 0 bits after the first 4*(level+1)
100 #define uad_baseok(ua) \
101 (esid2base(ua->ua_base, ua->ua_level) == ua->ua_base)
103 static inline uint64_t
104 esid2base(uint64_t esid, int level)
109 shift = (level + 1) * 4;
110 mask = ~((1ULL << shift) - 1);
111 return (esid & mask);
115 * Allocate a new leaf node for the specified esid/vmhandle from the
119 make_new_leaf(uint64_t esid, uint64_t slbv, struct slbtnode *parent)
121 struct slbtnode *child;
125 idx = esid2idx(esid, parent->ua_level);
126 KASSERT(parent->u.ua_child[idx] == NULL, ("Child already exists!"));
128 /* unlock and M_WAITOK and loop? */
129 child = uma_zalloc(slbt_zone, M_NOWAIT | M_ZERO);
130 KASSERT(child != NULL, ("unhandled NULL case"));
132 child->ua_level = UAD_LEAF_LEVEL;
133 child->ua_base = esid2base(esid, child->ua_level);
134 idx = esid2idx(esid, child->ua_level);
135 child->u.slb_entries[idx].slbv = slbv;
136 child->u.slb_entries[idx].slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID;
137 setbit(&child->ua_alloc, idx);
139 retval = &child->u.slb_entries[idx];
142 * The above stores must be visible before the next one, so
143 * that a lockless searcher always sees a valid path through
148 idx = esid2idx(esid, parent->ua_level);
149 parent->u.ua_child[idx] = child;
150 setbit(&parent->ua_alloc, idx);
156 * Allocate a new intermediate node to fit between the parent and
159 static struct slbtnode*
160 make_intermediate(uint64_t esid, struct slbtnode *parent)
162 struct slbtnode *child, *inter;
165 idx = esid2idx(esid, parent->ua_level);
166 child = parent->u.ua_child[idx];
167 KASSERT(esid2base(esid, child->ua_level) != child->ua_base,
168 ("No need for an intermediate node?"));
171 * Find the level where the existing child and our new esid
172 * meet. It must be lower than parent->ua_level or we would
173 * have chosen a different index in parent.
175 level = child->ua_level + 1;
176 while (esid2base(esid, level) !=
177 esid2base(child->ua_base, level))
179 KASSERT(level < parent->ua_level,
180 ("Found splitting level %d for %09jx and %09jx, "
181 "but it's the same as %p's",
182 level, esid, child->ua_base, parent));
184 /* unlock and M_WAITOK and loop? */
185 inter = uma_zalloc(slbt_zone, M_NOWAIT | M_ZERO);
186 KASSERT(inter != NULL, ("unhandled NULL case"));
188 /* Set up intermediate node to point to child ... */
189 inter->ua_level = level;
190 inter->ua_base = esid2base(esid, inter->ua_level);
191 idx = esid2idx(child->ua_base, inter->ua_level);
192 inter->u.ua_child[idx] = child;
193 setbit(&inter->ua_alloc, idx);
196 /* Set up parent to point to intermediate node ... */
197 idx = esid2idx(inter->ua_base, parent->ua_level);
198 parent->u.ua_child[idx] = inter;
199 setbit(&parent->ua_alloc, idx);
205 kernel_va_to_slbv(vm_offset_t va)
209 /* Set kernel VSID to deterministic value */
210 slbv = (KERNEL_VSID((uintptr_t)va >> ADDR_SR_SHFT)) << SLBV_VSID_SHIFT;
213 * Figure out if this is a large-page mapping.
215 if (hw_direct_map && va > DMAP_BASE_ADDRESS && va < DMAP_MAX_ADDRESS) {
217 * XXX: If we have set up a direct map, assumes
218 * all physical memory is mapped with large pages.
221 if (mem_valid(DMAP_TO_PHYS(va), 0) == 0)
223 } else if (moea64_large_page_size != 0 &&
224 va >= (vm_offset_t)vm_page_array &&
225 va <= (uintptr_t)(&vm_page_array[vm_page_array_size]))
232 user_va_to_slb_entry(pmap_t pm, vm_offset_t va)
234 uint64_t esid = va >> ADDR_SR_SHFT;
238 ua = pm->pm_slb_tree_root;
241 KASSERT(uad_baseok(ua), ("uad base %016jx level %d bad!",
242 ua->ua_base, ua->ua_level));
243 idx = esid2idx(esid, ua->ua_level);
246 * This code is specific to ppc64 where a load is
247 * atomic, so no need for atomic_load macro.
249 if (ua->ua_level == UAD_LEAF_LEVEL)
250 return ((ua->u.slb_entries[idx].slbe & SLBE_VALID) ?
251 &ua->u.slb_entries[idx] : NULL);
254 * The following accesses are implicitly ordered under the POWER
255 * ISA by load dependencies (the store ordering is provided by
256 * the powerpc_lwsync() calls elsewhere) and so are run without
259 ua = ua->u.ua_child[idx];
261 esid2base(esid, ua->ua_level) != ua->ua_base)
269 va_to_vsid(pmap_t pm, vm_offset_t va)
273 /* Shortcut kernel case */
274 if (pm == kernel_pmap)
275 return (KERNEL_VSID((uintptr_t)va >> ADDR_SR_SHFT));
278 * If there is no vsid for this VA, we need to add a new entry
279 * to the PMAP's segment table.
282 entry = user_va_to_slb_entry(pm, va);
285 return (allocate_user_vsid(pm,
286 (uintptr_t)va >> ADDR_SR_SHFT, 0));
288 return ((entry->slbv & SLBV_VSID_MASK) >> SLBV_VSID_SHIFT);
292 allocate_user_vsid(pmap_t pm, uint64_t esid, int large)
295 struct slbtnode *ua, *next, *inter;
299 KASSERT(pm != kernel_pmap, ("Attempting to allocate a kernel VSID"));
301 PMAP_LOCK_ASSERT(pm, MA_OWNED);
302 vsid = moea64_get_unique_vsid();
304 slbv = vsid << SLBV_VSID_SHIFT;
308 ua = pm->pm_slb_tree_root;
310 /* Descend to the correct leaf or NULL pointer. */
312 KASSERT(uad_baseok(ua),
313 ("uad base %09jx level %d bad!", ua->ua_base, ua->ua_level));
314 idx = esid2idx(esid, ua->ua_level);
316 if (ua->ua_level == UAD_LEAF_LEVEL) {
317 ua->u.slb_entries[idx].slbv = slbv;
319 ua->u.slb_entries[idx].slbe = (esid << SLBE_ESID_SHIFT)
321 setbit(&ua->ua_alloc, idx);
322 slb = &ua->u.slb_entries[idx];
326 next = ua->u.ua_child[idx];
328 slb = make_new_leaf(esid, slbv, ua);
333 * Check if the next item down has an okay ua_base.
334 * If not, we need to allocate an intermediate node.
336 if (esid2base(esid, next->ua_level) != next->ua_base) {
337 inter = make_intermediate(esid, ua);
338 slb = make_new_leaf(esid, slbv, inter);
346 * Someone probably wants this soon, and it may be a wired
347 * SLB mapping, so pre-spill this entry.
350 slb_insert_user(pm, slb);
356 free_vsid(pmap_t pm, uint64_t esid, int large)
361 PMAP_LOCK_ASSERT(pm, MA_OWNED);
363 ua = pm->pm_slb_tree_root;
364 /* Descend to the correct leaf. */
366 KASSERT(uad_baseok(ua),
367 ("uad base %09jx level %d bad!", ua->ua_base, ua->ua_level));
369 idx = esid2idx(esid, ua->ua_level);
370 if (ua->ua_level == UAD_LEAF_LEVEL) {
371 ua->u.slb_entries[idx].slbv = 0;
373 ua->u.slb_entries[idx].slbe = 0;
374 clrbit(&ua->ua_alloc, idx);
378 ua = ua->u.ua_child[idx];
380 esid2base(esid, ua->ua_level) != ua->ua_base) {
381 /* Perhaps just return instead of assert? */
383 ("Asked to remove an entry that was never inserted!"));
390 free_slb_tree_node(struct slbtnode *ua)
394 for (idx = 0; idx < 16; idx++) {
395 if (ua->ua_level != UAD_LEAF_LEVEL) {
396 if (ua->u.ua_child[idx] != NULL)
397 free_slb_tree_node(ua->u.ua_child[idx]);
399 if (ua->u.slb_entries[idx].slbv != 0)
400 moea64_release_vsid(ua->u.slb_entries[idx].slbv
405 uma_zfree(slbt_zone, ua);
409 slb_free_tree(pmap_t pm)
412 free_slb_tree_node(pm->pm_slb_tree_root);
418 struct slbtnode *root;
420 root = uma_zalloc(slbt_zone, M_NOWAIT | M_ZERO);
421 KASSERT(root != NULL, ("unhandled NULL case"));
422 root->ua_level = UAD_ROOT_LEVEL;
427 /* Lock entries mapping kernel text and stacks */
430 slb_insert_kernel(uint64_t slbe, uint64_t slbv)
432 struct slb *slbcache;
435 /* We don't want to be preempted while modifying the kernel map */
438 slbcache = PCPU_GET(aim.slb);
440 /* Check for an unused slot, abusing the user slot as a full flag */
441 if (slbcache[USER_SLB_SLOT].slbe == 0) {
442 for (i = 0; i < n_slbs; i++) {
443 if (i == USER_SLB_SLOT)
445 if (!(slbcache[i].slbe & SLBE_VALID))
450 slbcache[USER_SLB_SLOT].slbe = 1;
454 if (i == USER_SLB_SLOT)
458 KASSERT(i != USER_SLB_SLOT,
459 ("Filling user SLB slot with a kernel mapping"));
460 slbcache[i].slbv = slbv;
461 slbcache[i].slbe = slbe | (uint64_t)i;
463 /* If it is for this CPU, put it in the SLB right away */
464 if (pmap_bootstrapped) {
465 /* slbie not required */
466 __asm __volatile ("slbmte %0, %1" ::
467 "r"(slbcache[i].slbv), "r"(slbcache[i].slbe));
474 slb_insert_user(pmap_t pm, struct slb *slb)
478 PMAP_LOCK_ASSERT(pm, MA_OWNED);
480 if (pm->pm_slb_len < n_slbs) {
487 /* Note that this replacement is atomic with respect to trap_subr */
492 slb_uma_real_alloc(uma_zone_t zone, vm_size_t bytes, int domain,
493 u_int8_t *flags, int wait)
495 static vm_offset_t realmax = 0;
500 realmax = platform_real_maxaddr();
502 *flags = UMA_SLAB_PRIV;
503 m = vm_page_alloc_noobj_contig_domain(domain, malloc2vm_flags(wait) |
504 VM_ALLOC_WIRED, 1, 0, realmax, PAGE_SIZE, PAGE_SIZE,
510 va = (void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
512 va = (void *)(VM_PAGE_TO_PHYS(m) | DMAP_BASE_ADDRESS);
513 pmap_kenter((vm_offset_t)va, VM_PAGE_TO_PHYS(m));
520 slb_zone_init(void *dummy)
522 slbt_zone = uma_zcreate("SLB tree node", sizeof(struct slbtnode),
523 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
524 UMA_ZONE_CONTIG | UMA_ZONE_VM);
525 slb_cache_zone = uma_zcreate("SLB cache",
526 (n_slbs + 1)*sizeof(struct slb *), NULL, NULL, NULL, NULL,
527 UMA_ALIGN_PTR, UMA_ZONE_CONTIG | UMA_ZONE_VM);
529 if (platform_real_maxaddr() != VM_MAX_ADDRESS) {
530 uma_zone_set_allocf(slb_cache_zone, slb_uma_real_alloc);
531 uma_zone_set_allocf(slbt_zone, slb_uma_real_alloc);
536 slb_alloc_user_cache(void)
538 return (uma_zalloc(slb_cache_zone, M_ZERO));
542 slb_free_user_cache(struct slb **slb)
544 uma_zfree(slb_cache_zone, slb);
547 /* Handle kernel SLB faults -- runs in real mode, all seat belts off */
549 handle_kernel_slb_spill(int type, register_t dar, register_t srr0)
551 struct slb *slbcache;
556 addr = (type == EXC_ISE) ? srr0 : dar;
557 slbcache = PCPU_GET(aim.slb);
558 esid = (uintptr_t)addr >> ADDR_SR_SHFT;
559 slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID;
561 /* See if the hardware flushed this somehow (can happen in LPARs) */
562 for (i = 0; i < n_slbs; i++)
563 if (slbcache[i].slbe == (slbe | (uint64_t)i))
566 /* Not in the map, needs to actually be added */
567 slbv = kernel_va_to_slbv(addr);
568 if (slbcache[USER_SLB_SLOT].slbe == 0) {
569 for (i = 0; i < n_slbs; i++) {
570 if (i == USER_SLB_SLOT)
572 if (!(slbcache[i].slbe & SLBE_VALID))
577 slbcache[USER_SLB_SLOT].slbe = 1;
580 /* Sacrifice a random SLB entry that is not the user entry */
582 if (i == USER_SLB_SLOT)
586 /* Write new entry */
587 slbcache[i].slbv = slbv;
588 slbcache[i].slbe = slbe | (uint64_t)i;
590 /* Trap handler will restore from cache on exit */
594 handle_user_slb_spill(pmap_t pm, vm_offset_t addr)
596 struct slb *user_entry;
600 if (pm->pm_slb == NULL)
603 esid = (uintptr_t)addr >> ADDR_SR_SHFT;
606 user_entry = user_va_to_slb_entry(pm, addr);
608 if (user_entry == NULL) {
609 /* allocate_vsid auto-spills it */
610 (void)allocate_user_vsid(pm, esid, 0);
613 * Check that another CPU has not already mapped this.
614 * XXX: Per-thread SLB caches would be better.
616 for (i = 0; i < pm->pm_slb_len; i++)
617 if (pm->pm_slb[i] == user_entry)
620 if (i == pm->pm_slb_len)
621 slb_insert_user(pm, user_entry);