2 * Copyright (c) 2010 Nathan Whitehorn
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 #include <sys/param.h>
30 #include <sys/kernel.h>
32 #include <sys/malloc.h>
33 #include <sys/mutex.h>
35 #include <sys/systm.h>
41 #include <vm/vm_map.h>
42 #include <vm/vm_page.h>
43 #include <vm/vm_pageout.h>
45 #include <machine/md_var.h>
46 #include <machine/platform.h>
47 #include <machine/vmparam.h>
49 uintptr_t moea64_get_unique_vsid(void);
50 void moea64_release_vsid(uint64_t vsid);
51 static void slb_zone_init(void *);
53 static uma_zone_t slbt_zone;
54 static uma_zone_t slb_cache_zone;
57 SYSINIT(slb_zone_init, SI_SUB_KMEM, SI_ORDER_ANY, slb_zone_init, NULL);
62 /* Only 36 bits needed for full 64-bit address space. */
65 struct slbtnode *ua_child[16];
66 struct slb slb_entries[16];
71 * For a full 64-bit address space, there are 36 bits in play in an
72 * esid, so 8 levels, with the leaf being at level 0.
74 * |3333|3322|2222|2222|1111|1111|11 | | | esid
75 * |5432|1098|7654|3210|9876|5432|1098|7654|3210| bits
76 * +----+----+----+----+----+----+----+----+----+--------
77 * | 8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 | level
79 #define UAD_ROOT_LEVEL 8
80 #define UAD_LEAF_LEVEL 0
83 esid2idx(uint64_t esid, int level)
88 return ((esid >> shift) & 0xF);
92 * The ua_base field should have 0 bits after the first 4*(level+1)
95 #define uad_baseok(ua) \
96 (esid2base(ua->ua_base, ua->ua_level) == ua->ua_base)
99 static inline uint64_t
100 esid2base(uint64_t esid, int level)
105 shift = (level + 1) * 4;
106 mask = ~((1ULL << shift) - 1);
107 return (esid & mask);
111 * Allocate a new leaf node for the specified esid/vmhandle from the
115 make_new_leaf(uint64_t esid, uint64_t slbv, struct slbtnode *parent)
117 struct slbtnode *child;
121 idx = esid2idx(esid, parent->ua_level);
122 KASSERT(parent->u.ua_child[idx] == NULL, ("Child already exists!"));
124 /* unlock and M_WAITOK and loop? */
125 child = uma_zalloc(slbt_zone, M_NOWAIT | M_ZERO);
126 KASSERT(child != NULL, ("unhandled NULL case"));
128 child->ua_level = UAD_LEAF_LEVEL;
129 child->ua_base = esid2base(esid, child->ua_level);
130 idx = esid2idx(esid, child->ua_level);
131 child->u.slb_entries[idx].slbv = slbv;
132 child->u.slb_entries[idx].slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID;
133 setbit(&child->ua_alloc, idx);
135 retval = &child->u.slb_entries[idx];
138 * The above stores must be visible before the next one, so
139 * that a lockless searcher always sees a valid path through
144 idx = esid2idx(esid, parent->ua_level);
145 parent->u.ua_child[idx] = child;
146 setbit(&parent->ua_alloc, idx);
152 * Allocate a new intermediate node to fit between the parent and
155 static struct slbtnode*
156 make_intermediate(uint64_t esid, struct slbtnode *parent)
158 struct slbtnode *child, *inter;
161 idx = esid2idx(esid, parent->ua_level);
162 child = parent->u.ua_child[idx];
163 KASSERT(esid2base(esid, child->ua_level) != child->ua_base,
164 ("No need for an intermediate node?"));
167 * Find the level where the existing child and our new esid
168 * meet. It must be lower than parent->ua_level or we would
169 * have chosen a different index in parent.
171 level = child->ua_level + 1;
172 while (esid2base(esid, level) !=
173 esid2base(child->ua_base, level))
175 KASSERT(level < parent->ua_level,
176 ("Found splitting level %d for %09jx and %09jx, "
177 "but it's the same as %p's",
178 level, esid, child->ua_base, parent));
180 /* unlock and M_WAITOK and loop? */
181 inter = uma_zalloc(slbt_zone, M_NOWAIT | M_ZERO);
182 KASSERT(inter != NULL, ("unhandled NULL case"));
184 /* Set up intermediate node to point to child ... */
185 inter->ua_level = level;
186 inter->ua_base = esid2base(esid, inter->ua_level);
187 idx = esid2idx(child->ua_base, inter->ua_level);
188 inter->u.ua_child[idx] = child;
189 setbit(&inter->ua_alloc, idx);
192 /* Set up parent to point to intermediate node ... */
193 idx = esid2idx(inter->ua_base, parent->ua_level);
194 parent->u.ua_child[idx] = inter;
195 setbit(&parent->ua_alloc, idx);
201 kernel_va_to_slbv(vm_offset_t va)
205 /* Set kernel VSID to deterministic value */
206 slbv = (KERNEL_VSID((uintptr_t)va >> ADDR_SR_SHFT)) << SLBV_VSID_SHIFT;
208 /* Figure out if this is a large-page mapping */
209 if (hw_direct_map && va < VM_MIN_KERNEL_ADDRESS) {
211 * XXX: If we have set up a direct map, assumes
212 * all physical memory is mapped with large pages.
214 if (mem_valid(va, 0) == 0)
222 user_va_to_slb_entry(pmap_t pm, vm_offset_t va)
224 uint64_t esid = va >> ADDR_SR_SHFT;
228 ua = pm->pm_slb_tree_root;
231 KASSERT(uad_baseok(ua), ("uad base %016jx level %d bad!",
232 ua->ua_base, ua->ua_level));
233 idx = esid2idx(esid, ua->ua_level);
236 * This code is specific to ppc64 where a load is
237 * atomic, so no need for atomic_load macro.
239 if (ua->ua_level == UAD_LEAF_LEVEL)
240 return ((ua->u.slb_entries[idx].slbe & SLBE_VALID) ?
241 &ua->u.slb_entries[idx] : NULL);
244 * The following accesses are implicitly ordered under the POWER
245 * ISA by load dependencies (the store ordering is provided by
246 * the powerpc_lwsync() calls elsewhere) and so are run without
249 ua = ua->u.ua_child[idx];
251 esid2base(esid, ua->ua_level) != ua->ua_base)
259 va_to_vsid(pmap_t pm, vm_offset_t va)
263 /* Shortcut kernel case */
264 if (pm == kernel_pmap)
265 return (KERNEL_VSID((uintptr_t)va >> ADDR_SR_SHFT));
268 * If there is no vsid for this VA, we need to add a new entry
269 * to the PMAP's segment table.
272 entry = user_va_to_slb_entry(pm, va);
275 return (allocate_user_vsid(pm,
276 (uintptr_t)va >> ADDR_SR_SHFT, 0));
278 return ((entry->slbv & SLBV_VSID_MASK) >> SLBV_VSID_SHIFT);
282 allocate_user_vsid(pmap_t pm, uint64_t esid, int large)
285 struct slbtnode *ua, *next, *inter;
289 KASSERT(pm != kernel_pmap, ("Attempting to allocate a kernel VSID"));
291 PMAP_LOCK_ASSERT(pm, MA_OWNED);
292 vsid = moea64_get_unique_vsid();
294 slbv = vsid << SLBV_VSID_SHIFT;
298 ua = pm->pm_slb_tree_root;
300 /* Descend to the correct leaf or NULL pointer. */
302 KASSERT(uad_baseok(ua),
303 ("uad base %09jx level %d bad!", ua->ua_base, ua->ua_level));
304 idx = esid2idx(esid, ua->ua_level);
306 if (ua->ua_level == UAD_LEAF_LEVEL) {
307 ua->u.slb_entries[idx].slbv = slbv;
309 ua->u.slb_entries[idx].slbe = (esid << SLBE_ESID_SHIFT)
311 setbit(&ua->ua_alloc, idx);
312 slb = &ua->u.slb_entries[idx];
316 next = ua->u.ua_child[idx];
318 slb = make_new_leaf(esid, slbv, ua);
323 * Check if the next item down has an okay ua_base.
324 * If not, we need to allocate an intermediate node.
326 if (esid2base(esid, next->ua_level) != next->ua_base) {
327 inter = make_intermediate(esid, ua);
328 slb = make_new_leaf(esid, slbv, inter);
336 * Someone probably wants this soon, and it may be a wired
337 * SLB mapping, so pre-spill this entry.
340 slb_insert_user(pm, slb);
346 free_vsid(pmap_t pm, uint64_t esid, int large)
351 PMAP_LOCK_ASSERT(pm, MA_OWNED);
353 ua = pm->pm_slb_tree_root;
354 /* Descend to the correct leaf. */
356 KASSERT(uad_baseok(ua),
357 ("uad base %09jx level %d bad!", ua->ua_base, ua->ua_level));
359 idx = esid2idx(esid, ua->ua_level);
360 if (ua->ua_level == UAD_LEAF_LEVEL) {
361 ua->u.slb_entries[idx].slbv = 0;
363 ua->u.slb_entries[idx].slbe = 0;
364 clrbit(&ua->ua_alloc, idx);
368 ua = ua->u.ua_child[idx];
370 esid2base(esid, ua->ua_level) != ua->ua_base) {
371 /* Perhaps just return instead of assert? */
373 ("Asked to remove an entry that was never inserted!"));
380 free_slb_tree_node(struct slbtnode *ua)
384 for (idx = 0; idx < 16; idx++) {
385 if (ua->ua_level != UAD_LEAF_LEVEL) {
386 if (ua->u.ua_child[idx] != NULL)
387 free_slb_tree_node(ua->u.ua_child[idx]);
389 if (ua->u.slb_entries[idx].slbv != 0)
390 moea64_release_vsid(ua->u.slb_entries[idx].slbv
395 uma_zfree(slbt_zone, ua);
399 slb_free_tree(pmap_t pm)
402 free_slb_tree_node(pm->pm_slb_tree_root);
408 struct slbtnode *root;
410 root = uma_zalloc(slbt_zone, M_NOWAIT | M_ZERO);
411 root->ua_level = UAD_ROOT_LEVEL;
416 /* Lock entries mapping kernel text and stacks */
419 slb_insert_kernel(uint64_t slbe, uint64_t slbv)
421 struct slb *slbcache;
424 /* We don't want to be preempted while modifying the kernel map */
427 slbcache = PCPU_GET(slb);
429 /* Check for an unused slot, abusing the user slot as a full flag */
430 if (slbcache[USER_SLB_SLOT].slbe == 0) {
431 for (i = 0; i < n_slbs; i++) {
432 if (i == USER_SLB_SLOT)
434 if (!(slbcache[i].slbe & SLBE_VALID))
439 slbcache[USER_SLB_SLOT].slbe = 1;
443 if (i == USER_SLB_SLOT)
447 KASSERT(i != USER_SLB_SLOT,
448 ("Filling user SLB slot with a kernel mapping"));
449 slbcache[i].slbv = slbv;
450 slbcache[i].slbe = slbe | (uint64_t)i;
452 /* If it is for this CPU, put it in the SLB right away */
453 if (pmap_bootstrapped) {
454 /* slbie not required */
455 __asm __volatile ("slbmte %0, %1" ::
456 "r"(slbcache[i].slbv), "r"(slbcache[i].slbe));
463 slb_insert_user(pmap_t pm, struct slb *slb)
467 PMAP_LOCK_ASSERT(pm, MA_OWNED);
469 if (pm->pm_slb_len < n_slbs) {
476 /* Note that this replacement is atomic with respect to trap_subr */
481 slb_uma_real_alloc(uma_zone_t zone, vm_size_t bytes, u_int8_t *flags, int wait)
483 static vm_offset_t realmax = 0;
489 realmax = platform_real_maxaddr();
491 *flags = UMA_SLAB_PRIV;
492 pflags = malloc2vm_flags(wait) | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED;
495 m = vm_page_alloc_contig(NULL, 0, pflags, 1, 0, realmax,
496 PAGE_SIZE, PAGE_SIZE, VM_MEMATTR_DEFAULT);
505 va = (void *) VM_PAGE_TO_PHYS(m);
508 pmap_kenter((vm_offset_t)va, VM_PAGE_TO_PHYS(m));
510 if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0)
511 bzero(va, PAGE_SIZE);
517 slb_zone_init(void *dummy)
520 slbt_zone = uma_zcreate("SLB tree node", sizeof(struct slbtnode),
521 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM);
522 slb_cache_zone = uma_zcreate("SLB cache",
523 (n_slbs + 1)*sizeof(struct slb *), NULL, NULL, NULL, NULL,
524 UMA_ALIGN_PTR, UMA_ZONE_VM);
526 if (platform_real_maxaddr() != VM_MAX_ADDRESS) {
527 uma_zone_set_allocf(slb_cache_zone, slb_uma_real_alloc);
528 uma_zone_set_allocf(slbt_zone, slb_uma_real_alloc);
533 slb_alloc_user_cache(void)
535 return (uma_zalloc(slb_cache_zone, M_ZERO));
539 slb_free_user_cache(struct slb **slb)
541 uma_zfree(slb_cache_zone, slb);