2 * Copyright (C) 2010 Nathan Whitehorn
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
18 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
20 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
21 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
22 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
23 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 #include <sys/cdefs.h>
27 __FBSDID("$FreeBSD$");
29 #include <sys/param.h>
30 #include <sys/kernel.h>
33 #include <sys/msgbuf.h>
34 #include <sys/mutex.h>
36 #include <sys/sysctl.h>
37 #include <sys/systm.h>
38 #include <sys/vmmeter.h>
41 #include <vm/vm_param.h>
42 #include <vm/vm_kern.h>
43 #include <vm/vm_page.h>
44 #include <vm/vm_map.h>
45 #include <vm/vm_object.h>
46 #include <vm/vm_extern.h>
47 #include <vm/vm_pageout.h>
48 #include <vm/vm_pager.h>
51 #include <powerpc/aim/mmu_oea64.h>
54 #include "moea64_if.h"
55 #include "ps3-hvcall.h"
57 #define VSID_HASH_MASK 0x0000007fffffffffUL
58 #define PTESYNC() __asm __volatile("ptesync")
60 extern int ps3fb_remap(void);
62 static uint64_t mps3_vas_id;
65 * Kernel MMU interface
68 static void mps3_bootstrap(mmu_t mmup, vm_offset_t kernelstart,
69 vm_offset_t kernelend);
70 static void mps3_cpu_bootstrap(mmu_t mmup, int ap);
71 static void mps3_pte_synch(mmu_t, uintptr_t pt, struct lpte *pvo_pt);
72 static void mps3_pte_clear(mmu_t, uintptr_t pt, struct lpte *pvo_pt,
73 uint64_t vpn, uint64_t ptebit);
74 static void mps3_pte_unset(mmu_t, uintptr_t pt, struct lpte *pvo_pt,
76 static void mps3_pte_change(mmu_t, uintptr_t pt, struct lpte *pvo_pt,
78 static int mps3_pte_insert(mmu_t, u_int ptegidx, struct lpte *pvo_pt);
79 static uintptr_t mps3_pvo_to_pte(mmu_t, const struct pvo_entry *pvo);
82 static mmu_method_t mps3_methods[] = {
83 MMUMETHOD(mmu_bootstrap, mps3_bootstrap),
84 MMUMETHOD(mmu_cpu_bootstrap, mps3_cpu_bootstrap),
86 MMUMETHOD(moea64_pte_synch, mps3_pte_synch),
87 MMUMETHOD(moea64_pte_clear, mps3_pte_clear),
88 MMUMETHOD(moea64_pte_unset, mps3_pte_unset),
89 MMUMETHOD(moea64_pte_change, mps3_pte_change),
90 MMUMETHOD(moea64_pte_insert, mps3_pte_insert),
91 MMUMETHOD(moea64_pvo_to_pte, mps3_pvo_to_pte),
96 MMU_DEF_INHERIT(ps3_mmu, "mmu_ps3", mps3_methods, 0, oea64_mmu);
99 mps3_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
101 uint64_t final_pteg_count;
103 moea64_early_bootstrap(mmup, kernelstart, kernelend);
105 lv1_construct_virtual_address_space(
106 20 /* log_2(moea64_pteg_count) */, 2 /* n page sizes */,
107 (24UL << 56) | (16UL << 48) /* page sizes 16 MB + 64 KB */,
108 &mps3_vas_id, &final_pteg_count
111 moea64_pteg_count = final_pteg_count / sizeof(struct lpteg);
113 moea64_mid_bootstrap(mmup, kernelstart, kernelend);
114 moea64_late_bootstrap(mmup, kernelstart, kernelend);
118 mps3_cpu_bootstrap(mmu_t mmup, int ap)
120 struct slb *slb = PCPU_GET(slb);
124 mtmsr(mfmsr() & ~PSL_DR & ~PSL_IR);
127 * Destroy the loader's address space if we are coming up for
128 * the first time, and redo the FB mapping so we can continue
133 lv1_destruct_virtual_address_space(0);
135 lv1_select_virtual_address_space(mps3_vas_id);
141 * Install kernel SLB entries
144 __asm __volatile ("slbia");
145 __asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) : "r"(0));
146 for (i = 0; i < 64; i++) {
147 if (!(slb[i].slbe & SLBE_VALID))
150 __asm __volatile ("slbmte %0, %1" ::
151 "r"(slb[i].slbv), "r"(slb[i].slbe));
156 mps3_pte_synch(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt)
158 uint64_t halfbucket[4], rcbits;
161 lv1_read_htab_entries(mps3_vas_id, slot & ~0x3UL, &halfbucket[0],
162 &halfbucket[1], &halfbucket[2], &halfbucket[3], &rcbits);
165 * rcbits contains the low 12 bits of each PTEs 2nd part,
166 * spaced at 16-bit intervals
169 KASSERT((halfbucket[slot & 0x3] & LPTE_AVPN_MASK) ==
170 (pvo_pt->pte_hi & LPTE_AVPN_MASK),
171 ("PTE upper word %#lx != %#lx\n",
172 halfbucket[slot & 0x3], pvo_pt->pte_hi));
174 pvo_pt->pte_lo |= (rcbits >> ((3 - (slot & 0x3))*16)) &
175 (LPTE_CHG | LPTE_REF);
179 mps3_pte_clear(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn,
183 lv1_write_htab_entry(mps3_vas_id, slot, pvo_pt->pte_hi,
184 pvo_pt->pte_lo & ~ptebit);
188 mps3_pte_unset(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn)
191 mps3_pte_synch(mmu, slot, pvo_pt);
192 pvo_pt->pte_hi &= ~LPTE_VALID;
193 lv1_write_htab_entry(mps3_vas_id, slot, 0, 0);
198 mps3_pte_change(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn)
201 mps3_pte_synch(mmu, slot, pvo_pt);
202 lv1_write_htab_entry(mps3_vas_id, slot, pvo_pt->pte_hi,
207 mps3_pte_insert(mmu_t mmu, u_int ptegidx, struct lpte *pvo_pt)
211 struct pvo_entry *pvo;
214 pvo_pt->pte_hi |= LPTE_VALID;
215 pvo_pt->pte_hi &= ~LPTE_HID;
218 result = lv1_insert_htab_entry(mps3_vas_id, ptegidx << 3,
219 pvo_pt->pte_hi, pvo_pt->pte_lo, LPTE_LOCKED | LPTE_WIRED, 0,
220 &index, &evicted.pte_hi, &evicted.pte_lo);
223 /* No freeable slots in either PTEG? We're hosed. */
224 panic("mps3_pte_insert: overflow (%d)", result);
229 * See where we ended up.
231 if (index >> 3 != ptegidx)
232 pvo_pt->pte_hi |= LPTE_HID;
237 return (index & 0x7);
240 * Synchronize the sacrifice PTE with its PVO, then mark both
241 * invalid. The PVO will be reused when/if the VM system comes
242 * here after a fault.
245 ptegidx = index >> 3; /* Where the sacrifice PTE was found */
246 if (evicted.pte_hi & LPTE_HID)
247 ptegidx ^= moea64_pteg_mask; /* PTEs indexed by primary */
249 KASSERT((evicted.pte_hi & (LPTE_WIRED | LPTE_LOCKED)) == 0,
250 ("Evicted a wired PTE"));
253 LIST_FOREACH(pvo, &moea64_pvo_table[ptegidx], pvo_olink) {
254 if (!PVO_PTEGIDX_ISSET(pvo))
257 if (pvo->pvo_pte.lpte.pte_hi == (evicted.pte_hi | LPTE_VALID)) {
258 KASSERT(pvo->pvo_pte.lpte.pte_hi & LPTE_VALID,
259 ("Invalid PVO for valid PTE!"));
260 pvo->pvo_pte.lpte.pte_hi &= ~LPTE_VALID;
261 pvo->pvo_pte.lpte.pte_lo |=
262 evicted.pte_lo & (LPTE_REF | LPTE_CHG);
263 PVO_PTEGIDX_CLR(pvo);
265 moea64_pte_overflow++;
271 KASSERT(result == 1, ("PVO for sacrifice PTE not found"));
273 return (index & 0x7);
276 static __inline u_int
277 va_to_pteg(uint64_t vsid, vm_offset_t addr, int large)
282 shift = large ? moea64_large_page_shift : ADDR_PIDX_SHFT;
283 hash = (vsid & VSID_HASH_MASK) ^ (((uint64_t)addr & ADDR_PIDX) >>
285 return (hash & moea64_pteg_mask);
289 mps3_pvo_to_pte(mmu_t mmu, const struct pvo_entry *pvo)
294 /* If the PTEG index is not set, then there is no page table entry */
295 if (!PVO_PTEGIDX_ISSET(pvo))
298 vsid = PVO_VSID(pvo);
299 ptegidx = va_to_pteg(vsid, PVO_VADDR(pvo), pvo->pvo_vaddr & PVO_LARGE);
302 * We can find the actual pte entry without searching by grabbing
303 * the PTEG index from 3 unused bits in pvo_vaddr and by
304 * noticing the HID bit.
306 if (pvo->pvo_pte.lpte.pte_hi & LPTE_HID)
307 ptegidx ^= moea64_pteg_mask;
309 return ((ptegidx << 3) | PVO_PTEGIDX_GET(pvo));