2 * Copyright (c) 2004-2010 Juli Mallett <jmallett@FreeBSD.org>
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 #include <sys/param.h>
32 #include <sys/kernel.h>
33 #include <sys/systm.h>
40 #include <machine/pte.h>
41 #include <machine/tlb.h>
43 #if defined(CPU_CNMIPS)
44 #define MIPS_MAX_TLB_ENTRIES 128
45 #elif defined(CPU_NLM)
46 #define MIPS_MAX_TLB_ENTRIES (2048 + 128)
48 #define MIPS_MAX_TLB_ENTRIES 64
58 } entry[MIPS_MAX_TLB_ENTRIES];
61 static struct tlb_state tlb_state[MAXCPU];
65 * PageMask must increment in steps of 2 bits.
67 COMPILE_TIME_ASSERT(POPCNT(TLBMASK_MASK) % 2 == 0);
73 __asm __volatile ("tlbp" : : : "memory");
80 __asm __volatile ("tlbr" : : : "memory");
85 tlb_write_indexed(void)
87 __asm __volatile ("tlbwi" : : : "memory");
91 static void tlb_invalidate_one(unsigned);
94 tlb_insert_wired(unsigned i, vm_offset_t va, pt_entry_t pte0, pt_entry_t pte1)
102 asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
106 mips_wr_entryhi(TLBHI_ENTRY(va, 0));
107 mips_wr_entrylo0(pte0);
108 mips_wr_entrylo1(pte1);
111 mips_wr_entryhi(asid);
116 tlb_invalidate_address(struct pmap *pmap, vm_offset_t va)
125 asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
128 mips_wr_entryhi(TLBHI_ENTRY(va, pmap_asid(pmap)));
132 tlb_invalidate_one(i);
134 mips_wr_entryhi(asid);
139 tlb_invalidate_all(void)
146 asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
148 for (i = mips_rd_wired(); i < num_tlbentries; i++)
149 tlb_invalidate_one(i);
151 mips_wr_entryhi(asid);
156 tlb_invalidate_all_user(struct pmap *pmap)
163 asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
165 for (i = mips_rd_wired(); i < num_tlbentries; i++) {
171 uasid = mips_rd_entryhi() & TLBHI_ASID_MASK;
174 * Invalidate all non-kernel entries.
180 * Invalidate this pmap's entries.
182 if (uasid != pmap_asid(pmap))
185 tlb_invalidate_one(i);
188 mips_wr_entryhi(asid);
193 * Invalidates any TLB entries that map a virtual page from the specified
194 * address range. If "end" is zero, then every virtual page is considered to
195 * be within the address range's upper bound.
198 tlb_invalidate_range(pmap_t pmap, vm_offset_t start, vm_offset_t end)
200 register_t asid, end_hi, hi, hi_pagemask, s, save_asid, start_hi;
203 KASSERT(start < end || (end == 0 && start > 0),
204 ("tlb_invalidate_range: invalid range"));
207 * Truncate the virtual address "start" to an even page frame number,
208 * and round the virtual address "end" to an even page frame number.
210 start &= ~((1 << TLBMASK_SHIFT) - 1);
211 end = roundup2(end, 1 << TLBMASK_SHIFT);
214 save_asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
216 asid = pmap_asid(pmap);
217 start_hi = TLBHI_ENTRY(start, asid);
218 end_hi = TLBHI_ENTRY(end, asid);
221 * Select the fastest method for invalidating the TLB entries.
223 if (end - start < num_tlbentries << TLBMASK_SHIFT || (end == 0 &&
224 start >= -(num_tlbentries << TLBMASK_SHIFT))) {
226 * The virtual address range is small compared to the size of
227 * the TLB. Probe the TLB for each even numbered page frame
228 * within the virtual address range.
230 for (hi = start_hi; hi != end_hi; hi += 1 << TLBMASK_SHIFT) {
236 tlb_invalidate_one(i);
240 * The virtual address range is large compared to the size of
241 * the TLB. Test every non-wired TLB entry.
243 for (i = mips_rd_wired(); i < num_tlbentries; i++) {
246 hi = mips_rd_entryhi();
247 if ((hi & TLBHI_ASID_MASK) == asid && (hi < end_hi ||
250 * If "hi" is a large page that spans
251 * "start_hi", then it must be invalidated.
253 hi_pagemask = mips_rd_pagemask();
254 if (hi >= (start_hi & ~(hi_pagemask <<
256 tlb_invalidate_one(i);
261 mips_wr_entryhi(save_asid);
265 /* XXX Only if DDB? */
269 unsigned ntlb, i, cpu;
271 cpu = PCPU_GET(cpuid);
272 if (num_tlbentries > MIPS_MAX_TLB_ENTRIES)
273 ntlb = MIPS_MAX_TLB_ENTRIES;
275 ntlb = num_tlbentries;
276 tlb_state[cpu].wired = mips_rd_wired();
277 for (i = 0; i < ntlb; i++) {
281 tlb_state[cpu].entry[i].entryhi = mips_rd_entryhi();
282 tlb_state[cpu].entry[i].pagemask = mips_rd_pagemask();
283 tlb_state[cpu].entry[i].entrylo0 = mips_rd_entrylo0();
284 tlb_state[cpu].entry[i].entrylo1 = mips_rd_entrylo1();
289 tlb_update(struct pmap *pmap, vm_offset_t va, pt_entry_t pte)
296 pte &= ~TLBLO_SWBITS_MASK;
299 asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
302 mips_wr_entryhi(TLBHI_ENTRY(va, pmap_asid(pmap)));
308 if ((va & PAGE_SIZE) == 0) {
309 mips_wr_entrylo0(pte);
311 mips_wr_entrylo1(pte);
316 mips_wr_entryhi(asid);
321 tlb_invalidate_one(unsigned i)
323 /* XXX an invalid ASID? */
324 mips_wr_entryhi(TLBHI_ENTRY(MIPS_KSEG0_START + (2 * i * PAGE_SIZE), 0));
335 DB_SHOW_COMMAND(tlb, ddb_dump_tlb)
337 register_t ehi, elo0, elo1, epagemask;
338 unsigned i, cpu, ntlb;
342 * The worst conversion from hex to decimal ever.
345 cpu = ((addr >> 4) % 16) * 10 + (addr % 16);
347 cpu = PCPU_GET(cpuid);
349 if (cpu < 0 || cpu >= mp_ncpus) {
350 db_printf("Invalid CPU %u\n", cpu);
353 if (num_tlbentries > MIPS_MAX_TLB_ENTRIES) {
354 ntlb = MIPS_MAX_TLB_ENTRIES;
355 db_printf("Warning: Only %d of %d TLB entries saved!\n",
356 ntlb, num_tlbentries);
358 ntlb = num_tlbentries;
360 if (cpu == PCPU_GET(cpuid))
363 db_printf("Beginning TLB dump for CPU %u...\n", cpu);
364 for (i = 0; i < ntlb; i++) {
365 if (i == tlb_state[cpu].wired) {
367 db_printf("^^^ WIRED ENTRIES ^^^\n");
369 db_printf("(No wired entries.)\n");
373 ehi = tlb_state[cpu].entry[i].entryhi;
374 elo0 = tlb_state[cpu].entry[i].entrylo0;
375 elo1 = tlb_state[cpu].entry[i].entrylo1;
376 epagemask = tlb_state[cpu].entry[i].pagemask;
378 if (elo0 == 0 && elo1 == 0)
381 db_printf("#%u\t=> %jx (pagemask %jx)\n", i, (intmax_t)ehi, (intmax_t) epagemask);
382 db_printf(" Lo0\t%jx\t(%#jx)\n", (intmax_t)elo0, (intmax_t)TLBLO_PTE_TO_PA(elo0));
383 db_printf(" Lo1\t%jx\t(%#jx)\n", (intmax_t)elo1, (intmax_t)TLBLO_PTE_TO_PA(elo1));
385 db_printf("Finished.\n");