2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2004-2010 Juli Mallett <jmallett@FreeBSD.org>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33 #include <sys/param.h>
34 #include <sys/kernel.h>
35 #include <sys/systm.h>
42 #include <machine/pte.h>
43 #include <machine/tlb.h>
45 #if defined(CPU_CNMIPS)
46 #define MIPS_MAX_TLB_ENTRIES 128
47 #elif defined(CPU_NLM)
48 #define MIPS_MAX_TLB_ENTRIES (2048 + 128)
50 #define MIPS_MAX_TLB_ENTRIES 64
60 } entry[MIPS_MAX_TLB_ENTRIES];
63 static struct tlb_state tlb_state[MAXCPU];
67 * PageMask must increment in steps of 2 bits.
69 COMPILE_TIME_ASSERT(POPCNT(TLBMASK_MASK) % 2 == 0);
75 __asm __volatile ("tlbp" : : : "memory");
82 __asm __volatile ("tlbr" : : : "memory");
87 tlb_write_indexed(void)
89 __asm __volatile ("tlbwi" : : : "memory");
93 static void tlb_invalidate_one(unsigned);
96 tlb_insert_wired(unsigned i, vm_offset_t va, pt_entry_t pte0, pt_entry_t pte1)
104 asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
108 mips_wr_entryhi(TLBHI_ENTRY(va, 0));
109 mips_wr_entrylo0(pte0);
110 mips_wr_entrylo1(pte1);
113 mips_wr_entryhi(asid);
118 tlb_invalidate_address(struct pmap *pmap, vm_offset_t va)
127 asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
130 mips_wr_entryhi(TLBHI_ENTRY(va, pmap_asid(pmap)));
134 tlb_invalidate_one(i);
136 mips_wr_entryhi(asid);
141 tlb_invalidate_all(void)
148 asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
150 for (i = mips_rd_wired(); i < num_tlbentries; i++)
151 tlb_invalidate_one(i);
153 mips_wr_entryhi(asid);
158 tlb_invalidate_all_user(struct pmap *pmap)
165 asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
167 for (i = mips_rd_wired(); i < num_tlbentries; i++) {
173 uasid = mips_rd_entryhi() & TLBHI_ASID_MASK;
176 * Invalidate all non-kernel entries.
182 * Invalidate this pmap's entries.
184 if (uasid != pmap_asid(pmap))
187 tlb_invalidate_one(i);
190 mips_wr_entryhi(asid);
195 * Invalidates any TLB entries that map a virtual page from the specified
196 * address range. If "end" is zero, then every virtual page is considered to
197 * be within the address range's upper bound.
200 tlb_invalidate_range(pmap_t pmap, vm_offset_t start, vm_offset_t end)
202 register_t asid, end_hi, hi, hi_pagemask, s, save_asid, start_hi;
205 KASSERT(start < end || (end == 0 && start > 0),
206 ("tlb_invalidate_range: invalid range"));
209 * Truncate the virtual address "start" to an even page frame number,
210 * and round the virtual address "end" to an even page frame number.
212 start &= ~((1 << TLBMASK_SHIFT) - 1);
213 end = roundup2(end, 1 << TLBMASK_SHIFT);
216 save_asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
218 asid = pmap_asid(pmap);
219 start_hi = TLBHI_ENTRY(start, asid);
220 end_hi = TLBHI_ENTRY(end, asid);
223 * Select the fastest method for invalidating the TLB entries.
225 if (end - start < num_tlbentries << TLBMASK_SHIFT || (end == 0 &&
226 start >= -(num_tlbentries << TLBMASK_SHIFT))) {
228 * The virtual address range is small compared to the size of
229 * the TLB. Probe the TLB for each even numbered page frame
230 * within the virtual address range.
232 for (hi = start_hi; hi != end_hi; hi += 1 << TLBMASK_SHIFT) {
238 tlb_invalidate_one(i);
242 * The virtual address range is large compared to the size of
243 * the TLB. Test every non-wired TLB entry.
245 for (i = mips_rd_wired(); i < num_tlbentries; i++) {
248 hi = mips_rd_entryhi();
249 if ((hi & TLBHI_ASID_MASK) == asid && (hi < end_hi ||
252 * If "hi" is a large page that spans
253 * "start_hi", then it must be invalidated.
255 hi_pagemask = mips_rd_pagemask();
256 if (hi >= (start_hi & ~(hi_pagemask <<
258 tlb_invalidate_one(i);
263 mips_wr_entryhi(save_asid);
267 /* XXX Only if DDB? */
271 unsigned ntlb, i, cpu;
273 cpu = PCPU_GET(cpuid);
274 if (num_tlbentries > MIPS_MAX_TLB_ENTRIES)
275 ntlb = MIPS_MAX_TLB_ENTRIES;
277 ntlb = num_tlbentries;
278 tlb_state[cpu].wired = mips_rd_wired();
279 for (i = 0; i < ntlb; i++) {
283 tlb_state[cpu].entry[i].entryhi = mips_rd_entryhi();
284 tlb_state[cpu].entry[i].pagemask = mips_rd_pagemask();
285 tlb_state[cpu].entry[i].entrylo0 = mips_rd_entrylo0();
286 tlb_state[cpu].entry[i].entrylo1 = mips_rd_entrylo1();
291 tlb_update(struct pmap *pmap, vm_offset_t va, pt_entry_t pte)
298 pte &= ~TLBLO_SWBITS_MASK;
301 asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
304 mips_wr_entryhi(TLBHI_ENTRY(va, pmap_asid(pmap)));
310 if ((va & PAGE_SIZE) == 0) {
311 mips_wr_entrylo0(pte);
313 mips_wr_entrylo1(pte);
318 mips_wr_entryhi(asid);
323 tlb_invalidate_one(unsigned i)
325 /* XXX an invalid ASID? */
326 mips_wr_entryhi(TLBHI_ENTRY(MIPS_KSEG0_START + (2 * i * PAGE_SIZE), 0));
337 DB_SHOW_COMMAND(tlb, ddb_dump_tlb)
339 register_t ehi, elo0, elo1, epagemask;
340 unsigned i, cpu, ntlb;
344 * The worst conversion from hex to decimal ever.
347 cpu = ((addr >> 4) % 16) * 10 + (addr % 16);
349 cpu = PCPU_GET(cpuid);
351 if (cpu >= mp_ncpus) {
352 db_printf("Invalid CPU %u\n", cpu);
355 if (num_tlbentries > MIPS_MAX_TLB_ENTRIES) {
356 ntlb = MIPS_MAX_TLB_ENTRIES;
357 db_printf("Warning: Only %d of %d TLB entries saved!\n",
358 ntlb, num_tlbentries);
360 ntlb = num_tlbentries;
362 if (cpu == PCPU_GET(cpuid))
365 db_printf("Beginning TLB dump for CPU %u...\n", cpu);
366 for (i = 0; i < ntlb; i++) {
367 if (i == tlb_state[cpu].wired) {
369 db_printf("^^^ WIRED ENTRIES ^^^\n");
371 db_printf("(No wired entries.)\n");
375 ehi = tlb_state[cpu].entry[i].entryhi;
376 elo0 = tlb_state[cpu].entry[i].entrylo0;
377 elo1 = tlb_state[cpu].entry[i].entrylo1;
378 epagemask = tlb_state[cpu].entry[i].pagemask;
380 if (elo0 == 0 && elo1 == 0)
383 db_printf("#%u\t=> %jx (pagemask %jx)\n", i, (intmax_t)ehi, (intmax_t) epagemask);
384 db_printf(" Lo0\t%jx\t(%#jx)\n", (intmax_t)elo0, (intmax_t)TLBLO_PTE_TO_PA(elo0));
385 db_printf(" Lo1\t%jx\t(%#jx)\n", (intmax_t)elo1, (intmax_t)TLBLO_PTE_TO_PA(elo1));
387 db_printf("Finished.\n");