1 /* $NetBSD: cache_mipsNN.c,v 1.10 2005/12/24 20:07:19 perry Exp $ */
4 * SPDX-License-Identifier: BSD-4-Clause
6 * Copyright 2001 Wasabi Systems, Inc.
9 * Written by Jason R. Thorpe and Simon Burge for Wasabi Systems, Inc.
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 * 3. All advertising materials mentioning features or use of this software
20 * must display the following acknowledgement:
21 * This product includes software developed for the NetBSD Project by
22 * Wasabi Systems, Inc.
23 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
24 * or promote products derived from this software without specific prior
27 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
28 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
29 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
30 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
31 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
32 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
33 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
34 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
35 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
36 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
37 * POSSIBILITY OF SUCH DAMAGE.
40 #include <sys/cdefs.h>
41 __FBSDID("$FreeBSD$");
43 #include <sys/types.h>
44 #include <sys/systm.h>
45 #include <sys/param.h>
47 #include <machine/cache.h>
48 #include <machine/cache_r4k.h>
49 #include <machine/cpuinfo.h>
51 #define round_line16(x) (((x) + 15) & ~15)
52 #define trunc_line16(x) ((x) & ~15)
54 #define round_line32(x) (((x) + 31) & ~31)
55 #define trunc_line32(x) ((x) & ~31)
57 #define round_line64(x) (((x) + 63) & ~63)
58 #define trunc_line64(x) ((x) & ~63)
60 #define round_line128(x) (((x) + 127) & ~127)
61 #define trunc_line128(x) ((x) & ~127)
81 #if defined(SB1250_PASS1)
82 #define SYNC __asm volatile("sync; sync")
83 #elif defined(CPU_NLM)
84 #define SYNC xlp_sync()
86 #define SYNC __asm volatile("sync")
89 #if defined(CPU_CNMIPS)
90 #define SYNCI mips_sync_icache();
91 #elif defined(CPU_NLM)
92 #define SYNCI xlp_sync()
98 * Exported variables for consumers like bus_dma code
100 int mips_picache_linesize;
101 int mips_pdcache_linesize;
102 int mips_sdcache_linesize;
103 int mips_dcache_max_linesize;
105 static int picache_size;
106 static int picache_stride;
107 static int picache_loopcount;
108 static int picache_way_mask;
109 static int pdcache_size;
110 static int pdcache_stride;
111 static int pdcache_loopcount;
112 static int pdcache_way_mask;
113 static int sdcache_size;
114 static int sdcache_stride;
115 static int sdcache_loopcount;
116 static int sdcache_way_mask;
119 mipsNN_cache_init(struct mips_cpuinfo * cpuinfo)
121 int flush_multiple_lines_per_way;
123 flush_multiple_lines_per_way = cpuinfo->l1.ic_nsets * cpuinfo->l1.ic_linesize * cpuinfo->l1.ic_linesize > PAGE_SIZE;
124 if (cpuinfo->icache_virtual) {
126 * With a virtual Icache we don't need to flush
127 * multiples of the page size with index ops; we just
128 * need to flush one pages' worth.
130 flush_multiple_lines_per_way = 0;
133 if (flush_multiple_lines_per_way) {
134 picache_stride = PAGE_SIZE;
135 picache_loopcount = (cpuinfo->l1.ic_nsets * cpuinfo->l1.ic_linesize / PAGE_SIZE) *
136 cpuinfo->l1.ic_nways;
138 picache_stride = cpuinfo->l1.ic_nsets * cpuinfo->l1.ic_linesize;
139 picache_loopcount = cpuinfo->l1.ic_nways;
142 if (cpuinfo->l1.dc_nsets * cpuinfo->l1.dc_linesize < PAGE_SIZE) {
143 pdcache_stride = cpuinfo->l1.dc_nsets * cpuinfo->l1.dc_linesize;
144 pdcache_loopcount = cpuinfo->l1.dc_nways;
146 pdcache_stride = PAGE_SIZE;
147 pdcache_loopcount = (cpuinfo->l1.dc_nsets * cpuinfo->l1.dc_linesize / PAGE_SIZE) *
148 cpuinfo->l1.dc_nways;
151 mips_picache_linesize = cpuinfo->l1.ic_linesize;
152 mips_pdcache_linesize = cpuinfo->l1.dc_linesize;
154 picache_size = cpuinfo->l1.ic_size;
155 picache_way_mask = cpuinfo->l1.ic_nways - 1;
156 pdcache_size = cpuinfo->l1.dc_size;
157 pdcache_way_mask = cpuinfo->l1.dc_nways - 1;
159 sdcache_stride = cpuinfo->l2.dc_nsets * cpuinfo->l2.dc_linesize;
160 sdcache_loopcount = cpuinfo->l2.dc_nways;
161 sdcache_size = cpuinfo->l2.dc_size;
162 sdcache_way_mask = cpuinfo->l2.dc_nways - 1;
164 mips_sdcache_linesize = cpuinfo->l2.dc_linesize;
165 mips_dcache_max_linesize = MAX(mips_pdcache_linesize,
166 mips_sdcache_linesize);
170 printf("Cache info:\n");
171 if (cpuinfo->icache_virtual)
172 printf(" icache is virtual\n");
173 printf(" picache_stride = %d\n", picache_stride);
174 printf(" picache_loopcount = %d\n", picache_loopcount);
175 printf(" pdcache_stride = %d\n", pdcache_stride);
176 printf(" pdcache_loopcount = %d\n", pdcache_loopcount);
177 printf(" max line size = %d\n", mips_dcache_max_linesize);
182 mipsNN_icache_sync_all_16(void)
186 va = MIPS_PHYS_TO_KSEG0(0);
187 eva = va + picache_size;
190 * Since we're hitting the whole thing, we don't have to
191 * worry about the N different "ways".
194 mips_intern_dcache_wbinv_all();
197 cache_r4k_op_32lines_16(va, CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
205 mipsNN_icache_sync_all_32(void)
209 va = MIPS_PHYS_TO_KSEG0(0);
210 eva = va + picache_size;
213 * Since we're hitting the whole thing, we don't have to
214 * worry about the N different "ways".
217 mips_intern_dcache_wbinv_all();
220 cache_r4k_op_32lines_32(va, CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
228 mipsNN_icache_sync_all_64(void)
232 va = MIPS_PHYS_TO_KSEG0(0);
233 eva = va + picache_size;
236 * Since we're hitting the whole thing, we don't have to
237 * worry about the N different "ways".
240 mips_intern_dcache_wbinv_all();
243 cache_r4k_op_32lines_64(va, CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
251 mipsNN_icache_sync_range_16(vm_offset_t va, vm_size_t size)
255 eva = round_line16(va + size);
256 va = trunc_line16(va);
258 mips_intern_dcache_wb_range(va, (eva - va));
260 while ((eva - va) >= (32 * 16)) {
261 cache_r4k_op_32lines_16(va, CACHE_R4K_I|CACHEOP_R4K_HIT_INV);
266 cache_op_r4k_line(va, CACHE_R4K_I|CACHEOP_R4K_HIT_INV);
274 mipsNN_icache_sync_range_32(vm_offset_t va, vm_size_t size)
278 eva = round_line32(va + size);
279 va = trunc_line32(va);
281 mips_intern_dcache_wb_range(va, (eva - va));
283 while ((eva - va) >= (32 * 32)) {
284 cache_r4k_op_32lines_32(va, CACHE_R4K_I|CACHEOP_R4K_HIT_INV);
289 cache_op_r4k_line(va, CACHE_R4K_I|CACHEOP_R4K_HIT_INV);
297 mipsNN_icache_sync_range_64(vm_offset_t va, vm_size_t size)
301 eva = round_line64(va + size);
302 va = trunc_line64(va);
304 mips_intern_dcache_wb_range(va, (eva - va));
306 while ((eva - va) >= (32 * 64)) {
307 cache_r4k_op_32lines_64(va, CACHE_R4K_I|CACHEOP_R4K_HIT_INV);
312 cache_op_r4k_line(va, CACHE_R4K_I|CACHEOP_R4K_HIT_INV);
320 mipsNN_icache_sync_range_index_16(vm_offset_t va, vm_size_t size)
322 vm_offset_t eva, tmpva;
323 int i, stride, loopcount;
326 * Since we're doing Index ops, we expect to not be able
327 * to access the address we've been given. So, get the
328 * bits that determine the cache index, and make a KSEG0
329 * address out of them.
331 va = MIPS_PHYS_TO_KSEG0(va & picache_way_mask);
333 eva = round_line16(va + size);
334 va = trunc_line16(va);
337 * GCC generates better code in the loops if we reference local
338 * copies of these global variables.
340 stride = picache_stride;
341 loopcount = picache_loopcount;
343 mips_intern_dcache_wbinv_range_index(va, (eva - va));
345 while ((eva - va) >= (8 * 16)) {
347 for (i = 0; i < loopcount; i++, tmpva += stride)
348 cache_r4k_op_8lines_16(tmpva,
349 CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
355 for (i = 0; i < loopcount; i++, tmpva += stride)
356 cache_op_r4k_line(tmpva,
357 CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
363 mipsNN_icache_sync_range_index_32(vm_offset_t va, vm_size_t size)
365 vm_offset_t eva, tmpva;
366 int i, stride, loopcount;
369 * Since we're doing Index ops, we expect to not be able
370 * to access the address we've been given. So, get the
371 * bits that determine the cache index, and make a KSEG0
372 * address out of them.
374 va = MIPS_PHYS_TO_KSEG0(va & picache_way_mask);
376 eva = round_line32(va + size);
377 va = trunc_line32(va);
380 * GCC generates better code in the loops if we reference local
381 * copies of these global variables.
383 stride = picache_stride;
384 loopcount = picache_loopcount;
386 mips_intern_dcache_wbinv_range_index(va, (eva - va));
388 while ((eva - va) >= (8 * 32)) {
390 for (i = 0; i < loopcount; i++, tmpva += stride)
391 cache_r4k_op_8lines_32(tmpva,
392 CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
398 for (i = 0; i < loopcount; i++, tmpva += stride)
399 cache_op_r4k_line(tmpva,
400 CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
406 mipsNN_icache_sync_range_index_64(vm_offset_t va, vm_size_t size)
408 vm_offset_t eva, tmpva;
409 int i, stride, loopcount;
412 * Since we're doing Index ops, we expect to not be able
413 * to access the address we've been given. So, get the
414 * bits that determine the cache index, and make a KSEG0
415 * address out of them.
417 va = MIPS_PHYS_TO_KSEG0(va & picache_way_mask);
419 eva = round_line64(va + size);
420 va = trunc_line64(va);
423 * GCC generates better code in the loops if we reference local
424 * copies of these global variables.
426 stride = picache_stride;
427 loopcount = picache_loopcount;
429 mips_intern_dcache_wbinv_range_index(va, (eva - va));
431 while ((eva - va) >= (8 * 64)) {
433 for (i = 0; i < loopcount; i++, tmpva += stride)
434 cache_r4k_op_8lines_64(tmpva,
435 CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
441 for (i = 0; i < loopcount; i++, tmpva += stride)
442 cache_op_r4k_line(tmpva,
443 CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
449 mipsNN_pdcache_wbinv_all_16(void)
453 va = MIPS_PHYS_TO_KSEG0(0);
454 eva = va + pdcache_size;
457 * Since we're hitting the whole thing, we don't have to
458 * worry about the N different "ways".
462 cache_r4k_op_32lines_16(va,
463 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
471 mipsNN_pdcache_wbinv_all_32(void)
475 va = MIPS_PHYS_TO_KSEG0(0);
476 eva = va + pdcache_size;
479 * Since we're hitting the whole thing, we don't have to
480 * worry about the N different "ways".
484 cache_r4k_op_32lines_32(va,
485 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
493 mipsNN_pdcache_wbinv_all_64(void)
497 va = MIPS_PHYS_TO_KSEG0(0);
498 eva = va + pdcache_size;
501 * Since we're hitting the whole thing, we don't have to
502 * worry about the N different "ways".
506 cache_r4k_op_32lines_64(va,
507 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
515 mipsNN_pdcache_wbinv_range_16(vm_offset_t va, vm_size_t size)
519 eva = round_line16(va + size);
520 va = trunc_line16(va);
522 while ((eva - va) >= (32 * 16)) {
523 cache_r4k_op_32lines_16(va,
524 CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
529 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
537 mipsNN_pdcache_wbinv_range_32(vm_offset_t va, vm_size_t size)
541 eva = round_line32(va + size);
542 va = trunc_line32(va);
544 while ((eva - va) >= (32 * 32)) {
545 cache_r4k_op_32lines_32(va,
546 CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
551 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
559 mipsNN_pdcache_wbinv_range_64(vm_offset_t va, vm_size_t size)
563 eva = round_line64(va + size);
564 va = trunc_line64(va);
566 while ((eva - va) >= (32 * 64)) {
567 cache_r4k_op_32lines_64(va,
568 CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
573 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
581 mipsNN_pdcache_wbinv_range_index_16(vm_offset_t va, vm_size_t size)
583 vm_offset_t eva, tmpva;
584 int i, stride, loopcount;
587 * Since we're doing Index ops, we expect to not be able
588 * to access the address we've been given. So, get the
589 * bits that determine the cache index, and make a KSEG0
590 * address out of them.
592 va = MIPS_PHYS_TO_KSEG0(va & pdcache_way_mask);
594 eva = round_line16(va + size);
595 va = trunc_line16(va);
598 * GCC generates better code in the loops if we reference local
599 * copies of these global variables.
601 stride = pdcache_stride;
602 loopcount = pdcache_loopcount;
604 while ((eva - va) >= (8 * 16)) {
606 for (i = 0; i < loopcount; i++, tmpva += stride)
607 cache_r4k_op_8lines_16(tmpva,
608 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
614 for (i = 0; i < loopcount; i++, tmpva += stride)
615 cache_op_r4k_line(tmpva,
616 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
622 mipsNN_pdcache_wbinv_range_index_32(vm_offset_t va, vm_size_t size)
624 vm_offset_t eva, tmpva;
625 int i, stride, loopcount;
628 * Since we're doing Index ops, we expect to not be able
629 * to access the address we've been given. So, get the
630 * bits that determine the cache index, and make a KSEG0
631 * address out of them.
633 va = MIPS_PHYS_TO_KSEG0(va & pdcache_way_mask);
635 eva = round_line32(va + size);
636 va = trunc_line32(va);
639 * GCC generates better code in the loops if we reference local
640 * copies of these global variables.
642 stride = pdcache_stride;
643 loopcount = pdcache_loopcount;
645 while ((eva - va) >= (8 * 32)) {
647 for (i = 0; i < loopcount; i++, tmpva += stride)
648 cache_r4k_op_8lines_32(tmpva,
649 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
655 for (i = 0; i < loopcount; i++, tmpva += stride)
656 cache_op_r4k_line(tmpva,
657 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
663 mipsNN_pdcache_wbinv_range_index_64(vm_offset_t va, vm_size_t size)
665 vm_offset_t eva, tmpva;
666 int i, stride, loopcount;
669 * Since we're doing Index ops, we expect to not be able
670 * to access the address we've been given. So, get the
671 * bits that determine the cache index, and make a KSEG0
672 * address out of them.
674 va = MIPS_PHYS_TO_KSEG0(va & pdcache_way_mask);
676 eva = round_line64(va + size);
677 va = trunc_line64(va);
680 * GCC generates better code in the loops if we reference local
681 * copies of these global variables.
683 stride = pdcache_stride;
684 loopcount = pdcache_loopcount;
686 while ((eva - va) >= (8 * 64)) {
688 for (i = 0; i < loopcount; i++, tmpva += stride)
689 cache_r4k_op_8lines_64(tmpva,
690 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
696 for (i = 0; i < loopcount; i++, tmpva += stride)
697 cache_op_r4k_line(tmpva,
698 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
704 mipsNN_pdcache_inv_range_16(vm_offset_t va, vm_size_t size)
708 eva = round_line16(va + size);
709 va = trunc_line16(va);
711 while ((eva - va) >= (32 * 16)) {
712 cache_r4k_op_32lines_16(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
717 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
725 mipsNN_pdcache_inv_range_32(vm_offset_t va, vm_size_t size)
729 eva = round_line32(va + size);
730 va = trunc_line32(va);
732 while ((eva - va) >= (32 * 32)) {
733 cache_r4k_op_32lines_32(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
738 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
746 mipsNN_pdcache_inv_range_64(vm_offset_t va, vm_size_t size)
750 eva = round_line64(va + size);
751 va = trunc_line64(va);
753 while ((eva - va) >= (32 * 64)) {
754 cache_r4k_op_32lines_64(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
759 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
767 mipsNN_pdcache_wb_range_16(vm_offset_t va, vm_size_t size)
771 eva = round_line16(va + size);
772 va = trunc_line16(va);
774 while ((eva - va) >= (32 * 16)) {
775 cache_r4k_op_32lines_16(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
780 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
788 mipsNN_pdcache_wb_range_32(vm_offset_t va, vm_size_t size)
792 eva = round_line32(va + size);
793 va = trunc_line32(va);
795 while ((eva - va) >= (32 * 32)) {
796 cache_r4k_op_32lines_32(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
801 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
809 mipsNN_pdcache_wb_range_64(vm_offset_t va, vm_size_t size)
813 eva = round_line64(va + size);
814 va = trunc_line64(va);
816 while ((eva - va) >= (32 * 64)) {
817 cache_r4k_op_32lines_64(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
822 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
832 mipsNN_icache_sync_all_128(void)
838 mipsNN_icache_sync_range_128(vm_offset_t va, vm_size_t size)
844 mipsNN_icache_sync_range_index_128(vm_offset_t va, vm_size_t size)
850 mipsNN_pdcache_wbinv_all_128(void)
856 mipsNN_pdcache_wbinv_range_128(vm_offset_t va, vm_size_t size)
862 mipsNN_pdcache_wbinv_range_index_128(vm_offset_t va, vm_size_t size)
867 mipsNN_pdcache_inv_range_128(vm_offset_t va, vm_size_t size)
872 mipsNN_pdcache_wb_range_128(vm_offset_t va, vm_size_t size)
880 mipsNN_icache_sync_all_128(void)
884 va = MIPS_PHYS_TO_KSEG0(0);
885 eva = va + picache_size;
888 * Since we're hitting the whole thing, we don't have to
889 * worry about the N different "ways".
892 mips_intern_dcache_wbinv_all();
895 cache_r4k_op_32lines_128(va, CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
903 mipsNN_icache_sync_range_128(vm_offset_t va, vm_size_t size)
907 eva = round_line128(va + size);
908 va = trunc_line128(va);
910 mips_intern_dcache_wb_range(va, (eva - va));
912 while ((eva - va) >= (32 * 128)) {
913 cache_r4k_op_32lines_128(va, CACHE_R4K_I|CACHEOP_R4K_HIT_INV);
918 cache_op_r4k_line(va, CACHE_R4K_I|CACHEOP_R4K_HIT_INV);
926 mipsNN_icache_sync_range_index_128(vm_offset_t va, vm_size_t size)
928 vm_offset_t eva, tmpva;
929 int i, stride, loopcount;
932 * Since we're doing Index ops, we expect to not be able
933 * to access the address we've been given. So, get the
934 * bits that determine the cache index, and make a KSEG0
935 * address out of them.
937 va = MIPS_PHYS_TO_KSEG0(va & picache_way_mask);
939 eva = round_line128(va + size);
940 va = trunc_line128(va);
943 * GCC generates better code in the loops if we reference local
944 * copies of these global variables.
946 stride = picache_stride;
947 loopcount = picache_loopcount;
949 mips_intern_dcache_wbinv_range_index(va, (eva - va));
951 while ((eva - va) >= (32 * 128)) {
953 for (i = 0; i < loopcount; i++, tmpva += stride)
954 cache_r4k_op_32lines_128(tmpva,
955 CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
961 for (i = 0; i < loopcount; i++, tmpva += stride)
962 cache_op_r4k_line(tmpva,
963 CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
969 mipsNN_pdcache_wbinv_all_128(void)
973 va = MIPS_PHYS_TO_KSEG0(0);
974 eva = va + pdcache_size;
977 * Since we're hitting the whole thing, we don't have to
978 * worry about the N different "ways".
982 cache_r4k_op_32lines_128(va,
983 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
992 mipsNN_pdcache_wbinv_range_128(vm_offset_t va, vm_size_t size)
996 eva = round_line128(va + size);
997 va = trunc_line128(va);
999 while ((eva - va) >= (32 * 128)) {
1000 cache_r4k_op_32lines_128(va,
1001 CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
1006 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
1014 mipsNN_pdcache_wbinv_range_index_128(vm_offset_t va, vm_size_t size)
1016 vm_offset_t eva, tmpva;
1017 int i, stride, loopcount;
1020 * Since we're doing Index ops, we expect to not be able
1021 * to access the address we've been given. So, get the
1022 * bits that determine the cache index, and make a KSEG0
1023 * address out of them.
1025 va = MIPS_PHYS_TO_KSEG0(va & pdcache_way_mask);
1027 eva = round_line128(va + size);
1028 va = trunc_line128(va);
1031 * GCC generates better code in the loops if we reference local
1032 * copies of these global variables.
1034 stride = pdcache_stride;
1035 loopcount = pdcache_loopcount;
1037 while ((eva - va) >= (32 * 128)) {
1039 for (i = 0; i < loopcount; i++, tmpva += stride)
1040 cache_r4k_op_32lines_128(tmpva,
1041 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
1047 for (i = 0; i < loopcount; i++, tmpva += stride)
1048 cache_op_r4k_line(tmpva,
1049 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
1055 mipsNN_pdcache_inv_range_128(vm_offset_t va, vm_size_t size)
1059 eva = round_line128(va + size);
1060 va = trunc_line128(va);
1062 while ((eva - va) >= (32 * 128)) {
1063 cache_r4k_op_32lines_128(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
1068 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
1076 mipsNN_pdcache_wb_range_128(vm_offset_t va, vm_size_t size)
1080 eva = round_line128(va + size);
1081 va = trunc_line128(va);
1083 while ((eva - va) >= (32 * 128)) {
1084 cache_r4k_op_32lines_128(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
1089 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
1099 mipsNN_sdcache_wbinv_all_32(void)
1101 vm_offset_t va = MIPS_PHYS_TO_KSEG0(0);
1102 vm_offset_t eva = va + sdcache_size;
1105 cache_r4k_op_32lines_32(va,
1106 CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1112 mipsNN_sdcache_wbinv_all_64(void)
1114 vm_offset_t va = MIPS_PHYS_TO_KSEG0(0);
1115 vm_offset_t eva = va + sdcache_size;
1118 cache_r4k_op_32lines_64(va,
1119 CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1125 mipsNN_sdcache_wbinv_range_32(vm_offset_t va, vm_size_t size)
1127 vm_offset_t eva = round_line32(va + size);
1129 va = trunc_line32(va);
1131 while ((eva - va) >= (32 * 32)) {
1132 cache_r4k_op_32lines_32(va,
1133 CACHE_R4K_SD|CACHEOP_R4K_HIT_WB_INV);
1138 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB_INV);
1144 mipsNN_sdcache_wbinv_range_64(vm_offset_t va, vm_size_t size)
1146 vm_offset_t eva = round_line64(va + size);
1148 va = trunc_line64(va);
1150 while ((eva - va) >= (32 * 64)) {
1151 cache_r4k_op_32lines_64(va,
1152 CACHE_R4K_SD|CACHEOP_R4K_HIT_WB_INV);
1157 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB_INV);
1163 mipsNN_sdcache_wbinv_range_index_32(vm_offset_t va, vm_size_t size)
1168 * Since we're doing Index ops, we expect to not be able
1169 * to access the address we've been given. So, get the
1170 * bits that determine the cache index, and make a KSEG0
1171 * address out of them.
1173 va = MIPS_PHYS_TO_KSEG0(va & (sdcache_size - 1));
1175 eva = round_line32(va + size);
1176 va = trunc_line32(va);
1178 while ((eva - va) >= (32 * 32)) {
1179 cache_r4k_op_32lines_32(va,
1180 CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1185 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1191 mipsNN_sdcache_wbinv_range_index_64(vm_offset_t va, vm_size_t size)
1196 * Since we're doing Index ops, we expect to not be able
1197 * to access the address we've been given. So, get the
1198 * bits that determine the cache index, and make a KSEG0
1199 * address out of them.
1201 va = MIPS_PHYS_TO_KSEG0(va & (sdcache_size - 1));
1203 eva = round_line64(va + size);
1204 va = trunc_line64(va);
1206 while ((eva - va) >= (32 * 64)) {
1207 cache_r4k_op_32lines_64(va,
1208 CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1213 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1219 mipsNN_sdcache_inv_range_32(vm_offset_t va, vm_size_t size)
1221 vm_offset_t eva = round_line32(va + size);
1223 va = trunc_line32(va);
1225 while ((eva - va) >= (32 * 32)) {
1226 cache_r4k_op_32lines_32(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_INV);
1231 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_INV);
1237 mipsNN_sdcache_inv_range_64(vm_offset_t va, vm_size_t size)
1239 vm_offset_t eva = round_line64(va + size);
1241 va = trunc_line64(va);
1243 while ((eva - va) >= (32 * 64)) {
1244 cache_r4k_op_32lines_64(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_INV);
1249 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_INV);
1255 mipsNN_sdcache_wb_range_32(vm_offset_t va, vm_size_t size)
1257 vm_offset_t eva = round_line32(va + size);
1259 va = trunc_line32(va);
1261 while ((eva - va) >= (32 * 32)) {
1262 cache_r4k_op_32lines_32(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB);
1267 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB);
1273 mipsNN_sdcache_wb_range_64(vm_offset_t va, vm_size_t size)
1275 vm_offset_t eva = round_line64(va + size);
1277 va = trunc_line64(va);
1279 while ((eva - va) >= (32 * 64)) {
1280 cache_r4k_op_32lines_64(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB);
1285 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB);
1291 mipsNN_sdcache_wbinv_all_128(void)
1293 vm_offset_t va = MIPS_PHYS_TO_KSEG0(0);
1294 vm_offset_t eva = va + sdcache_size;
1297 cache_r4k_op_32lines_128(va,
1298 CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1304 mipsNN_sdcache_wbinv_range_128(vm_offset_t va, vm_size_t size)
1306 vm_offset_t eva = round_line128(va + size);
1308 va = trunc_line128(va);
1310 while ((eva - va) >= (32 * 128)) {
1311 cache_r4k_op_32lines_128(va,
1312 CACHE_R4K_SD|CACHEOP_R4K_HIT_WB_INV);
1317 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB_INV);
1323 mipsNN_sdcache_wbinv_range_index_128(vm_offset_t va, vm_size_t size)
1328 * Since we're doing Index ops, we expect to not be able
1329 * to access the address we've been given. So, get the
1330 * bits that determine the cache index, and make a KSEG0
1331 * address out of them.
1333 va = MIPS_PHYS_TO_KSEG0(va & (sdcache_size - 1));
1335 eva = round_line128(va + size);
1336 va = trunc_line128(va);
1338 while ((eva - va) >= (32 * 128)) {
1339 cache_r4k_op_32lines_128(va,
1340 CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1345 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1351 mipsNN_sdcache_inv_range_128(vm_offset_t va, vm_size_t size)
1353 vm_offset_t eva = round_line128(va + size);
1355 va = trunc_line128(va);
1357 while ((eva - va) >= (32 * 128)) {
1358 cache_r4k_op_32lines_128(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_INV);
1363 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_INV);
1369 mipsNN_sdcache_wb_range_128(vm_offset_t va, vm_size_t size)
1371 vm_offset_t eva = round_line128(va + size);
1373 va = trunc_line128(va);
1375 while ((eva - va) >= (32 * 128)) {
1376 cache_r4k_op_32lines_128(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB);
1381 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB);