1 /* $NetBSD: cache_mipsNN.c,v 1.10 2005/12/24 20:07:19 perry Exp $ */
4 * Copyright 2001 Wasabi Systems, Inc.
7 * Written by Jason R. Thorpe and Simon Burge for Wasabi Systems, Inc.
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 3. All advertising materials mentioning features or use of this software
18 * must display the following acknowledgement:
19 * This product includes software developed for the NetBSD Project by
20 * Wasabi Systems, Inc.
21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
22 * or promote products derived from this software without specific prior
25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
35 * POSSIBILITY OF SUCH DAMAGE.
38 #include <sys/cdefs.h>
39 __FBSDID("$FreeBSD$");
41 #include <sys/types.h>
42 #include <sys/systm.h>
43 #include <sys/param.h>
45 #include <machine/cache.h>
46 #include <machine/cache_r4k.h>
47 #include <machine/cpuinfo.h>
49 #define round_line16(x) (((x) + 15) & ~15)
50 #define trunc_line16(x) ((x) & ~15)
52 #define round_line32(x) (((x) + 31) & ~31)
53 #define trunc_line32(x) ((x) & ~31)
55 #define round_line64(x) (((x) + 63) & ~63)
56 #define trunc_line64(x) ((x) & ~63)
58 #define round_line128(x) (((x) + 127) & ~127)
59 #define trunc_line128(x) ((x) & ~127)
79 #if defined(SB1250_PASS1)
80 #define SYNC __asm volatile("sync; sync")
81 #elif defined(CPU_NLM)
82 #define SYNC xlp_sync()
84 #define SYNC __asm volatile("sync")
87 #if defined(CPU_CNMIPS)
88 #define SYNCI mips_sync_icache();
89 #elif defined(CPU_NLM)
90 #define SYNCI xlp_sync()
96 * Exported variables for consumers like bus_dma code
98 int mips_picache_linesize;
99 int mips_pdcache_linesize;
101 static int picache_size;
102 static int picache_stride;
103 static int picache_loopcount;
104 static int picache_way_mask;
105 static int pdcache_size;
106 static int pdcache_stride;
107 static int pdcache_loopcount;
108 static int pdcache_way_mask;
109 static int sdcache_size;
110 static int sdcache_stride;
111 static int sdcache_loopcount;
112 static int sdcache_way_mask;
115 mipsNN_cache_init(struct mips_cpuinfo * cpuinfo)
117 int flush_multiple_lines_per_way;
119 flush_multiple_lines_per_way = cpuinfo->l1.ic_nsets * cpuinfo->l1.ic_linesize * cpuinfo->l1.ic_linesize > PAGE_SIZE;
120 if (cpuinfo->icache_virtual) {
122 * With a virtual Icache we don't need to flush
123 * multiples of the page size with index ops; we just
124 * need to flush one pages' worth.
126 flush_multiple_lines_per_way = 0;
129 if (flush_multiple_lines_per_way) {
130 picache_stride = PAGE_SIZE;
131 picache_loopcount = (cpuinfo->l1.ic_nsets * cpuinfo->l1.ic_linesize / PAGE_SIZE) *
132 cpuinfo->l1.ic_nways;
134 picache_stride = cpuinfo->l1.ic_nsets * cpuinfo->l1.ic_linesize;
135 picache_loopcount = cpuinfo->l1.ic_nways;
138 if (cpuinfo->l1.dc_nsets * cpuinfo->l1.dc_linesize < PAGE_SIZE) {
139 pdcache_stride = cpuinfo->l1.dc_nsets * cpuinfo->l1.dc_linesize;
140 pdcache_loopcount = cpuinfo->l1.dc_nways;
142 pdcache_stride = PAGE_SIZE;
143 pdcache_loopcount = (cpuinfo->l1.dc_nsets * cpuinfo->l1.dc_linesize / PAGE_SIZE) *
144 cpuinfo->l1.dc_nways;
147 mips_picache_linesize = cpuinfo->l1.ic_linesize;
148 mips_pdcache_linesize = cpuinfo->l1.dc_linesize;
150 picache_size = cpuinfo->l1.ic_size;
151 picache_way_mask = cpuinfo->l1.ic_nways - 1;
152 pdcache_size = cpuinfo->l1.dc_size;
153 pdcache_way_mask = cpuinfo->l1.dc_nways - 1;
155 sdcache_stride = cpuinfo->l2.dc_nsets * cpuinfo->l2.dc_linesize;
156 sdcache_loopcount = cpuinfo->l2.dc_nways;
157 sdcache_size = cpuinfo->l2.dc_size;
158 sdcache_way_mask = cpuinfo->l2.dc_nways - 1;
162 printf("Cache info:\n");
163 if (cpuinfo->icache_virtual)
164 printf(" icache is virtual\n");
165 printf(" picache_stride = %d\n", picache_stride);
166 printf(" picache_loopcount = %d\n", picache_loopcount);
167 printf(" pdcache_stride = %d\n", pdcache_stride);
168 printf(" pdcache_loopcount = %d\n", pdcache_loopcount);
173 mipsNN_icache_sync_all_16(void)
177 va = MIPS_PHYS_TO_KSEG0(0);
178 eva = va + picache_size;
181 * Since we're hitting the whole thing, we don't have to
182 * worry about the N different "ways".
185 mips_intern_dcache_wbinv_all();
188 cache_r4k_op_32lines_16(va, CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
196 mipsNN_icache_sync_all_32(void)
200 va = MIPS_PHYS_TO_KSEG0(0);
201 eva = va + picache_size;
204 * Since we're hitting the whole thing, we don't have to
205 * worry about the N different "ways".
208 mips_intern_dcache_wbinv_all();
211 cache_r4k_op_32lines_32(va, CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
219 mipsNN_icache_sync_all_64(void)
223 va = MIPS_PHYS_TO_KSEG0(0);
224 eva = va + picache_size;
227 * Since we're hitting the whole thing, we don't have to
228 * worry about the N different "ways".
231 mips_intern_dcache_wbinv_all();
234 cache_r4k_op_32lines_64(va, CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
242 mipsNN_icache_sync_range_16(vm_offset_t va, vm_size_t size)
246 eva = round_line16(va + size);
247 va = trunc_line16(va);
249 mips_intern_dcache_wb_range(va, (eva - va));
251 while ((eva - va) >= (32 * 16)) {
252 cache_r4k_op_32lines_16(va, CACHE_R4K_I|CACHEOP_R4K_HIT_INV);
257 cache_op_r4k_line(va, CACHE_R4K_I|CACHEOP_R4K_HIT_INV);
265 mipsNN_icache_sync_range_32(vm_offset_t va, vm_size_t size)
269 eva = round_line32(va + size);
270 va = trunc_line32(va);
272 mips_intern_dcache_wb_range(va, (eva - va));
274 while ((eva - va) >= (32 * 32)) {
275 cache_r4k_op_32lines_32(va, CACHE_R4K_I|CACHEOP_R4K_HIT_INV);
280 cache_op_r4k_line(va, CACHE_R4K_I|CACHEOP_R4K_HIT_INV);
288 mipsNN_icache_sync_range_64(vm_offset_t va, vm_size_t size)
292 eva = round_line64(va + size);
293 va = trunc_line64(va);
295 mips_intern_dcache_wb_range(va, (eva - va));
297 while ((eva - va) >= (32 * 64)) {
298 cache_r4k_op_32lines_64(va, CACHE_R4K_I|CACHEOP_R4K_HIT_INV);
303 cache_op_r4k_line(va, CACHE_R4K_I|CACHEOP_R4K_HIT_INV);
311 mipsNN_icache_sync_range_index_16(vm_offset_t va, vm_size_t size)
313 vm_offset_t eva, tmpva;
314 int i, stride, loopcount;
317 * Since we're doing Index ops, we expect to not be able
318 * to access the address we've been given. So, get the
319 * bits that determine the cache index, and make a KSEG0
320 * address out of them.
322 va = MIPS_PHYS_TO_KSEG0(va & picache_way_mask);
324 eva = round_line16(va + size);
325 va = trunc_line16(va);
328 * GCC generates better code in the loops if we reference local
329 * copies of these global variables.
331 stride = picache_stride;
332 loopcount = picache_loopcount;
334 mips_intern_dcache_wbinv_range_index(va, (eva - va));
336 while ((eva - va) >= (8 * 16)) {
338 for (i = 0; i < loopcount; i++, tmpva += stride)
339 cache_r4k_op_8lines_16(tmpva,
340 CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
346 for (i = 0; i < loopcount; i++, tmpva += stride)
347 cache_op_r4k_line(tmpva,
348 CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
354 mipsNN_icache_sync_range_index_32(vm_offset_t va, vm_size_t size)
356 vm_offset_t eva, tmpva;
357 int i, stride, loopcount;
360 * Since we're doing Index ops, we expect to not be able
361 * to access the address we've been given. So, get the
362 * bits that determine the cache index, and make a KSEG0
363 * address out of them.
365 va = MIPS_PHYS_TO_KSEG0(va & picache_way_mask);
367 eva = round_line32(va + size);
368 va = trunc_line32(va);
371 * GCC generates better code in the loops if we reference local
372 * copies of these global variables.
374 stride = picache_stride;
375 loopcount = picache_loopcount;
377 mips_intern_dcache_wbinv_range_index(va, (eva - va));
379 while ((eva - va) >= (8 * 32)) {
381 for (i = 0; i < loopcount; i++, tmpva += stride)
382 cache_r4k_op_8lines_32(tmpva,
383 CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
389 for (i = 0; i < loopcount; i++, tmpva += stride)
390 cache_op_r4k_line(tmpva,
391 CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
397 mipsNN_icache_sync_range_index_64(vm_offset_t va, vm_size_t size)
399 vm_offset_t eva, tmpva;
400 int i, stride, loopcount;
403 * Since we're doing Index ops, we expect to not be able
404 * to access the address we've been given. So, get the
405 * bits that determine the cache index, and make a KSEG0
406 * address out of them.
408 va = MIPS_PHYS_TO_KSEG0(va & picache_way_mask);
410 eva = round_line64(va + size);
411 va = trunc_line64(va);
414 * GCC generates better code in the loops if we reference local
415 * copies of these global variables.
417 stride = picache_stride;
418 loopcount = picache_loopcount;
420 mips_intern_dcache_wbinv_range_index(va, (eva - va));
422 while ((eva - va) >= (8 * 64)) {
424 for (i = 0; i < loopcount; i++, tmpva += stride)
425 cache_r4k_op_8lines_64(tmpva,
426 CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
432 for (i = 0; i < loopcount; i++, tmpva += stride)
433 cache_op_r4k_line(tmpva,
434 CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
440 mipsNN_pdcache_wbinv_all_16(void)
444 va = MIPS_PHYS_TO_KSEG0(0);
445 eva = va + pdcache_size;
448 * Since we're hitting the whole thing, we don't have to
449 * worry about the N different "ways".
453 cache_r4k_op_32lines_16(va,
454 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
462 mipsNN_pdcache_wbinv_all_32(void)
466 va = MIPS_PHYS_TO_KSEG0(0);
467 eva = va + pdcache_size;
470 * Since we're hitting the whole thing, we don't have to
471 * worry about the N different "ways".
475 cache_r4k_op_32lines_32(va,
476 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
484 mipsNN_pdcache_wbinv_all_64(void)
488 va = MIPS_PHYS_TO_KSEG0(0);
489 eva = va + pdcache_size;
492 * Since we're hitting the whole thing, we don't have to
493 * worry about the N different "ways".
497 cache_r4k_op_32lines_64(va,
498 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
506 mipsNN_pdcache_wbinv_range_16(vm_offset_t va, vm_size_t size)
510 eva = round_line16(va + size);
511 va = trunc_line16(va);
513 while ((eva - va) >= (32 * 16)) {
514 cache_r4k_op_32lines_16(va,
515 CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
520 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
528 mipsNN_pdcache_wbinv_range_32(vm_offset_t va, vm_size_t size)
532 eva = round_line32(va + size);
533 va = trunc_line32(va);
535 while ((eva - va) >= (32 * 32)) {
536 cache_r4k_op_32lines_32(va,
537 CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
542 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
550 mipsNN_pdcache_wbinv_range_64(vm_offset_t va, vm_size_t size)
554 eva = round_line64(va + size);
555 va = trunc_line64(va);
557 while ((eva - va) >= (32 * 64)) {
558 cache_r4k_op_32lines_64(va,
559 CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
564 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
572 mipsNN_pdcache_wbinv_range_index_16(vm_offset_t va, vm_size_t size)
574 vm_offset_t eva, tmpva;
575 int i, stride, loopcount;
578 * Since we're doing Index ops, we expect to not be able
579 * to access the address we've been given. So, get the
580 * bits that determine the cache index, and make a KSEG0
581 * address out of them.
583 va = MIPS_PHYS_TO_KSEG0(va & pdcache_way_mask);
585 eva = round_line16(va + size);
586 va = trunc_line16(va);
589 * GCC generates better code in the loops if we reference local
590 * copies of these global variables.
592 stride = pdcache_stride;
593 loopcount = pdcache_loopcount;
595 while ((eva - va) >= (8 * 16)) {
597 for (i = 0; i < loopcount; i++, tmpva += stride)
598 cache_r4k_op_8lines_16(tmpva,
599 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
605 for (i = 0; i < loopcount; i++, tmpva += stride)
606 cache_op_r4k_line(tmpva,
607 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
613 mipsNN_pdcache_wbinv_range_index_32(vm_offset_t va, vm_size_t size)
615 vm_offset_t eva, tmpva;
616 int i, stride, loopcount;
619 * Since we're doing Index ops, we expect to not be able
620 * to access the address we've been given. So, get the
621 * bits that determine the cache index, and make a KSEG0
622 * address out of them.
624 va = MIPS_PHYS_TO_KSEG0(va & pdcache_way_mask);
626 eva = round_line32(va + size);
627 va = trunc_line32(va);
630 * GCC generates better code in the loops if we reference local
631 * copies of these global variables.
633 stride = pdcache_stride;
634 loopcount = pdcache_loopcount;
636 while ((eva - va) >= (8 * 32)) {
638 for (i = 0; i < loopcount; i++, tmpva += stride)
639 cache_r4k_op_8lines_32(tmpva,
640 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
646 for (i = 0; i < loopcount; i++, tmpva += stride)
647 cache_op_r4k_line(tmpva,
648 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
654 mipsNN_pdcache_wbinv_range_index_64(vm_offset_t va, vm_size_t size)
656 vm_offset_t eva, tmpva;
657 int i, stride, loopcount;
660 * Since we're doing Index ops, we expect to not be able
661 * to access the address we've been given. So, get the
662 * bits that determine the cache index, and make a KSEG0
663 * address out of them.
665 va = MIPS_PHYS_TO_KSEG0(va & pdcache_way_mask);
667 eva = round_line64(va + size);
668 va = trunc_line64(va);
671 * GCC generates better code in the loops if we reference local
672 * copies of these global variables.
674 stride = pdcache_stride;
675 loopcount = pdcache_loopcount;
677 while ((eva - va) >= (8 * 64)) {
679 for (i = 0; i < loopcount; i++, tmpva += stride)
680 cache_r4k_op_8lines_64(tmpva,
681 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
687 for (i = 0; i < loopcount; i++, tmpva += stride)
688 cache_op_r4k_line(tmpva,
689 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
695 mipsNN_pdcache_inv_range_16(vm_offset_t va, vm_size_t size)
699 eva = round_line16(va + size);
700 va = trunc_line16(va);
702 while ((eva - va) >= (32 * 16)) {
703 cache_r4k_op_32lines_16(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
708 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
716 mipsNN_pdcache_inv_range_32(vm_offset_t va, vm_size_t size)
720 eva = round_line32(va + size);
721 va = trunc_line32(va);
723 while ((eva - va) >= (32 * 32)) {
724 cache_r4k_op_32lines_32(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
729 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
737 mipsNN_pdcache_inv_range_64(vm_offset_t va, vm_size_t size)
741 eva = round_line64(va + size);
742 va = trunc_line64(va);
744 while ((eva - va) >= (32 * 64)) {
745 cache_r4k_op_32lines_64(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
750 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
758 mipsNN_pdcache_wb_range_16(vm_offset_t va, vm_size_t size)
762 eva = round_line16(va + size);
763 va = trunc_line16(va);
765 while ((eva - va) >= (32 * 16)) {
766 cache_r4k_op_32lines_16(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
771 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
779 mipsNN_pdcache_wb_range_32(vm_offset_t va, vm_size_t size)
783 eva = round_line32(va + size);
784 va = trunc_line32(va);
786 while ((eva - va) >= (32 * 32)) {
787 cache_r4k_op_32lines_32(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
792 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
800 mipsNN_pdcache_wb_range_64(vm_offset_t va, vm_size_t size)
804 eva = round_line64(va + size);
805 va = trunc_line64(va);
807 while ((eva - va) >= (32 * 64)) {
808 cache_r4k_op_32lines_64(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
813 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
823 mipsNN_icache_sync_all_128(void)
829 mipsNN_icache_sync_range_128(vm_offset_t va, vm_size_t size)
835 mipsNN_icache_sync_range_index_128(vm_offset_t va, vm_size_t size)
841 mipsNN_pdcache_wbinv_all_128(void)
847 mipsNN_pdcache_wbinv_range_128(vm_offset_t va, vm_size_t size)
853 mipsNN_pdcache_wbinv_range_index_128(vm_offset_t va, vm_size_t size)
858 mipsNN_pdcache_inv_range_128(vm_offset_t va, vm_size_t size)
863 mipsNN_pdcache_wb_range_128(vm_offset_t va, vm_size_t size)
871 mipsNN_icache_sync_all_128(void)
875 va = MIPS_PHYS_TO_KSEG0(0);
876 eva = va + picache_size;
879 * Since we're hitting the whole thing, we don't have to
880 * worry about the N different "ways".
883 mips_intern_dcache_wbinv_all();
886 cache_r4k_op_32lines_128(va, CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
894 mipsNN_icache_sync_range_128(vm_offset_t va, vm_size_t size)
898 eva = round_line128(va + size);
899 va = trunc_line128(va);
901 mips_intern_dcache_wb_range(va, (eva - va));
903 while ((eva - va) >= (32 * 128)) {
904 cache_r4k_op_32lines_128(va, CACHE_R4K_I|CACHEOP_R4K_HIT_INV);
909 cache_op_r4k_line(va, CACHE_R4K_I|CACHEOP_R4K_HIT_INV);
917 mipsNN_icache_sync_range_index_128(vm_offset_t va, vm_size_t size)
919 vm_offset_t eva, tmpva;
920 int i, stride, loopcount;
923 * Since we're doing Index ops, we expect to not be able
924 * to access the address we've been given. So, get the
925 * bits that determine the cache index, and make a KSEG0
926 * address out of them.
928 va = MIPS_PHYS_TO_KSEG0(va & picache_way_mask);
930 eva = round_line128(va + size);
931 va = trunc_line128(va);
934 * GCC generates better code in the loops if we reference local
935 * copies of these global variables.
937 stride = picache_stride;
938 loopcount = picache_loopcount;
940 mips_intern_dcache_wbinv_range_index(va, (eva - va));
942 while ((eva - va) >= (32 * 128)) {
944 for (i = 0; i < loopcount; i++, tmpva += stride)
945 cache_r4k_op_32lines_128(tmpva,
946 CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
952 for (i = 0; i < loopcount; i++, tmpva += stride)
953 cache_op_r4k_line(tmpva,
954 CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
960 mipsNN_pdcache_wbinv_all_128(void)
964 va = MIPS_PHYS_TO_KSEG0(0);
965 eva = va + pdcache_size;
968 * Since we're hitting the whole thing, we don't have to
969 * worry about the N different "ways".
973 cache_r4k_op_32lines_128(va,
974 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
983 mipsNN_pdcache_wbinv_range_128(vm_offset_t va, vm_size_t size)
987 eva = round_line128(va + size);
988 va = trunc_line128(va);
990 while ((eva - va) >= (32 * 128)) {
991 cache_r4k_op_32lines_128(va,
992 CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
997 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
1005 mipsNN_pdcache_wbinv_range_index_128(vm_offset_t va, vm_size_t size)
1007 vm_offset_t eva, tmpva;
1008 int i, stride, loopcount;
1011 * Since we're doing Index ops, we expect to not be able
1012 * to access the address we've been given. So, get the
1013 * bits that determine the cache index, and make a KSEG0
1014 * address out of them.
1016 va = MIPS_PHYS_TO_KSEG0(va & pdcache_way_mask);
1018 eva = round_line128(va + size);
1019 va = trunc_line128(va);
1022 * GCC generates better code in the loops if we reference local
1023 * copies of these global variables.
1025 stride = pdcache_stride;
1026 loopcount = pdcache_loopcount;
1028 while ((eva - va) >= (32 * 128)) {
1030 for (i = 0; i < loopcount; i++, tmpva += stride)
1031 cache_r4k_op_32lines_128(tmpva,
1032 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
1038 for (i = 0; i < loopcount; i++, tmpva += stride)
1039 cache_op_r4k_line(tmpva,
1040 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
1046 mipsNN_pdcache_inv_range_128(vm_offset_t va, vm_size_t size)
1050 eva = round_line128(va + size);
1051 va = trunc_line128(va);
1053 while ((eva - va) >= (32 * 128)) {
1054 cache_r4k_op_32lines_128(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
1059 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
1067 mipsNN_pdcache_wb_range_128(vm_offset_t va, vm_size_t size)
1071 eva = round_line128(va + size);
1072 va = trunc_line128(va);
1074 while ((eva - va) >= (32 * 128)) {
1075 cache_r4k_op_32lines_128(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
1080 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
1090 mipsNN_sdcache_wbinv_all_32(void)
1092 vm_offset_t va = MIPS_PHYS_TO_KSEG0(0);
1093 vm_offset_t eva = va + sdcache_size;
1096 cache_r4k_op_32lines_32(va,
1097 CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1103 mipsNN_sdcache_wbinv_all_64(void)
1105 vm_offset_t va = MIPS_PHYS_TO_KSEG0(0);
1106 vm_offset_t eva = va + sdcache_size;
1109 cache_r4k_op_32lines_64(va,
1110 CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1116 mipsNN_sdcache_wbinv_range_32(vm_offset_t va, vm_size_t size)
1118 vm_offset_t eva = round_line32(va + size);
1120 va = trunc_line32(va);
1122 while ((eva - va) >= (32 * 32)) {
1123 cache_r4k_op_32lines_32(va,
1124 CACHE_R4K_SD|CACHEOP_R4K_HIT_WB_INV);
1129 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB_INV);
1135 mipsNN_sdcache_wbinv_range_64(vm_offset_t va, vm_size_t size)
1137 vm_offset_t eva = round_line64(va + size);
1139 va = trunc_line64(va);
1141 while ((eva - va) >= (32 * 64)) {
1142 cache_r4k_op_32lines_64(va,
1143 CACHE_R4K_SD|CACHEOP_R4K_HIT_WB_INV);
1148 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB_INV);
1154 mipsNN_sdcache_wbinv_range_index_32(vm_offset_t va, vm_size_t size)
1159 * Since we're doing Index ops, we expect to not be able
1160 * to access the address we've been given. So, get the
1161 * bits that determine the cache index, and make a KSEG0
1162 * address out of them.
1164 va = MIPS_PHYS_TO_KSEG0(va & (sdcache_size - 1));
1166 eva = round_line32(va + size);
1167 va = trunc_line32(va);
1169 while ((eva - va) >= (32 * 32)) {
1170 cache_r4k_op_32lines_32(va,
1171 CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1176 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1182 mipsNN_sdcache_wbinv_range_index_64(vm_offset_t va, vm_size_t size)
1187 * Since we're doing Index ops, we expect to not be able
1188 * to access the address we've been given. So, get the
1189 * bits that determine the cache index, and make a KSEG0
1190 * address out of them.
1192 va = MIPS_PHYS_TO_KSEG0(va & (sdcache_size - 1));
1194 eva = round_line64(va + size);
1195 va = trunc_line64(va);
1197 while ((eva - va) >= (32 * 64)) {
1198 cache_r4k_op_32lines_64(va,
1199 CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1204 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1210 mipsNN_sdcache_inv_range_32(vm_offset_t va, vm_size_t size)
1212 vm_offset_t eva = round_line32(va + size);
1214 va = trunc_line32(va);
1216 while ((eva - va) >= (32 * 32)) {
1217 cache_r4k_op_32lines_32(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_INV);
1222 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_INV);
1228 mipsNN_sdcache_inv_range_64(vm_offset_t va, vm_size_t size)
1230 vm_offset_t eva = round_line64(va + size);
1232 va = trunc_line64(va);
1234 while ((eva - va) >= (32 * 64)) {
1235 cache_r4k_op_32lines_64(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_INV);
1240 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_INV);
1246 mipsNN_sdcache_wb_range_32(vm_offset_t va, vm_size_t size)
1248 vm_offset_t eva = round_line32(va + size);
1250 va = trunc_line32(va);
1252 while ((eva - va) >= (32 * 32)) {
1253 cache_r4k_op_32lines_32(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB);
1258 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB);
1264 mipsNN_sdcache_wb_range_64(vm_offset_t va, vm_size_t size)
1266 vm_offset_t eva = round_line64(va + size);
1268 va = trunc_line64(va);
1270 while ((eva - va) >= (32 * 64)) {
1271 cache_r4k_op_32lines_64(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB);
1276 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB);
1282 mipsNN_sdcache_wbinv_all_128(void)
1284 vm_offset_t va = MIPS_PHYS_TO_KSEG0(0);
1285 vm_offset_t eva = va + sdcache_size;
1288 cache_r4k_op_32lines_128(va,
1289 CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1295 mipsNN_sdcache_wbinv_range_128(vm_offset_t va, vm_size_t size)
1297 vm_offset_t eva = round_line128(va + size);
1299 va = trunc_line128(va);
1301 while ((eva - va) >= (32 * 128)) {
1302 cache_r4k_op_32lines_128(va,
1303 CACHE_R4K_SD|CACHEOP_R4K_HIT_WB_INV);
1308 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB_INV);
1314 mipsNN_sdcache_wbinv_range_index_128(vm_offset_t va, vm_size_t size)
1319 * Since we're doing Index ops, we expect to not be able
1320 * to access the address we've been given. So, get the
1321 * bits that determine the cache index, and make a KSEG0
1322 * address out of them.
1324 va = MIPS_PHYS_TO_KSEG0(va & (sdcache_size - 1));
1326 eva = round_line128(va + size);
1327 va = trunc_line128(va);
1329 while ((eva - va) >= (32 * 128)) {
1330 cache_r4k_op_32lines_128(va,
1331 CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1336 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1342 mipsNN_sdcache_inv_range_128(vm_offset_t va, vm_size_t size)
1344 vm_offset_t eva = round_line128(va + size);
1346 va = trunc_line128(va);
1348 while ((eva - va) >= (32 * 128)) {
1349 cache_r4k_op_32lines_128(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_INV);
1354 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_INV);
1360 mipsNN_sdcache_wb_range_128(vm_offset_t va, vm_size_t size)
1362 vm_offset_t eva = round_line128(va + size);
1364 va = trunc_line128(va);
1366 while ((eva - va) >= (32 * 128)) {
1367 cache_r4k_op_32lines_128(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB);
1372 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB);