2 * Copyright (c) 2002 Jake Burkholder.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 #include <machine/asm.h>
28 __FBSDID("$FreeBSD$");
30 #include <machine/asi.h>
31 #include <machine/asmacros.h>
32 #include <machine/cache.h>
33 #include <machine/ktr.h>
34 #include <machine/pstate.h>
38 .register %g2, #ignore
39 .register %g3, #ignore
41 #define IPI_DONE(r1, r2, r3, r4, r5, r6) \
43 lduw [PCPU(CPUID)], r2 ; \
48 sllx r5, PTR_SHIFT, r5 ; \
55 ATOMIC_CLEAR_LONG(r1, r2, r3, r4)
58 * Invalidate a physical page in the data cache. For UltraSPARC I and II.
60 ENTRY(tl_ipi_spitfire_dcache_page_inval)
61 #if KTR_COMPILE & KTR_SMP
62 CATR(KTR_SMP, "tl_ipi_spitfire_dcache_page_inval: pa=%#lx"
63 , %g1, %g2, %g3, 7, 8, 9)
64 ldx [%g5 + ICA_PA], %g2
65 stx %g2, [%g1 + KTR_PARM1]
69 ldx [%g5 + ICA_PA], %g6
70 srlx %g6, PAGE_SHIFT - DC_TAG_SHIFT, %g6
72 lduw [PCPU(CACHE) + DC_SIZE], %g3
73 lduw [PCPU(CACHE) + DC_LINESIZE], %g4
76 1: ldxa [%g2] ASI_DCACHE_TAG, %g1
77 srlx %g1, DC_VALID_SHIFT, %g3
78 andcc %g3, DC_VALID_MASK, %g0
81 sllx %g3, DC_TAG_SHIFT, %g3
86 stxa %g1, [%g2] ASI_DCACHE_TAG
92 IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6)
94 END(tl_ipi_spitfire_dcache_page_inval)
97 * Invalidate a physical page in the instruction cache. For UltraSPARC I and
100 ENTRY(tl_ipi_spitfire_icache_page_inval)
101 #if KTR_COMPILE & KTR_SMP
102 CATR(KTR_SMP, "tl_ipi_spitfire_icache_page_inval: pa=%#lx"
103 , %g1, %g2, %g3, 7, 8, 9)
104 ldx [%g5 + ICA_PA], %g2
105 stx %g2, [%g1 + KTR_PARM1]
109 ldx [%g5 + ICA_PA], %g6
110 srlx %g6, PAGE_SHIFT - IC_TAG_SHIFT, %g6
112 lduw [PCPU(CACHE) + IC_SIZE], %g3
113 lduw [PCPU(CACHE) + IC_LINESIZE], %g4
116 1: ldda [%g2] ASI_ICACHE_TAG, %g0 /*, %g1 */
117 srlx %g1, IC_VALID_SHIFT, %g3
118 andcc %g3, IC_VALID_MASK, %g0
121 sllx %g3, IC_TAG_SHIFT, %g3
126 stxa %g1, [%g2] ASI_ICACHE_TAG
132 IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6)
134 END(tl_ipi_spitfire_icache_page_inval)
137 * Invalidate a physical page in the data cache. For UltraSPARC III.
139 ENTRY(tl_ipi_cheetah_dcache_page_inval)
140 #if KTR_COMPILE & KTR_SMP
141 CATR(KTR_SMP, "tl_ipi_cheetah_dcache_page_inval: pa=%#lx"
142 , %g1, %g2, %g3, 7, 8, 9)
143 ldx [%g5 + ICA_PA], %g2
144 stx %g2, [%g1 + KTR_PARM1]
148 ldx [%g5 + ICA_PA], %g1
153 lduw [PCPU(CACHE) + DC_LINESIZE], %g2
155 1: stxa %g0, [%g1] ASI_DCACHE_INVALIDATE
163 IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6)
165 END(tl_ipi_cheetah_dcache_page_inval)
168 * Trigger a softint at the desired level.
171 #if KTR_COMPILE & KTR_SMP
172 CATR(KTR_SMP, "tl_ipi_level: cpuid=%d mid=%d d1=%#lx d2=%#lx"
173 , %g1, %g2, %g3, 7, 8, 9)
174 lduw [PCPU(CPUID)], %g2
175 stx %g2, [%g1 + KTR_PARM1]
176 lduw [PCPU(MID)], %g2
177 stx %g2, [%g1 + KTR_PARM2]
178 stx %g4, [%g1 + KTR_PARM3]
179 stx %g5, [%g1 + KTR_PARM4]
185 wr %g1, 0, %set_softint
190 * Demap a page from the dtlb and/or itlb.
192 ENTRY(tl_ipi_tlb_page_demap)
193 #if KTR_COMPILE & KTR_SMP
194 CATR(KTR_SMP, "ipi_tlb_page_demap: pm=%p va=%#lx"
195 , %g1, %g2, %g3, 7, 8, 9)
196 ldx [%g5 + ITA_PMAP], %g2
197 stx %g2, [%g1 + KTR_PARM1]
198 ldx [%g5 + ITA_VA], %g2
199 stx %g2, [%g1 + KTR_PARM2]
203 ldx [%g5 + ITA_PMAP], %g1
205 SET(kernel_pmap_store, %g3, %g2)
206 mov TLB_DEMAP_NUCLEUS | TLB_DEMAP_PAGE, %g3
209 movne %xcc, TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, %g3
211 ldx [%g5 + ITA_VA], %g2
214 sethi %hi(KERNBASE), %g3
215 stxa %g0, [%g2] ASI_DMMU_DEMAP
216 stxa %g0, [%g2] ASI_IMMU_DEMAP
219 IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6)
221 END(tl_ipi_tlb_page_demap)
224 * Demap a range of pages from the dtlb and itlb.
226 ENTRY(tl_ipi_tlb_range_demap)
227 #if KTR_COMPILE & KTR_SMP
228 CATR(KTR_SMP, "ipi_tlb_range_demap: pm=%p start=%#lx end=%#lx"
229 , %g1, %g2, %g3, 7, 8, 9)
230 ldx [%g5 + ITA_PMAP], %g2
231 stx %g2, [%g1 + KTR_PARM1]
232 ldx [%g5 + ITA_START], %g2
233 stx %g2, [%g1 + KTR_PARM2]
234 ldx [%g5 + ITA_END], %g2
235 stx %g2, [%g1 + KTR_PARM3]
239 ldx [%g5 + ITA_PMAP], %g1
241 SET(kernel_pmap_store, %g3, %g2)
242 mov TLB_DEMAP_NUCLEUS | TLB_DEMAP_PAGE, %g3
245 movne %xcc, TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, %g3
247 ldx [%g5 + ITA_START], %g1
248 ldx [%g5 + ITA_END], %g2
250 sethi %hi(KERNBASE), %g6
252 stxa %g0, [%g4] ASI_DMMU_DEMAP
253 stxa %g0, [%g4] ASI_IMMU_DEMAP
260 sethi %hi(KERNBASE), %g6
262 IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6)
264 END(tl_ipi_tlb_range_demap)
267 * Demap the primary context from the dtlb and itlb.
269 ENTRY(tl_ipi_tlb_context_demap)
270 #if KTR_COMPILE & KTR_SMP
271 CATR(KTR_SMP, "tl_ipi_tlb_context_demap: pm=%p va=%#lx"
272 , %g1, %g2, %g3, 7, 8, 9)
273 ldx [%g5 + ITA_PMAP], %g2
274 stx %g2, [%g1 + KTR_PARM1]
275 ldx [%g5 + ITA_VA], %g2
276 stx %g2, [%g1 + KTR_PARM2]
280 mov TLB_DEMAP_PRIMARY | TLB_DEMAP_CONTEXT, %g1
281 sethi %hi(KERNBASE), %g3
282 stxa %g0, [%g1] ASI_DMMU_DEMAP
283 stxa %g0, [%g1] ASI_IMMU_DEMAP
286 IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6)
288 END(tl_ipi_tlb_context_demap)
293 ENTRY(tl_ipi_stick_rd)
294 ldx [%g5 + IRA_VAL], %g1
298 IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6)
305 ENTRY(tl_ipi_tick_rd)
306 ldx [%g5 + IRA_VAL], %g1
310 IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6)