2 * Copyright 2014 Svatopluk Kraus <onwahe@gmail.com>
3 * Copyright 2014 Michal Meloun <meloun@miracle.cz>
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 #include <machine/asm.h>
32 #include <machine/asmacros.h>
33 #include <machine/armreg.h>
34 #include <machine/sysreg.h>
36 #define GET_PCB(tmp) \
37 mrc CP15_TPIDRPRW(tmp); \
38 add tmp, tmp, #(TD_PCB)
41 * Define cache functions used by startup code, which counts on the fact that
42 * only r0-r3,r12 (ip) are modified and no stack space is used. These functions
43 * must be called with interrupts disabled. Moreover, these work only with
44 * caches integrated to CPU (accessible via CP15); systems with an external L2
45 * cache controller such as a PL310 need separate calls to that device driver
46 * to affect L2 caches. This is not a factor during early kernel startup, as
47 * any external L2 cache controller has not been enabled yet.
50 /* Invalidate D cache to PoC. (aka all cache levels)*/
51 ASENTRY_NP(dcache_inv_poc_all)
58 ands r0, r0, #0x07000000
59 mov r0, r0, lsr #23 /* Get LoC 'naturally' aligned for */
60 beq 4f /* use in the CSSELR register below */
63 mcr CP15_CSSELR(r0) /* set cache level */
65 mrc CP15_CCSIDR(r0) /* read CCSIDR */
67 ubfx r2, r0, #13, #15 /* get num sets - 1 from CCSIDR */
68 ubfx r3, r0, #3, #10 /* get num ways - 1 from CCSIDR */
69 clz r1, r3 /* number of bits to MSB of way */
70 lsl r3, r3, r1 /* shift into position */
72 lsl ip, ip, r1 /* ip now contains the way decr */
74 ubfx r0, r0, #0, #3 /* get linesize from CCSIDR */
75 add r0, r0, #4 /* apply bias */
76 lsl r2, r2, r0 /* shift sets by log2(linesize) */
77 add r3, r3, r2 /* merge numsets - 1 with numways - 1 */
78 sub ip, ip, r2 /* subtract numsets - 1 from way decr */
80 lsl r1, r1, r0 /* r1 now contains the set decr */
81 mov r2, ip /* r2 now contains set way decr */
83 /* r3 = ways/sets, r2 = way decr, r1 = set decr, r0 and ip are free */
84 2: mcr CP15_DCISW(r3) /* invalidate line */
85 movs r0, r3 /* get current way/set */
86 beq 3f /* at 0 means we are done */
87 movs r0, r0, lsl #10 /* clear way bits leaving only set bits*/
88 subne r3, r3, r1 /* non-zero?, decrement set */
89 subeq r3, r3, r2 /* zero?, decrement way and restore set count */
93 mrc CP15_CSSELR(r0) /* get cache level */
97 4: dsb /* wait for stores to finish */
102 #endif /* __ARM_ARCH == 6 */
103 END(dcache_inv_poc_all)
105 /* Invalidate D cache to PoU. (aka L1 cache only)*/
106 ASENTRY_NP(dcache_inv_pou_all)
113 ands r0, r0, #0x38000000
114 mov r0, r0, lsr #26 /* Get LoUU (naturally aligned) */
118 mcr CP15_CSSELR(r0) /* set cache level */
120 mrc CP15_CCSIDR(r0) /* read CCSIDR */
122 ubfx r2, r0, #13, #15 /* get num sets - 1 from CCSIDR */
123 ubfx r3, r0, #3, #10 /* get num ways - 1 from CCSIDR */
124 clz r1, r3 /* number of bits to MSB of way */
125 lsl r3, r3, r1 /* shift into position */
127 lsl ip, ip, r1 /* ip now contains the way decr */
129 ubfx r0, r0, #0, #3 /* get linesize from CCSIDR */
130 add r0, r0, #4 /* apply bias */
131 lsl r2, r2, r0 /* shift sets by log2(linesize) */
132 add r3, r3, r2 /* merge numsets - 1 with numways - 1 */
133 sub ip, ip, r2 /* subtract numsets - 1 from way decr */
135 lsl r1, r1, r0 /* r1 now contains the set decr */
136 mov r2, ip /* r2 now contains set way decr */
138 /* r3 = ways/sets, r2 = way decr, r1 = set decr, r0 and ip are free */
139 2: mcr CP15_DCISW(r3) /* invalidate line */
140 movs r0, r3 /* get current way/set */
141 beq 3f /* at 0 means we are done */
142 movs r0, r0, lsl #10 /* clear way bits leaving only set bits*/
143 subne r3, r3, r1 /* non-zero?, decrement set */
144 subeq r3, r3, r2 /* zero?, decrement way and restore set count */
148 mrc CP15_CSSELR(r0) /* get cache level */
152 4: dsb /* wait for stores to finish */
157 END(dcache_inv_pou_all)
159 /* Write back and Invalidate D cache to PoC. */
160 ASENTRY_NP(dcache_wbinv_poc_all)
167 ands r0, r0, #0x07000000
169 mov r0, #0 /* Clean from inner to outer levels */
171 1: mcr CP15_CSSELR(r0) /* set cache level */
173 mrc CP15_CCSIDR(r0) /* read CCSIDR */
175 ubfx r2, r0, #13, #15 /* get num sets - 1 from CCSIDR */
176 ubfx r3, r0, #3, #10 /* get num ways - 1 from CCSIDR */
177 clz r1, r3 /* number of bits to MSB of way */
178 lsl r3, r3, r1 /* shift into position */
180 lsl ip, ip, r1 /* ip now contains the way decr */
182 ubfx r0, r0, #0, #3 /* get linesize from CCSIDR */
183 add r0, r0, #4 /* apply bias */
184 lsl r2, r2, r0 /* shift sets by log2(linesize) */
185 add r3, r3, r2 /* merge numsets - 1 with numways - 1 */
186 sub ip, ip, r2 /* subtract numsets - 1 from way decr */
188 lsl r1, r1, r0 /* r1 now contains the set decr */
189 mov r2, ip /* r2 now contains set way decr */
191 /* r3 = ways/sets, r2 = way decr, r1 = set decr, r0 and ip are free */
192 2: mcr CP15_DCCISW(r3) /* clean and invalidate line */
193 movs r0, r3 /* get current way/set */
194 beq 3f /* at 0 means we are done */
195 movs r0, r0, lsl #10 /* clear way bits leaving only set bits*/
196 subne r3, r3, r1 /* non-zero?, decrement set */
197 subeq r3, r3, r2 /* zero?, decrement way and restore set count */
201 mrc CP15_CSSELR(r0) /* get cache level */
202 add r0, r0, #2 /* next level */
204 ands r1, r1, #0x07000000
205 mov r1, r1, lsr #23 /* Get LoC (naturally aligned) */
209 4: dsb /* wait for stores to finish */
213 #endif /* __ARM_ARCH == 6 */
214 END(dcache_wbinv_poc_all)
216 ASENTRY_NP(dcache_wb_pou_checked)
218 ldr ip, [ip, #DCACHE_LINE_SIZE]
223 adr r3, _C_LABEL(cachebailout)
224 str r3, [r2, #PCB_ONFAULT]
232 str r0, [r2, #PCB_ONFAULT]
233 mov r0, #1 /* cannot be faulting address */
238 END(dcache_wb_pou_checked)
240 ASENTRY_NP(icache_inv_pou_checked)
242 ldr ip, [ip, #ICACHE_LINE_SIZE]
247 adr r3, _C_LABEL(cachebailout)
248 str r3, [r2, #PCB_ONFAULT]
258 str r0, [r2, #PCB_ONFAULT]
259 mov r0, #1 /* cannot be faulting address */
261 END(icache_inv_pou_checked)
263 /* label must be global as trap-v6.c references it */
264 .global _C_LABEL(cachebailout)
265 _C_LABEL(cachebailout):
269 str r1, [r2, #PCB_ONFAULT]