2 * Copyright (C) 2006-2009 Semihalf, Rafal Jaworowski <raj@semihalf.com>
3 * Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com>
4 * Copyright (C) 2006 Juniper Networks, Inc.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. The name of the author may not be used to endorse or promote products
16 * derived from this software without specific prior written permission.
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
21 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
23 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
24 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
25 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
26 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 * Copyright (C) 1995, 1996 Wolfgang Solfrank.
33 * Copyright (C) 1995, 1996 TooLs GmbH.
34 * All rights reserved.
36 * Redistribution and use in source and binary forms, with or without
37 * modification, are permitted provided that the following conditions
39 * 1. Redistributions of source code must retain the above copyright
40 * notice, this list of conditions and the following disclaimer.
41 * 2. Redistributions in binary form must reproduce the above copyright
42 * notice, this list of conditions and the following disclaimer in the
43 * documentation and/or other materials provided with the distribution.
44 * 3. All advertising materials mentioning features or use of this software
45 * must display the following acknowledgement:
46 * This product includes software developed by TooLs GmbH.
47 * 4. The name of TooLs GmbH may not be used to endorse or promote products
48 * derived from this software without specific prior written permission.
50 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
51 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
52 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
53 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
54 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
55 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
56 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
57 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
58 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
59 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
61 * from: $NetBSD: trap_subr.S,v 1.20 2002/04/22 23:20:08 kleink Exp $
65 * NOTICE: This is not a standalone file. to use it, #include it in
66 * your port's locore.S, like so:
68 * #include <powerpc/booke/trap_subr.S>
74 * SPRG0 - pcpu pointer
75 * SPRG1 - all interrupts except TLB miss, critical, machine check
77 * SPRG3 - machine check
82 /* Get the per-CPU data structure */
83 #define GET_CPUINFO(r) mfsprg0 r
85 #define RES_GRANULE 64
86 #define RES_LOCK 0 /* offset to the 'lock' word */
88 #define RES_RECURSE 8 /* offset to the 'recurse' word */
90 #define RES_RECURSE 4 /* offset to the 'recurse' word */
94 * Standard interrupt prolog
96 * sprg_sp - SPRG{1-3} reg used to temporarily store the SP
97 * savearea - temp save area (pc_{tempsave, disisave, critsave, mchksave})
98 * isrr0-1 - save restore registers with CPU state at interrupt time (may be
99 * SRR0-1, CSRR0-1, MCSRR0-1
101 * 1. saves in the given savearea:
108 * 3. switches to kstack if needed
111 * - R31 can be used as scratch register until a new frame is laid on
112 * the stack with FRAME_SETUP
114 * - potential TLB miss: NO. Saveareas are always acessible via TLB1
115 * permanent entries, and within this prolog we do not dereference any
116 * locations potentially not in the TLB
118 #define STANDARD_PROLOG(sprg_sp, savearea, isrr0, isrr1) \
119 mtspr sprg_sp, %r1; /* Save SP */ \
120 GET_CPUINFO(%r1); /* Per-cpu structure */ \
121 STORE %r30, (savearea+CPUSAVE_R30)(%r1); \
122 STORE %r31, (savearea+CPUSAVE_R31)(%r1); \
123 mfspr %r30, SPR_DEAR; \
124 mfspr %r31, SPR_ESR; \
125 STORE %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1); \
126 STORE %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1); \
128 mfspr %r31, isrr1; /* MSR at interrupt time */ \
129 STORE %r30, (savearea+CPUSAVE_SRR0)(%r1); \
130 STORE %r31, (savearea+CPUSAVE_SRR1)(%r1); \
132 mfspr %r1, sprg_sp; /* Restore SP */ \
133 mfcr %r30; /* Save CR */ \
134 /* switch to per-thread kstack if intr taken in user mode */ \
135 mtcr %r31; /* MSR at interrupt time */ \
137 GET_CPUINFO(%r1); /* Per-cpu structure */ \
138 LOAD %r1, PC_CURPCB(%r1); /* Per-thread kernel stack */ \
141 #define STANDARD_CRIT_PROLOG(sprg_sp, savearea, isrr0, isrr1) \
142 mtspr sprg_sp, %r1; /* Save SP */ \
143 GET_CPUINFO(%r1); /* Per-cpu structure */ \
144 STORE %r30, (savearea+CPUSAVE_R30)(%r1); \
145 STORE %r31, (savearea+CPUSAVE_R31)(%r1); \
146 mfspr %r30, SPR_DEAR; \
147 mfspr %r31, SPR_ESR; \
148 STORE %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1); \
149 STORE %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1); \
151 mfspr %r31, isrr1; /* MSR at interrupt time */ \
152 STORE %r30, (savearea+CPUSAVE_SRR0)(%r1); \
153 STORE %r31, (savearea+CPUSAVE_SRR1)(%r1); \
154 mfspr %r30, SPR_SRR0; \
155 mfspr %r31, SPR_SRR1; /* MSR at interrupt time */ \
156 STORE %r30, (savearea+BOOKE_CRITSAVE_SRR0)(%r1); \
157 STORE %r31, (savearea+BOOKE_CRITSAVE_SRR1)(%r1); \
159 mfspr %r1, sprg_sp; /* Restore SP */ \
160 mfcr %r30; /* Save CR */ \
161 /* switch to per-thread kstack if intr taken in user mode */ \
162 mtcr %r31; /* MSR at interrupt time */ \
164 GET_CPUINFO(%r1); /* Per-cpu structure */ \
165 LOAD %r1, PC_CURPCB(%r1); /* Per-thread kernel stack */ \
169 * FRAME_SETUP assumes:
170 * SPRG{1-3} SP at the time interrupt occurred
171 * savearea r30-r31, DEAR, ESR, xSRR0-1
176 * sprg_sp - SPRG reg containing SP at the time interrupt occurred
177 * savearea - temp save
178 * exc - exception number (EXC_xxx)
180 * 1. sets a new frame
181 * 2. saves in the frame:
182 * - R0, R1 (SP at the time of interrupt), R2, LR, CR
183 * - R3-31 (R30-31 first restored from savearea)
184 * - XER, CTR, DEAR, ESR (from savearea), xSRR0-1
187 * - potential TLB miss: YES, since we make dereferences to kstack, which
188 * can happen not covered (we can have up to two DTLB misses if fortunate
189 * enough i.e. when kstack crosses page boundary and both pages are
193 #define SAVE_REGS(r) \
194 std %r3, FRAME_3+CALLSIZE(r); \
195 std %r4, FRAME_4+CALLSIZE(r); \
196 std %r5, FRAME_5+CALLSIZE(r); \
197 std %r6, FRAME_6+CALLSIZE(r); \
198 std %r7, FRAME_7+CALLSIZE(r); \
199 std %r8, FRAME_8+CALLSIZE(r); \
200 std %r9, FRAME_9+CALLSIZE(r); \
201 std %r10, FRAME_10+CALLSIZE(r); \
202 std %r11, FRAME_11+CALLSIZE(r); \
203 std %r12, FRAME_12+CALLSIZE(r); \
204 std %r13, FRAME_13+CALLSIZE(r); \
205 std %r14, FRAME_14+CALLSIZE(r); \
206 std %r15, FRAME_15+CALLSIZE(r); \
207 std %r16, FRAME_16+CALLSIZE(r); \
208 std %r17, FRAME_17+CALLSIZE(r); \
209 std %r18, FRAME_18+CALLSIZE(r); \
210 std %r19, FRAME_19+CALLSIZE(r); \
211 std %r20, FRAME_20+CALLSIZE(r); \
212 std %r21, FRAME_21+CALLSIZE(r); \
213 std %r22, FRAME_22+CALLSIZE(r); \
214 std %r23, FRAME_23+CALLSIZE(r); \
215 std %r24, FRAME_24+CALLSIZE(r); \
216 std %r25, FRAME_25+CALLSIZE(r); \
217 std %r26, FRAME_26+CALLSIZE(r); \
218 std %r27, FRAME_27+CALLSIZE(r); \
219 std %r28, FRAME_28+CALLSIZE(r); \
220 std %r29, FRAME_29+CALLSIZE(r); \
221 std %r30, FRAME_30+CALLSIZE(r); \
222 std %r31, FRAME_31+CALLSIZE(r)
224 ld %r3, FRAME_3+CALLSIZE(r); \
225 ld %r4, FRAME_4+CALLSIZE(r); \
226 ld %r5, FRAME_5+CALLSIZE(r); \
227 ld %r6, FRAME_6+CALLSIZE(r); \
228 ld %r7, FRAME_7+CALLSIZE(r); \
229 ld %r8, FRAME_8+CALLSIZE(r); \
230 ld %r9, FRAME_9+CALLSIZE(r); \
231 ld %r10, FRAME_10+CALLSIZE(r); \
232 ld %r11, FRAME_11+CALLSIZE(r); \
233 ld %r12, FRAME_12+CALLSIZE(r); \
234 ld %r13, FRAME_13+CALLSIZE(r); \
235 ld %r14, FRAME_14+CALLSIZE(r); \
236 ld %r15, FRAME_15+CALLSIZE(r); \
237 ld %r16, FRAME_16+CALLSIZE(r); \
238 ld %r17, FRAME_17+CALLSIZE(r); \
239 ld %r18, FRAME_18+CALLSIZE(r); \
240 ld %r19, FRAME_19+CALLSIZE(r); \
241 ld %r20, FRAME_20+CALLSIZE(r); \
242 ld %r21, FRAME_21+CALLSIZE(r); \
243 ld %r22, FRAME_22+CALLSIZE(r); \
244 ld %r23, FRAME_23+CALLSIZE(r); \
245 ld %r24, FRAME_24+CALLSIZE(r); \
246 ld %r25, FRAME_25+CALLSIZE(r); \
247 ld %r26, FRAME_26+CALLSIZE(r); \
248 ld %r27, FRAME_27+CALLSIZE(r); \
249 ld %r28, FRAME_28+CALLSIZE(r); \
250 ld %r29, FRAME_29+CALLSIZE(r); \
251 ld %r30, FRAME_30+CALLSIZE(r); \
252 ld %r31, FRAME_31+CALLSIZE(r)
254 #define SAVE_REGS(r) \
255 stmw %r3, FRAME_3+CALLSIZE(r)
257 lmw %r3, FRAME_3+CALLSIZE(r)
259 #define FRAME_SETUP(sprg_sp, savearea, exc) \
260 mfspr %r31, sprg_sp; /* get saved SP */ \
261 /* establish a new stack frame and put everything on it */ \
262 STU %r31, -(FRAMELEN+REDZONE)(%r1); \
263 STORE %r0, FRAME_0+CALLSIZE(%r1); /* save r0 in the trapframe */ \
264 STORE %r31, FRAME_1+CALLSIZE(%r1); /* save SP " " */ \
265 STORE %r2, FRAME_2+CALLSIZE(%r1); /* save r2 " " */ \
267 STORE %r31, FRAME_LR+CALLSIZE(%r1); /* save LR " " */ \
268 STORE %r30, FRAME_CR+CALLSIZE(%r1); /* save CR " " */ \
270 LOAD %r30, (savearea+CPUSAVE_R30)(%r2); /* get saved r30 */ \
271 LOAD %r31, (savearea+CPUSAVE_R31)(%r2); /* get saved r31 */ \
274 /* save DEAR, ESR */ \
275 LOAD %r28, (savearea+CPUSAVE_BOOKE_DEAR)(%r2); \
276 LOAD %r29, (savearea+CPUSAVE_BOOKE_ESR)(%r2); \
277 STORE %r28, FRAME_BOOKE_DEAR+CALLSIZE(%r1); \
278 STORE %r29, FRAME_BOOKE_ESR+CALLSIZE(%r1); \
279 /* save XER, CTR, exc number */ \
282 STORE %r3, FRAME_XER+CALLSIZE(%r1); \
283 STORE %r4, FRAME_CTR+CALLSIZE(%r1); \
285 STORE %r5, FRAME_EXC+CALLSIZE(%r1); \
287 mfspr %r3, SPR_DBCR0; \
288 STORE %r3, FRAME_BOOKE_DBCR0+CALLSIZE(%r1); \
290 LOAD %r30, (savearea+CPUSAVE_SRR0)(%r2); \
291 LOAD %r31, (savearea+CPUSAVE_SRR1)(%r2); \
292 STORE %r30, FRAME_SRR0+CALLSIZE(%r1); \
293 STORE %r31, FRAME_SRR1+CALLSIZE(%r1); \
294 LOAD THREAD_REG, PC_CURTHREAD(%r2); \
298 * isrr0-1 - save restore registers to restore CPU state to (may be
299 * SRR0-1, CSRR0-1, MCSRR0-1
302 * - potential TLB miss: YES. The deref'd kstack may be not covered
304 #define FRAME_LEAVE(isrr0, isrr1) \
306 /* restore CTR, XER, LR, CR */ \
307 LOAD %r4, FRAME_CTR+CALLSIZE(%r1); \
308 LOAD %r5, FRAME_XER+CALLSIZE(%r1); \
309 LOAD %r6, FRAME_LR+CALLSIZE(%r1); \
310 LOAD %r7, FRAME_CR+CALLSIZE(%r1); \
315 /* restore DBCR0 */ \
316 LOAD %r4, FRAME_BOOKE_DBCR0+CALLSIZE(%r1); \
317 mtspr SPR_DBCR0, %r4; \
318 /* restore xSRR0-1 */ \
319 LOAD %r30, FRAME_SRR0+CALLSIZE(%r1); \
320 LOAD %r31, FRAME_SRR1+CALLSIZE(%r1); \
323 /* restore R2-31, SP */ \
325 LOAD %r2, FRAME_2+CALLSIZE(%r1); \
326 LOAD %r0, FRAME_0+CALLSIZE(%r1); \
327 LOAD %r1, FRAME_1+CALLSIZE(%r1); \
333 * saves LR, CR, SRR0-1, R20-31 in the TLBSAVE area
336 * - potential TLB miss: NO. It is crucial that we do not generate a TLB
337 * miss within the TLB prolog itself!
338 * - TLBSAVE is always translated
341 #define TLB_SAVE_REGS(br) \
342 std %r20, (TLBSAVE_BOOKE_R20)(br); \
343 std %r21, (TLBSAVE_BOOKE_R21)(br); \
344 std %r22, (TLBSAVE_BOOKE_R22)(br); \
345 std %r23, (TLBSAVE_BOOKE_R23)(br); \
346 std %r24, (TLBSAVE_BOOKE_R24)(br); \
347 std %r25, (TLBSAVE_BOOKE_R25)(br); \
348 std %r26, (TLBSAVE_BOOKE_R26)(br); \
349 std %r27, (TLBSAVE_BOOKE_R27)(br); \
350 std %r28, (TLBSAVE_BOOKE_R28)(br); \
351 std %r29, (TLBSAVE_BOOKE_R29)(br); \
352 std %r30, (TLBSAVE_BOOKE_R30)(br); \
353 std %r31, (TLBSAVE_BOOKE_R31)(br);
354 #define TLB_RESTORE_REGS(br) \
355 ld %r20, (TLBSAVE_BOOKE_R20)(br); \
356 ld %r21, (TLBSAVE_BOOKE_R21)(br); \
357 ld %r22, (TLBSAVE_BOOKE_R22)(br); \
358 ld %r23, (TLBSAVE_BOOKE_R23)(br); \
359 ld %r24, (TLBSAVE_BOOKE_R24)(br); \
360 ld %r25, (TLBSAVE_BOOKE_R25)(br); \
361 ld %r26, (TLBSAVE_BOOKE_R26)(br); \
362 ld %r27, (TLBSAVE_BOOKE_R27)(br); \
363 ld %r28, (TLBSAVE_BOOKE_R28)(br); \
364 ld %r29, (TLBSAVE_BOOKE_R29)(br); \
365 ld %r30, (TLBSAVE_BOOKE_R30)(br); \
366 ld %r31, (TLBSAVE_BOOKE_R31)(br);
367 #define TLB_NEST(outr,inr) \
368 rlwinm outr, inr, 7, 23, 24; /* 8 x TLBSAVE_LEN */
370 #define TLB_SAVE_REGS(br) \
371 stmw %r20, TLBSAVE_BOOKE_R20(br)
372 #define TLB_RESTORE_REGS(br) \
373 lmw %r20, TLBSAVE_BOOKE_R20(br)
374 #define TLB_NEST(outr,inr) \
375 rlwinm outr, inr, 6, 24, 25; /* 4 x TLBSAVE_LEN */
378 mtspr SPR_SPRG4, %r1; /* Save SP */ \
379 mtspr SPR_SPRG5, %r28; \
380 mtspr SPR_SPRG6, %r29; \
381 /* calculate TLB nesting level and TLBSAVE instance address */ \
382 GET_CPUINFO(%r1); /* Per-cpu structure */ \
383 LOAD %r28, PC_BOOKE_TLB_LEVEL(%r1); \
384 TLB_NEST(%r29,%r28); \
385 addi %r28, %r28, 1; \
386 STORE %r28, PC_BOOKE_TLB_LEVEL(%r1); \
387 addi %r29, %r29, PC_BOOKE_TLBSAVE@l; \
388 add %r1, %r1, %r29; /* current TLBSAVE ptr */ \
391 mfspr %r28, SPR_SPRG5; \
392 mfspr %r29, SPR_SPRG6; \
393 TLB_SAVE_REGS(%r1); \
397 STORE %r30, (TLBSAVE_BOOKE_LR)(%r1); \
398 STORE %r31, (TLBSAVE_BOOKE_CR)(%r1); \
400 mfsrr0 %r30; /* execution addr at interrupt time */ \
401 mfsrr1 %r31; /* MSR at interrupt time*/ \
402 STORE %r30, (TLBSAVE_BOOKE_SRR0)(%r1); /* save SRR0 */ \
403 STORE %r31, (TLBSAVE_BOOKE_SRR1)(%r1); /* save SRR1 */ \
408 * restores LR, CR, SRR0-1, R20-31 from the TLBSAVE area
410 * same notes as for the TLB_PROLOG
412 #define TLB_RESTORE \
413 mtspr SPR_SPRG4, %r1; /* Save SP */ \
414 GET_CPUINFO(%r1); /* Per-cpu structure */ \
415 /* calculate TLB nesting level and TLBSAVE instance addr */ \
416 LOAD %r28, PC_BOOKE_TLB_LEVEL(%r1); \
417 subi %r28, %r28, 1; \
418 STORE %r28, PC_BOOKE_TLB_LEVEL(%r1); \
419 TLB_NEST(%r29,%r28); \
420 addi %r29, %r29, PC_BOOKE_TLBSAVE@l; \
421 add %r1, %r1, %r29; \
423 /* restore LR, CR */ \
424 LOAD %r30, (TLBSAVE_BOOKE_LR)(%r1); \
425 LOAD %r31, (TLBSAVE_BOOKE_CR)(%r1); \
428 /* restore SRR0-1 */ \
429 LOAD %r30, (TLBSAVE_BOOKE_SRR0)(%r1); \
430 LOAD %r31, (TLBSAVE_BOOKE_SRR1)(%r1); \
433 /* restore R20-31 */ \
434 TLB_RESTORE_REGS(%r1); \
440 LOAD %r21, PC_CURTHREAD(%r20); \
441 LOAD %r22, PC_BOOKE_TLB_LOCK(%r20); \
443 1: LOADX %r23, 0, %r22; \
444 CMPI %r23, TLB_UNLOCKED; \
447 /* check if this is recursion */ \
448 CMPL cr0, %r21, %r23; \
451 2: /* try to acquire lock */ \
452 STOREX %r21, 0, %r22; \
455 /* got it, update recursion counter */ \
456 lwz %r21, RES_RECURSE(%r22); \
457 addi %r21, %r21, 1; \
458 stw %r21, RES_RECURSE(%r22); \
464 LOAD %r21, PC_CURTHREAD(%r20); \
465 LOAD %r22, PC_BOOKE_TLB_LOCK(%r20); \
467 /* update recursion counter */ \
468 lwz %r23, RES_RECURSE(%r22); \
469 subi %r23, %r23, 1; \
470 stw %r23, RES_RECURSE(%r22); \
477 /* release the lock */ \
478 li %r23, TLB_UNLOCKED; \
479 STORE %r23, 0(%r22); \
487 #define INTERRUPT(label) \
493 * Interrupt handling routines in BookE can be flexibly placed and do not have
494 * to live in pre-defined vectors location. Note they need to be TLB-mapped at
495 * all times in order to be able to handle exceptions. We thus arrange for
496 * them to be part of kernel text which is always TLB-accessible.
498 * The interrupt handling routines have to be 16 bytes aligned: we align them
499 * to 32 bytes (cache line length) which supposedly performs better.
503 .globl CNAME(interrupt_vector_base)
505 interrupt_vector_base:
506 /*****************************************************************************
507 * Catch-all handler to handle uninstalled IVORs
508 ****************************************************************************/
509 INTERRUPT(int_unknown)
510 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
511 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_RSVD)
514 /*****************************************************************************
515 * Critical input interrupt
516 ****************************************************************************/
517 INTERRUPT(int_critical_input)
518 STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1)
519 FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_CRIT)
521 addi %r3, %r1, CALLSIZE
522 bl CNAME(powerpc_interrupt)
524 FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1)
528 /*****************************************************************************
529 * Machine check interrupt
530 ****************************************************************************/
531 INTERRUPT(int_machine_check)
532 STANDARD_PROLOG(SPR_SPRG3, PC_BOOKE_MCHKSAVE, SPR_MCSRR0, SPR_MCSRR1)
533 FRAME_SETUP(SPR_SPRG3, PC_BOOKE_MCHKSAVE, EXC_MCHK)
535 addi %r3, %r1, CALLSIZE
536 bl CNAME(powerpc_interrupt)
538 FRAME_LEAVE(SPR_MCSRR0, SPR_MCSRR1)
542 /*****************************************************************************
543 * Data storage interrupt
544 ****************************************************************************/
545 INTERRUPT(int_data_storage)
546 STANDARD_PROLOG(SPR_SPRG1, PC_DISISAVE, SPR_SRR0, SPR_SRR1)
547 FRAME_SETUP(SPR_SPRG1, PC_DISISAVE, EXC_DSI)
551 /*****************************************************************************
552 * Instruction storage interrupt
553 ****************************************************************************/
554 INTERRUPT(int_instr_storage)
555 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
556 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ISI)
560 /*****************************************************************************
561 * External input interrupt
562 ****************************************************************************/
563 INTERRUPT(int_external_input)
564 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
565 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_EXI)
569 INTERRUPT(int_alignment)
570 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
571 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ALI)
575 INTERRUPT(int_program)
576 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
577 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_PGM)
582 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
583 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FPU)
587 /*****************************************************************************
589 ****************************************************************************/
590 INTERRUPT(int_syscall)
591 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
592 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SC)
596 /*****************************************************************************
597 * Decrementer interrupt
598 ****************************************************************************/
599 INTERRUPT(int_decrementer)
600 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
601 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_DECR)
605 /*****************************************************************************
606 * Fixed interval timer
607 ****************************************************************************/
608 INTERRUPT(int_fixed_interval_timer)
609 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
610 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FIT)
614 /*****************************************************************************
616 ****************************************************************************/
617 INTERRUPT(int_watchdog)
618 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
619 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_WDOG)
623 /*****************************************************************************
624 * Altivec Unavailable interrupt
625 ****************************************************************************/
627 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
628 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VEC)
632 /*****************************************************************************
633 * Altivec Assist interrupt
634 ****************************************************************************/
635 INTERRUPT(int_vecast)
636 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
637 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VECAST_E)
642 /*****************************************************************************
643 * Floating point Assist interrupt
644 ****************************************************************************/
645 INTERRUPT(int_spe_fpdata)
646 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
647 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SPFPD)
648 addi %r3, %r1, CALLSIZE
650 FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
653 INTERRUPT(int_spe_fpround)
654 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
655 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SPFPR)
656 addi %r3, %r1, CALLSIZE
657 bl spe_handle_fpround
658 FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
664 /*****************************************************************************
666 ****************************************************************************/
667 INTERRUPT(int_performance_counter)
668 STANDARD_PROLOG(SPR_SPRG3, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
669 FRAME_SETUP(SPR_SPRG3, PC_TEMPSAVE, EXC_PERF)
674 /*****************************************************************************
675 * Data TLB miss interrupt
677 * There can be nested TLB misses - while handling a TLB miss we reference
678 * data structures that may be not covered by translations. We support up to
679 * TLB_NESTED_MAX-1 nested misses.
690 * r20:r23 - scratch registers
691 ****************************************************************************/
692 INTERRUPT(int_data_tlb_error)
699 * Save MAS0-MAS2 registers. There might be another tlb miss during
700 * pte lookup overwriting current contents (which was hw filled).
706 /* Check faulting address. */
707 LOAD_ADDR(%r21, VM_MAXUSER_ADDRESS)
711 /* If it's kernel address, allow only supervisor mode misses. */
714 bt 17, search_failed /* check MSR[PR] */
718 cmpldi cr0, %r21, VM_MIN_KERNEL_ADDRESS@highest
720 lis %r21, VM_MIN_KERNEL_ADDRESS@h
721 cmplw cr0, %r31, %r21
726 /* Load r26 with kernel_pmap address */
729 .llong kernel_pmap_store-.
731 .long kernel_pmap_store-.
735 add %r26, %r21, %r26 /* kernel_pmap_store in r26 */
737 /* Force kernel tid, set TID to 0 in MAS1. */
739 rlwimi %r28, %r21, 0, 8, 15 /* clear TID bits */
742 /* This may result in nested tlb miss. */
743 bl pte_lookup /* returns PTE address in R25 */
745 CMPI %r25, 0 /* pte found? */
748 /* Finish up, write TLB entry. */
757 /* Load r26 with current user space process pmap */
759 LOAD %r26, PC_CURPMAP(%r26)
765 * Whenever we don't find a TLB mapping in PT, set a TLB0 entry with
766 * the faulting virtual address anyway, but put a fake RPN and no
767 * access rights. This should cause a following {D,I}SI exception.
769 lis %r23, 0xffff0000@h /* revoke all permissions */
771 /* Load MAS registers. */
786 /*****************************************************************************
788 * Return pte address that corresponds to given pmap/va. If there is no valid
793 * output: r25 - pte address
795 * scratch regs used: r21
797 ****************************************************************************/
800 beq 1f /* fail quickly if pmap is invalid */
803 rldicl %r21, %r31, (64 - PG_ROOT_L), (64 - PG_ROOT_NUM) /* pp2d offset */
804 slwi %r21, %r21, PG_ROOT_ENTRY_SHIFT /* multiply by pp2d entry size */
805 ld %r25, PM_ROOT(%r26) /* pmap pm_pp2d[] address */
806 ldx %r25, %r25, %r21 /* get pdir address, i.e. pmap->pm_pp2d[pp2d_idx] * */
811 rldicl %r21, %r31, (64 - PDIR_L1_L), (64 - PDIR_L1_NUM) /* pp2d offset */
812 slwi %r21, %r21, PDIR_L1_ENTRY_SHIFT /* multiply by pp2d entry size */
813 ldx %r25, %r25, %r21 /* get pdir address, i.e. pmap->pm_pp2d[pp2d_idx] * */
818 rldicl %r21, %r31, (64 - PDIR_L), (64 - PDIR_NUM) /* pdir offset */
819 slwi %r21, %r21, PDIR_ENTRY_SHIFT /* multiply by pdir entry size */
820 ldx %r25, %r25, %r21 /* get ptbl address, i.e. pmap->pm_pp2d[pp2d_idx][pdir_idx] */
825 rldicl %r21, %r31, (64 - PTBL_L), (64 - PTBL_NUM) /* ptbl offset */
826 slwi %r21, %r21, PTBL_ENTRY_SHIFT /* multiply by pte entry size */
829 srwi %r21, %r31, PDIR_SHIFT /* pdir offset */
830 slwi %r21, %r21, PDIR_ENTRY_SHIFT /* multiply by pdir entry size */
832 lwz %r25, PM_PDIR(%r26) /* pmap pm_dir[] address */
834 * Get ptbl address, i.e. pmap->pm_pdir[pdir_idx]
835 * This load may cause a Data TLB miss for non-kernel pmap!
837 lwzx %r25, %r25, %r21 /* offset within pm_pdir[] table */
841 lis %r21, PTBL_MASK@h
842 ori %r21, %r21, PTBL_MASK@l
845 /* ptbl offset, multiply by ptbl entry size */
846 srwi %r21, %r21, (PTBL_SHIFT - PTBL_ENTRY_SHIFT)
849 add %r25, %r25, %r21 /* address of pte entry */
852 * This load may cause a Data TLB miss for non-kernel pmap!
854 lwz %r21, PTE_FLAGS(%r25)
855 andi. %r21, %r21, PTE_VALID@l
862 /*****************************************************************************
864 * Load MAS1-MAS3 registers with data, write TLB entry
874 * scratch regs: r21-r23
876 ****************************************************************************/
879 * Update PTE flags: we have to do it atomically, as pmap_protect()
880 * running on other CPUs could attempt to update the flags at the same
885 lwarx %r21, %r23, %r25 /* get pte->flags */
886 oris %r21, %r21, PTE_REFERENCED@h /* set referenced bit */
888 andi. %r22, %r21, (PTE_SW | PTE_UW)@l /* check if writable */
890 ori %r21, %r21, PTE_MODIFIED@l /* set modified bit */
892 stwcx. %r21, %r23, %r25 /* write it back */
896 rlwimi %r27, %r21, 13, 27, 30 /* insert WIMG bits from pte */
898 /* Setup MAS3 value in r23. */
899 LOAD %r23, PTE_RPN(%r25) /* get pte->rpn */
901 rldicr %r22, %r23, 52, 51 /* extract MAS3 portion of RPN */
902 rldicl %r23, %r23, 20, 54 /* extract MAS7 portion of RPN */
904 rlwimi %r22, %r21, 30, 26, 31 /* insert protection bits from pte */
906 rlwinm %r22, %r23, 20, 0, 11 /* extract MAS3 portion of RPN */
908 rlwimi %r22, %r21, 30, 26, 31 /* insert protection bits from pte */
909 rlwimi %r22, %r21, 20, 12, 19 /* insert lower 8 RPN bits to MAS3 */
910 rlwinm %r23, %r23, 20, 24, 31 /* MAS7 portion of RPN */
913 /* Load MAS registers. */
926 /*****************************************************************************
927 * Instruction TLB miss interrupt
929 * Same notes as for the Data TLB miss
930 ****************************************************************************/
931 INTERRUPT(int_inst_tlb_error)
935 mfsrr0 %r31 /* faulting address */
938 * Save MAS0-MAS2 registers. There might be another tlb miss during pte
939 * lookup overwriting current contents (which was hw filled).
949 bt 17, search_user_pmap
953 .globl interrupt_vector_top
954 interrupt_vector_top:
956 /*****************************************************************************
958 ****************************************************************************/
960 STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1)
961 FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG)
963 FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1)
966 INTERRUPT(int_debug_ed)
967 STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_DSRR0, SPR_DSRR1)
968 FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG)
970 FRAME_LEAVE(SPR_DSRR0, SPR_DSRR1)
972 /* .long 0x4c00004e */
974 /* Internal helper for debug interrupt handling. */
975 /* Common code between e500v1/v2 and e500mc-based cores. */
979 LOAD %r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0)(%r3)
981 ADDR(interrupt_vector_base-.)
982 ADDR(interrupt_vector_top-.)
984 LOAD %r4,0(%r5) /* interrupt_vector_base in r4 */
988 LOAD %r4,WORD_SIZE(%r5) /* interrupt_vector_top in r4 */
993 /* Disable single-stepping for the interrupt handlers. */
994 LOAD %r3, FRAME_SRR1+CALLSIZE(%r1);
995 rlwinm %r3, %r3, 0, 23, 21
996 STORE %r3, FRAME_SRR1+CALLSIZE(%r1);
997 /* Restore srr0 and srr1 as they could have been clobbered. */
999 LOAD %r3, (PC_BOOKE_CRITSAVE+BOOKE_CRITSAVE_SRR0)(%r4);
1001 LOAD %r4, (PC_BOOKE_CRITSAVE+BOOKE_CRITSAVE_SRR1)(%r4);
1006 /*****************************************************************************
1008 ****************************************************************************/
1010 /* Call C trap dispatcher */
1012 addi %r3, %r1, CALLSIZE
1013 bl CNAME(powerpc_interrupt)
1016 .globl CNAME(trapexit) /* exported for db_backtrace use */
1018 /* disable interrupts */
1021 /* Test AST pending - makes sense for user process only */
1022 LOAD %r5, FRAME_SRR1+CALLSIZE(%r1)
1027 LOAD %r4, PC_CURTHREAD(%r3)
1028 lwz %r4, TD_AST(%r4)
1032 /* re-enable interrupts before calling ast() */
1035 addi %r3, %r1, CALLSIZE
1038 .globl CNAME(asttrapexit) /* db_backtrace code sentinel #2 */
1040 b trapexit /* test ast ret value ? */
1042 FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
1048 * Deliberate entry to dbtrap
1050 /* .globl CNAME(breakpoint)*/
1051 ASENTRY_NOPROF(breakpoint)
1055 li %r4, ~(PSL_EE | PSL_ME)@l
1056 oris %r4, %r4, ~(PSL_EE | PSL_ME)@h
1058 mtmsr %r3 /* disable interrupts */
1061 STORE %r30, (PC_DBSAVE+CPUSAVE_R30)(%r3)
1062 STORE %r31, (PC_DBSAVE+CPUSAVE_R31)(%r3)
1067 mfspr %r30, SPR_DEAR
1069 STORE %r30, (PC_DBSAVE+CPUSAVE_BOOKE_DEAR)(%r3)
1070 STORE %r31, (PC_DBSAVE+CPUSAVE_BOOKE_ESR)(%r3)
1074 STORE %r30, (PC_DBSAVE+CPUSAVE_SRR0)(%r3)
1075 STORE %r31, (PC_DBSAVE+CPUSAVE_SRR1)(%r3)
1081 * Now the kdb trap catching code.
1084 FRAME_SETUP(SPR_SPRG1, PC_DBSAVE, EXC_DEBUG)
1085 /* Call C trap code: */
1087 addi %r3, %r1, CALLSIZE
1088 bl CNAME(db_trap_glue)
1092 /* This wasn't for KDB, so switch to real trap: */
1096 FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
1104 LOAD %r5, PC_CURTHREAD(%r5)
1105 1: LOADX %r4, 0, %r3
1106 CMPI %r4, TLB_UNLOCKED
1118 li %r4, TLB_UNLOCKED
1126 * TLB miss spin locks. For each CPU we have a reservation granule (32 bytes);
1127 * only a single word from this granule will actually be used as a spin lock
1128 * for mutual exclusion between TLB miss handler and pmap layer that
1129 * manipulates page table contents.
1133 GLOBAL(tlb0_miss_locks)
1134 .space RES_GRANULE * MAXCPU