sys/powerpc/booke/trap_subr.S

   1 /*-
   2  * Copyright (C) 2006-2009 Semihalf, Rafal Jaworowski <raj@semihalf.com>
   3  * Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com>
   4  * Copyright (C) 2006 Juniper Networks, Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  * 3. The name of the author may not be used to endorse or promote products
  16  *    derived from this software without specific prior written permission.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  19  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  20  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN
  21  * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
  23  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  24  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  25  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  26  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  27  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28  *
  29  * $FreeBSD$
  30  */
  31 /*-
  32  * Copyright (C) 1995, 1996 Wolfgang Solfrank.
  33  * Copyright (C) 1995, 1996 TooLs GmbH.
  34  * All rights reserved.
  35  *
  36  * Redistribution and use in source and binary forms, with or without
  37  * modification, are permitted provided that the following conditions
  38  * are met:
  39  * 1. Redistributions of source code must retain the above copyright
  40  *    notice, this list of conditions and the following disclaimer.
  41  * 2. Redistributions in binary form must reproduce the above copyright
  42  *    notice, this list of conditions and the following disclaimer in the
  43  *    documentation and/or other materials provided with the distribution.
  44  * 3. All advertising materials mentioning features or use of this software
  45  *    must display the following acknowledgement:
  46  *      This product includes software developed by TooLs GmbH.
  47  * 4. The name of TooLs GmbH may not be used to endorse or promote products
  48  *    derived from this software without specific prior written permission.
  49  *
  50  * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
  51  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  52  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  53  * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  54  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  55  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  56  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  57  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  58  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  59  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  60  *
  61  *      from: $NetBSD: trap_subr.S,v 1.20 2002/04/22 23:20:08 kleink Exp $
  62  */
  63
  64 /*
  65  * NOTICE: This is not a standalone file.  to use it, #include it in
  66  * your port's locore.S, like so:
  67  *
  68  *      #include <powerpc/booke/trap_subr.S>
  69  */
  70
  71 /*
  72  * SPRG usage notes
  73  *
  74  * SPRG0 - pcpu pointer
  75  * SPRG1 - all interrupts except TLB miss, critical, machine check
  76  * SPRG2 - critical
  77  * SPRG3 - machine check
  78  * SPRG4-6 - scratch
  79  *
  80  */
  81
  82 /* Get the per-CPU data structure */
  83 #define GET_CPUINFO(r) mfsprg0 r
  84
  85 #define RES_GRANULE     64
  86 #define RES_LOCK        0       /* offset to the 'lock' word */
  87 #ifdef __powerpc64__
  88 #define RES_RECURSE     8       /* offset to the 'recurse' word */
  89 #else
  90 #define RES_RECURSE     4       /* offset to the 'recurse' word */
  91 #endif
  92
  93 /*
  94  * Standard interrupt prolog
  95  *
  96  * sprg_sp - SPRG{1-3} reg used to temporarily store the SP
  97  * savearea - temp save area (pc_{tempsave, disisave, critsave, mchksave})
  98  * isrr0-1 - save restore registers with CPU state at interrupt time (may be
  99  *           SRR0-1, CSRR0-1, MCSRR0-1
 100  *
 101  * 1. saves in the given savearea:
 102  *   - R30-31
 103  *   - DEAR, ESR
 104  *   - xSRR0-1
 105  *
 106  * 2. saves CR -> R30
 107  *
 108  * 3. switches to kstack if needed
 109  *
 110  * 4. notes:
 111  *   - R31 can be used as scratch register until a new frame is layed on
 112  *     the stack with FRAME_SETUP
 113  *
 114  *   - potential TLB miss: NO. Saveareas are always acessible via TLB1
 115  *     permanent entries, and within this prolog we do not dereference any
 116  *     locations potentially not in the TLB
 117  */
 118 #define STANDARD_PROLOG(sprg_sp, savearea, isrr0, isrr1)                \
 119         mtspr   sprg_sp, %r1;           /* Save SP */                   \
 120         GET_CPUINFO(%r1);               /* Per-cpu structure */         \
 121         STORE   %r30, (savearea+CPUSAVE_R30)(%r1);                      \
 122         STORE   %r31, (savearea+CPUSAVE_R31)(%r1);                      \
 123         mfdear  %r30;                                                   \
 124         mfesr   %r31;                                                   \
 125         STORE   %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1);               \
 126         STORE   %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1);                \
 127         mfspr   %r30, isrr0;                                            \
 128         mfspr   %r31, isrr1;            /* MSR at interrupt time */     \
 129         STORE   %r30, (savearea+CPUSAVE_SRR0)(%r1);                     \
 130         STORE   %r31, (savearea+CPUSAVE_SRR1)(%r1);                     \
 131         isync;                                                          \
 132         mfspr   %r1, sprg_sp;           /* Restore SP */                \
 133         mfcr    %r30;                   /* Save CR */                   \
 134         /* switch to per-thread kstack if intr taken in user mode */    \
 135         mtcr    %r31;                   /* MSR at interrupt time  */    \
 136         bf      17, 1f;                                                 \
 137         GET_CPUINFO(%r1);               /* Per-cpu structure */         \
 138         LOAD    %r1, PC_CURPCB(%r1);    /* Per-thread kernel stack */   \
 139 1:
 140
 141 #define STANDARD_CRIT_PROLOG(sprg_sp, savearea, isrr0, isrr1)           \
 142         mtspr   sprg_sp, %r1;           /* Save SP */                   \
 143         GET_CPUINFO(%r1);               /* Per-cpu structure */         \
 144         STORE   %r30, (savearea+CPUSAVE_R30)(%r1);                      \
 145         STORE   %r31, (savearea+CPUSAVE_R31)(%r1);                      \
 146         mfdear  %r30;                                                   \
 147         mfesr   %r31;                                                   \
 148         STORE   %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1);               \
 149         STORE   %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1);                \
 150         mfspr   %r30, isrr0;                                            \
 151         mfspr   %r31, isrr1;            /* MSR at interrupt time */     \
 152         STORE   %r30, (savearea+CPUSAVE_SRR0)(%r1);                     \
 153         STORE   %r31, (savearea+CPUSAVE_SRR1)(%r1);                     \
 154         mfspr   %r30, SPR_SRR0;                                         \
 155         mfspr   %r31, SPR_SRR1;         /* MSR at interrupt time */     \
 156         STORE   %r30, (savearea+BOOKE_CRITSAVE_SRR0)(%r1);              \
 157         STORE   %r31, (savearea+BOOKE_CRITSAVE_SRR1)(%r1);              \
 158         isync;                                                          \
 159         mfspr   %r1, sprg_sp;           /* Restore SP */                \
 160         mfcr    %r30;                   /* Save CR */                   \
 161         /* switch to per-thread kstack if intr taken in user mode */    \
 162         mtcr    %r31;                   /* MSR at interrupt time  */    \
 163         bf      17, 1f;                                                 \
 164         GET_CPUINFO(%r1);               /* Per-cpu structure */         \
 165         LOAD    %r1, PC_CURPCB(%r1);    /* Per-thread kernel stack */   \
 166 1:
 167
 168 /*
 169  * FRAME_SETUP assumes:
 170  *      SPRG{1-3}       SP at the time interrupt occured
 171  *      savearea        r30-r31, DEAR, ESR, xSRR0-1
 172  *      r30             CR
 173  *      r31             scratch
 174  *      r1              kernel stack
 175  *
 176  * sprg_sp - SPRG reg containing SP at the time interrupt occured
 177  * savearea - temp save
 178  * exc - exception number (EXC_xxx)
 179  *
 180  * 1. sets a new frame
 181  * 2. saves in the frame:
 182  *   - R0, R1 (SP at the time of interrupt), R2, LR, CR
 183  *   - R3-31 (R30-31 first restored from savearea)
 184  *   - XER, CTR, DEAR, ESR (from savearea), xSRR0-1
 185  *
 186  * Notes:
 187  * - potential TLB miss: YES, since we make dereferences to kstack, which
 188  *   can happen not covered (we can have up to two DTLB misses if fortunate
 189  *   enough i.e. when kstack crosses page boundary and both pages are
 190  *   untranslated)
 191  */
 192 #ifdef __powerpc64__
 193 #define SAVE_REGS(r)                                                    \
 194         std     %r3, FRAME_3+CALLSIZE(r);                               \
 195         std     %r4, FRAME_4+CALLSIZE(r);                               \
 196         std     %r5, FRAME_5+CALLSIZE(r);                               \
 197         std     %r6, FRAME_6+CALLSIZE(r);                               \
 198         std     %r7, FRAME_7+CALLSIZE(r);                               \
 199         std     %r8, FRAME_8+CALLSIZE(r);                               \
 200         std     %r9, FRAME_9+CALLSIZE(r);                               \
 201         std     %r10, FRAME_10+CALLSIZE(r);                             \
 202         std     %r11, FRAME_11+CALLSIZE(r);                             \
 203         std     %r12, FRAME_12+CALLSIZE(r);                             \
 204         std     %r13, FRAME_13+CALLSIZE(r);                             \
 205         std     %r14, FRAME_14+CALLSIZE(r);                             \
 206         std     %r15, FRAME_15+CALLSIZE(r);                             \
 207         std     %r16, FRAME_16+CALLSIZE(r);                             \
 208         std     %r17, FRAME_17+CALLSIZE(r);                             \
 209         std     %r18, FRAME_18+CALLSIZE(r);                             \
 210         std     %r19, FRAME_19+CALLSIZE(r);                             \
 211         std     %r20, FRAME_20+CALLSIZE(r);                             \
 212         std     %r21, FRAME_21+CALLSIZE(r);                             \
 213         std     %r22, FRAME_22+CALLSIZE(r);                             \
 214         std     %r23, FRAME_23+CALLSIZE(r);                             \
 215         std     %r24, FRAME_24+CALLSIZE(r);                             \
 216         std     %r25, FRAME_25+CALLSIZE(r);                             \
 217         std     %r26, FRAME_26+CALLSIZE(r);                             \
 218         std     %r27, FRAME_27+CALLSIZE(r);                             \
 219         std     %r28, FRAME_28+CALLSIZE(r);                             \
 220         std     %r29, FRAME_29+CALLSIZE(r);                             \
 221         std     %r30, FRAME_30+CALLSIZE(r);                             \
 222         std     %r31, FRAME_31+CALLSIZE(r)
 223 #define LD_REGS(r)                                                      \
 224         ld      %r3, FRAME_3+CALLSIZE(r);                               \
 225         ld      %r4, FRAME_4+CALLSIZE(r);                               \
 226         ld      %r5, FRAME_5+CALLSIZE(r);                               \
 227         ld      %r6, FRAME_6+CALLSIZE(r);                               \
 228         ld      %r7, FRAME_7+CALLSIZE(r);                               \
 229         ld      %r8, FRAME_8+CALLSIZE(r);                               \
 230         ld      %r9, FRAME_9+CALLSIZE(r);                               \
 231         ld      %r10, FRAME_10+CALLSIZE(r);                             \
 232         ld      %r11, FRAME_11+CALLSIZE(r);                             \
 233         ld      %r12, FRAME_12+CALLSIZE(r);                             \
 234         ld      %r13, FRAME_13+CALLSIZE(r);                             \
 235         ld      %r14, FRAME_14+CALLSIZE(r);                             \
 236         ld      %r15, FRAME_15+CALLSIZE(r);                             \
 237         ld      %r16, FRAME_16+CALLSIZE(r);                             \
 238         ld      %r17, FRAME_17+CALLSIZE(r);                             \
 239         ld      %r18, FRAME_18+CALLSIZE(r);                             \
 240         ld      %r19, FRAME_19+CALLSIZE(r);                             \
 241         ld      %r20, FRAME_20+CALLSIZE(r);                             \
 242         ld      %r21, FRAME_21+CALLSIZE(r);                             \
 243         ld      %r22, FRAME_22+CALLSIZE(r);                             \
 244         ld      %r23, FRAME_23+CALLSIZE(r);                             \
 245         ld      %r24, FRAME_24+CALLSIZE(r);                             \
 246         ld      %r25, FRAME_25+CALLSIZE(r);                             \
 247         ld      %r26, FRAME_26+CALLSIZE(r);                             \
 248         ld      %r27, FRAME_27+CALLSIZE(r);                             \
 249         ld      %r28, FRAME_28+CALLSIZE(r);                             \
 250         ld      %r29, FRAME_29+CALLSIZE(r);                             \
 251         ld      %r30, FRAME_30+CALLSIZE(r);                             \
 252         ld      %r31, FRAME_31+CALLSIZE(r)
 253 #else
 254 #define SAVE_REGS(r)                                                    \
 255         stmw    %r3,  FRAME_3+CALLSIZE(r)
 256 #define LD_REGS(r)                                                      \
 257         lmw     %r3,  FRAME_3+CALLSIZE(r)
 258 #endif
 259 #define FRAME_SETUP(sprg_sp, savearea, exc)                             \
 260         mfspr   %r31, sprg_sp;          /* get saved SP */              \
 261         /* establish a new stack frame and put everything on it */      \
 262         STU     %r31, -(FRAMELEN+REDZONE)(%r1);                         \
 263         STORE   %r0, FRAME_0+CALLSIZE(%r1);     /* save r0 in the trapframe */  \
 264         STORE   %r31, FRAME_1+CALLSIZE(%r1);    /* save SP   "     " */ \
 265         STORE   %r2, FRAME_2+CALLSIZE(%r1);     /* save r2   "     " */ \
 266         mflr    %r31;                                                   \
 267         STORE   %r31, FRAME_LR+CALLSIZE(%r1);   /* save LR   "     " */ \
 268         STORE   %r30, FRAME_CR+CALLSIZE(%r1);   /* save CR   "     " */ \
 269         GET_CPUINFO(%r2);                                               \
 270         LOAD    %r30, (savearea+CPUSAVE_R30)(%r2); /* get saved r30 */  \
 271         LOAD    %r31, (savearea+CPUSAVE_R31)(%r2); /* get saved r31 */  \
 272         /* save R3-31 */                                                \
 273         SAVE_REGS(%r1);                                                 \
 274         /* save DEAR, ESR */                                            \
 275         LOAD    %r28, (savearea+CPUSAVE_BOOKE_DEAR)(%r2);               \
 276         LOAD    %r29, (savearea+CPUSAVE_BOOKE_ESR)(%r2);                \
 277         STORE   %r28, FRAME_BOOKE_DEAR+CALLSIZE(%r1);                   \
 278         STORE   %r29, FRAME_BOOKE_ESR+CALLSIZE(%r1);                    \
 279         /* save XER, CTR, exc number */                                 \
 280         mfxer   %r3;                                                    \
 281         mfctr   %r4;                                                    \
 282         STORE   %r3, FRAME_XER+CALLSIZE(%r1);                           \
 283         STORE   %r4, FRAME_CTR+CALLSIZE(%r1);                           \
 284         li      %r5, exc;                                               \
 285         STORE   %r5, FRAME_EXC+CALLSIZE(%r1);                           \
 286         /* save DBCR0 */                                                \
 287         mfspr   %r3, SPR_DBCR0;                                         \
 288         STORE   %r3, FRAME_BOOKE_DBCR0+CALLSIZE(%r1);                   \
 289         /* save xSSR0-1 */                                              \
 290         LOAD    %r30, (savearea+CPUSAVE_SRR0)(%r2);                     \
 291         LOAD    %r31, (savearea+CPUSAVE_SRR1)(%r2);                     \
 292         STORE   %r30, FRAME_SRR0+CALLSIZE(%r1);                         \
 293         STORE   %r31, FRAME_SRR1+CALLSIZE(%r1);                         \
 294         LOAD    THREAD_REG, PC_CURTHREAD(%r2);                          \
 295
 296 /*
 297  *
 298  * isrr0-1 - save restore registers to restore CPU state to (may be
 299  *           SRR0-1, CSRR0-1, MCSRR0-1
 300  *
 301  * Notes:
 302  *  - potential TLB miss: YES. The deref'd kstack may be not covered
 303  */
 304 #define FRAME_LEAVE(isrr0, isrr1)                                       \
 305         wrteei 0;                                                       \
 306         /* restore CTR, XER, LR, CR */                                  \
 307         LOAD    %r4, FRAME_CTR+CALLSIZE(%r1);                           \
 308         LOAD    %r5, FRAME_XER+CALLSIZE(%r1);                           \
 309         LOAD    %r6, FRAME_LR+CALLSIZE(%r1);                            \
 310         LOAD    %r7, FRAME_CR+CALLSIZE(%r1);                            \
 311         mtctr   %r4;                                                    \
 312         mtxer   %r5;                                                    \
 313         mtlr    %r6;                                                    \
 314         mtcr    %r7;                                                    \
 315         /* restore DBCR0 */                                             \
 316         LOAD    %r4, FRAME_BOOKE_DBCR0+CALLSIZE(%r1);                   \
 317         mtspr   SPR_DBCR0, %r4;                                         \
 318         /* restore xSRR0-1 */                                           \
 319         LOAD    %r30, FRAME_SRR0+CALLSIZE(%r1);                         \
 320         LOAD    %r31, FRAME_SRR1+CALLSIZE(%r1);                         \
 321         mtspr   isrr0, %r30;                                            \
 322         mtspr   isrr1, %r31;                                            \
 323         /* restore R2-31, SP */                                         \
 324         LD_REGS(%r1);                                                   \
 325         LOAD    %r2, FRAME_2+CALLSIZE(%r1);                             \
 326         LOAD    %r0, FRAME_0+CALLSIZE(%r1);                             \
 327         LOAD    %r1, FRAME_1+CALLSIZE(%r1);                             \
 328         isync
 329
 330 /*
 331  * TLB miss prolog
 332  *
 333  * saves LR, CR, SRR0-1, R20-31 in the TLBSAVE area
 334  *
 335  * Notes:
 336  *  - potential TLB miss: NO. It is crucial that we do not generate a TLB
 337  *    miss within the TLB prolog itself!
 338  *  - TLBSAVE is always translated
 339  */
 340 #ifdef __powerpc64__
 341 #define TLB_SAVE_REGS(br)                                               \
 342         std     %r20, (TLBSAVE_BOOKE_R20)(br);                          \
 343         std     %r21, (TLBSAVE_BOOKE_R21)(br);                          \
 344         std     %r22, (TLBSAVE_BOOKE_R22)(br);                          \
 345         std     %r23, (TLBSAVE_BOOKE_R23)(br);                          \
 346         std     %r24, (TLBSAVE_BOOKE_R24)(br);                          \
 347         std     %r25, (TLBSAVE_BOOKE_R25)(br);                          \
 348         std     %r26, (TLBSAVE_BOOKE_R26)(br);                          \
 349         std     %r27, (TLBSAVE_BOOKE_R27)(br);                          \
 350         std     %r28, (TLBSAVE_BOOKE_R28)(br);                          \
 351         std     %r29, (TLBSAVE_BOOKE_R29)(br);                          \
 352         std     %r30, (TLBSAVE_BOOKE_R30)(br);                          \
 353         std     %r31, (TLBSAVE_BOOKE_R31)(br);
 354 #define TLB_RESTORE_REGS(br)                                            \
 355         ld      %r20, (TLBSAVE_BOOKE_R20)(br);                          \
 356         ld      %r21, (TLBSAVE_BOOKE_R21)(br);                          \
 357         ld      %r22, (TLBSAVE_BOOKE_R22)(br);                          \
 358         ld      %r23, (TLBSAVE_BOOKE_R23)(br);                          \
 359         ld      %r24, (TLBSAVE_BOOKE_R24)(br);                          \
 360         ld      %r25, (TLBSAVE_BOOKE_R25)(br);                          \
 361         ld      %r26, (TLBSAVE_BOOKE_R26)(br);                          \
 362         ld      %r27, (TLBSAVE_BOOKE_R27)(br);                          \
 363         ld      %r28, (TLBSAVE_BOOKE_R28)(br);                          \
 364         ld      %r29, (TLBSAVE_BOOKE_R29)(br);                          \
 365         ld      %r30, (TLBSAVE_BOOKE_R30)(br);                          \
 366         ld      %r31, (TLBSAVE_BOOKE_R31)(br);
 367 #define TLB_NEST(outr,inr)                                              \
 368         rlwinm  outr, inr, 7, 22, 24;   /* 8 x TLBSAVE_LEN */
 369 #else
 370 #define TLB_SAVE_REGS(br)                                               \
 371         stmw    %r20, TLBSAVE_BOOKE_R20(br)
 372 #define TLB_RESTORE_REGS(br)                                            \
 373         lmw     %r20, TLBSAVE_BOOKE_R20(br)
 374 #define TLB_NEST(outr,inr)                                              \
 375         rlwinm  outr, inr, 6, 23, 25;   /* 4 x TLBSAVE_LEN */
 376 #endif
 377 #define TLB_PROLOG                                                      \
 378         mtsprg4 %r1;                    /* Save SP */                   \
 379         mtsprg5 %r28;                                                   \
 380         mtsprg6 %r29;                                                   \
 381         /* calculate TLB nesting level and TLBSAVE instance address */  \
 382         GET_CPUINFO(%r1);               /* Per-cpu structure */         \
 383         LOAD    %r28, PC_BOOKE_TLB_LEVEL(%r1);                          \
 384         TLB_NEST(%r29,%r28);                                            \
 385         addi    %r28, %r28, 1;                                          \
 386         STORE   %r28, PC_BOOKE_TLB_LEVEL(%r1);                          \
 387         addi    %r29, %r29, PC_BOOKE_TLBSAVE@l;                         \
 388         add     %r1, %r1, %r29;         /* current TLBSAVE ptr */       \
 389                                                                         \
 390         /* save R20-31 */                                               \
 391         mfsprg5 %r28;                                                   \
 392         mfsprg6 %r29;                                                   \
 393         TLB_SAVE_REGS(%r1);                     \
 394         /* save LR, CR */                                               \
 395         mflr    %r30;                                                   \
 396         mfcr    %r31;                                                   \
 397         STORE   %r30, (TLBSAVE_BOOKE_LR)(%r1);                          \
 398         STORE   %r31, (TLBSAVE_BOOKE_CR)(%r1);                          \
 399         /* save SRR0-1 */                                               \
 400         mfsrr0  %r30;           /* execution addr at interrupt time */  \
 401         mfsrr1  %r31;           /* MSR at interrupt time*/              \
 402         STORE   %r30, (TLBSAVE_BOOKE_SRR0)(%r1);        /* save SRR0 */ \
 403         STORE   %r31, (TLBSAVE_BOOKE_SRR1)(%r1);        /* save SRR1 */ \
 404         isync;                                                          \
 405         mfsprg4 %r1
 406
 407 /*
 408  * restores LR, CR, SRR0-1, R20-31 from the TLBSAVE area
 409  *
 410  * same notes as for the TLB_PROLOG
 411  */
 412 #define TLB_RESTORE                                                     \
 413         mtsprg4 %r1;                    /* Save SP */                   \
 414         GET_CPUINFO(%r1);               /* Per-cpu structure */         \
 415         /* calculate TLB nesting level and TLBSAVE instance addr */     \
 416         LOAD    %r28, PC_BOOKE_TLB_LEVEL(%r1);                          \
 417         subi    %r28, %r28, 1;                                          \
 418         STORE   %r28, PC_BOOKE_TLB_LEVEL(%r1);                          \
 419         TLB_NEST(%r29,%r28);                                            \
 420         addi    %r29, %r29, PC_BOOKE_TLBSAVE@l;                         \
 421         add     %r1, %r1, %r29;                                         \
 422                                                                         \
 423         /* restore LR, CR */                                            \
 424         LOAD    %r30, (TLBSAVE_BOOKE_LR)(%r1);                          \
 425         LOAD    %r31, (TLBSAVE_BOOKE_CR)(%r1);                          \
 426         mtlr    %r30;                                                   \
 427         mtcr    %r31;                                                   \
 428         /* restore SRR0-1 */                                            \
 429         LOAD    %r30, (TLBSAVE_BOOKE_SRR0)(%r1);                        \
 430         LOAD    %r31, (TLBSAVE_BOOKE_SRR1)(%r1);                        \
 431         mtsrr0  %r30;                                                   \
 432         mtsrr1  %r31;                                                   \
 433         /* restore R20-31 */                                            \
 434         TLB_RESTORE_REGS(%r1);                                          \
 435         mfsprg4 %r1
 436
 437 #ifdef SMP
 438 #define TLB_LOCK                                                        \
 439         GET_CPUINFO(%r20);                                              \
 440         LOAD    %r21, PC_CURTHREAD(%r20);                               \
 441         LOAD    %r22, PC_BOOKE_TLB_LOCK(%r20);                          \
 442                                                                         \
 443 1:      LOADX   %r23, 0, %r22;                                          \
 444         CMPI    %r23, TLB_UNLOCKED;                                     \
 445         beq     2f;                                                     \
 446                                                                         \
 447         /* check if this is recursion */                                \
 448         CMPL    cr0, %r21, %r23;                                        \
 449         bne-    1b;                                                     \
 450                                                                         \
 451 2:      /* try to acquire lock */                                       \
 452         STOREX  %r21, 0, %r22;                                          \
 453         bne-    1b;                                                     \
 454                                                                         \
 455         /* got it, update recursion counter */                          \
 456         lwz     %r21, RES_RECURSE(%r22);                                \
 457         addi    %r21, %r21, 1;                                          \
 458         stw     %r21, RES_RECURSE(%r22);                                \
 459         isync;                                                          \
 460         msync
 461
 462 #define TLB_UNLOCK                                                      \
 463         GET_CPUINFO(%r20);                                              \
 464         LOAD    %r21, PC_CURTHREAD(%r20);                               \
 465         LOAD    %r22, PC_BOOKE_TLB_LOCK(%r20);                          \
 466                                                                         \
 467         /* update recursion counter */                                  \
 468         lwz     %r23, RES_RECURSE(%r22);                                \
 469         subi    %r23, %r23, 1;                                          \
 470         stw     %r23, RES_RECURSE(%r22);                                \
 471                                                                         \
 472         cmplwi  %r23, 0;                                                \
 473         bne     1f;                                                     \
 474         isync;                                                          \
 475         msync;                                                          \
 476                                                                         \
 477         /* release the lock */                                          \
 478         li      %r23, TLB_UNLOCKED;                                     \
 479         STORE   %r23, 0(%r22);                                          \
 480 1:      isync;                                                          \
 481         msync
 482 #else
 483 #define TLB_LOCK
 484 #define TLB_UNLOCK
 485 #endif  /* SMP */
 486
 487 #define INTERRUPT(label)                                                \
 488         .globl  label;                                                  \
 489         .align  5;                                                      \
 490         CNAME(label):
 491
 492 /*
 493  * Interrupt handling routines in BookE can be flexibly placed and do not have
 494  * to live in pre-defined vectors location. Note they need to be TLB-mapped at
 495  * all times in order to be able to handle exceptions. We thus arrange for
 496  * them to be part of kernel text which is always TLB-accessible.
 497  *
 498  * The interrupt handling routines have to be 16 bytes aligned: we align them
 499  * to 32 bytes (cache line length) which supposedly performs better.
 500  *
 501  */
 502         .text
 503         .globl CNAME(interrupt_vector_base)
 504         .align 5
 505 interrupt_vector_base:
 506 /*****************************************************************************
 507  * Catch-all handler to handle uninstalled IVORs
 508  ****************************************************************************/
 509 INTERRUPT(int_unknown)
 510         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 511         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_RSVD)
 512         b       trap_common
 513
 514 /*****************************************************************************
 515  * Critical input interrupt
 516  ****************************************************************************/
 517 INTERRUPT(int_critical_input)
 518         STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1)
 519         FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_CRIT)
 520         GET_TOCBASE(%r2)
 521         addi    %r3, %r1, CALLSIZE
 522         bl      CNAME(powerpc_interrupt)
 523         TOC_RESTORE
 524         FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1)
 525         rfci
 526
 527
 528 /*****************************************************************************
 529  * Machine check interrupt
 530  ****************************************************************************/
 531 INTERRUPT(int_machine_check)
 532         STANDARD_PROLOG(SPR_SPRG3, PC_BOOKE_MCHKSAVE, SPR_MCSRR0, SPR_MCSRR1)
 533         FRAME_SETUP(SPR_SPRG3, PC_BOOKE_MCHKSAVE, EXC_MCHK)
 534         GET_TOCBASE(%r2)
 535         addi    %r3, %r1, CALLSIZE
 536         bl      CNAME(powerpc_interrupt)
 537         TOC_RESTORE
 538         FRAME_LEAVE(SPR_MCSRR0, SPR_MCSRR1)
 539         rfmci
 540
 541
 542 /*****************************************************************************
 543  * Data storage interrupt
 544  ****************************************************************************/
 545 INTERRUPT(int_data_storage)
 546         STANDARD_PROLOG(SPR_SPRG1, PC_DISISAVE, SPR_SRR0, SPR_SRR1)
 547         FRAME_SETUP(SPR_SPRG1, PC_DISISAVE, EXC_DSI)
 548         b       trap_common
 549
 550
 551 /*****************************************************************************
 552  * Instruction storage interrupt
 553  ****************************************************************************/
 554 INTERRUPT(int_instr_storage)
 555         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 556         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ISI)
 557         b       trap_common
 558
 559
 560 /*****************************************************************************
 561  * External input interrupt
 562  ****************************************************************************/
 563 INTERRUPT(int_external_input)
 564         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 565         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_EXI)
 566         b       trap_common
 567
 568
 569 INTERRUPT(int_alignment)
 570         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 571         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ALI)
 572         b       trap_common
 573
 574
 575 INTERRUPT(int_program)
 576         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 577         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_PGM)
 578         b       trap_common
 579
 580
 581 INTERRUPT(int_fpu)
 582         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 583         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FPU)
 584         b       trap_common
 585
 586
 587 /*****************************************************************************
 588  * System call
 589  ****************************************************************************/
 590 INTERRUPT(int_syscall)
 591         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 592         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SC)
 593         b       trap_common
 594
 595
 596 /*****************************************************************************
 597  * Decrementer interrupt
 598  ****************************************************************************/
 599 INTERRUPT(int_decrementer)
 600         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 601         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_DECR)
 602         b       trap_common
 603
 604
 605 /*****************************************************************************
 606  * Fixed interval timer
 607  ****************************************************************************/
 608 INTERRUPT(int_fixed_interval_timer)
 609         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 610         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FIT)
 611         b       trap_common
 612
 613
 614 /*****************************************************************************
 615  * Watchdog interrupt
 616  ****************************************************************************/
 617 INTERRUPT(int_watchdog)
 618         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 619         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_WDOG)
 620         b       trap_common
 621
 622
 623 /*****************************************************************************
 624  * Altivec Unavailable interrupt
 625  ****************************************************************************/
 626 INTERRUPT(int_vec)
 627         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 628         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VEC)
 629         b       trap_common
 630
 631
 632 /*****************************************************************************
 633  * Altivec Assist interrupt
 634  ****************************************************************************/
 635 INTERRUPT(int_vecast)
 636         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 637         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VECAST_E)
 638         b       trap_common
 639
 640
 641 #ifdef __SPE__
 642 /*****************************************************************************
 643  * Floating point Assist interrupt
 644  ****************************************************************************/
 645 INTERRUPT(int_spe_fpdata)
 646         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 647         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SPFPD)
 648         addi    %r3, %r1, CALLSIZE
 649         bl      spe_handle_fpdata
 650         FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
 651         rfi
 652
 653 INTERRUPT(int_spe_fpround)
 654         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 655         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SPFPR)
 656         addi    %r3, %r1, CALLSIZE
 657         bl      spe_handle_fpround
 658         FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
 659         rfi
 660 #endif
 661
 662
 663 #ifdef HWPMC_HOOKS
 664 /*****************************************************************************
 665  * PMC Interrupt
 666  ****************************************************************************/
 667 INTERRUPT(int_performance_counter)
 668         STANDARD_PROLOG(SPR_SPRG3, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 669         FRAME_SETUP(SPR_SPRG3, PC_TEMPSAVE, EXC_PERF)
 670         b       trap_common
 671 #endif
 672
 673
 674 /*****************************************************************************
 675  * Data TLB miss interrupt
 676  *
 677  * There can be nested TLB misses - while handling a TLB miss we reference
 678  * data structures that may be not covered by translations. We support up to
 679  * TLB_NESTED_MAX-1 nested misses.
 680  *
 681  * Registers use:
 682  *      r31 - dear
 683  *      r30 - unused
 684  *      r29 - saved mas0
 685  *      r28 - saved mas1
 686  *      r27 - saved mas2
 687  *      r26 - pmap address
 688  *      r25 - pte address
 689  *
 690  *      r20:r23 - scratch registers
 691  ****************************************************************************/
 692 INTERRUPT(int_data_tlb_error)
 693         TLB_PROLOG
 694         TLB_LOCK
 695
 696         mfdear  %r31
 697
 698         /*
 699          * Save MAS0-MAS2 registers. There might be another tlb miss during
 700          * pte lookup overwriting current contents (which was hw filled).
 701          */
 702         mfspr   %r29, SPR_MAS0
 703         mfspr   %r28, SPR_MAS1
 704         mfspr   %r27, SPR_MAS2
 705
 706         /* Check faulting address. */
 707         LOAD_ADDR(%r21, VM_MAXUSER_ADDRESS)
 708         CMPL    cr0, %r31, %r21
 709         blt     search_user_pmap
 710
 711         /* If it's kernel address, allow only supervisor mode misses. */
 712         mfsrr1  %r21
 713         mtcr    %r21
 714         bt      17, search_failed       /* check MSR[PR] */
 715
 716 search_kernel_pmap:
 717         /* Load r26 with kernel_pmap address */
 718         bl      1f
 719 #ifdef __powerpc64__
 720         .llong kernel_pmap_store-.
 721 #else
 722         .long kernel_pmap_store-.
 723 #endif
 724 1:      mflr    %r21
 725         LOAD    %r26, 0(%r21)
 726         add     %r26, %r21, %r26        /* kernel_pmap_store in r26 */
 727
 728         /* Force kernel tid, set TID to 0 in MAS1. */
 729         li      %r21, 0
 730         rlwimi  %r28, %r21, 0, 8, 15    /* clear TID bits */
 731
 732 tlb_miss_handle:
 733         /* This may result in nested tlb miss. */
 734         bl      pte_lookup              /* returns PTE address in R25 */
 735
 736         CMPI    %r25, 0                 /* pte found? */
 737         beq     search_failed
 738
 739         /* Finish up, write TLB entry. */
 740         bl      tlb_fill_entry
 741
 742 tlb_miss_return:
 743         TLB_UNLOCK
 744         TLB_RESTORE
 745         rfi
 746
 747 search_user_pmap:
 748         /* Load r26 with current user space process pmap */
 749         GET_CPUINFO(%r26)
 750         LOAD    %r26, PC_CURPMAP(%r26)
 751
 752         b       tlb_miss_handle
 753
 754 search_failed:
 755         /*
 756          * Whenever we don't find a TLB mapping in PT, set a TLB0 entry with
 757          * the faulting virtual address anyway, but put a fake RPN and no
 758          * access rights. This should cause a following {D,I}SI exception.
 759          */
 760         lis     %r23, 0xffff0000@h      /* revoke all permissions */
 761
 762         /* Load MAS registers. */
 763         mtspr   SPR_MAS0, %r29
 764         mtspr   SPR_MAS1, %r28
 765         mtspr   SPR_MAS2, %r27
 766         mtspr   SPR_MAS3, %r23
 767
 768         li      %r23, 0
 769         mtspr   SPR_MAS7, %r23
 770         bl      zero_mas8
 771
 772         isync
 773         tlbwe
 774         msync
 775         isync
 776         b       tlb_miss_return
 777
 778 /*****************************************************************************
 779  *
 780  * Return pte address that corresponds to given pmap/va.  If there is no valid
 781  * entry return 0.
 782  *
 783  * input: r26 - pmap
 784  * input: r31 - dear
 785  * output: r25 - pte address
 786  *
 787  * scratch regs used: r21
 788  *
 789  ****************************************************************************/
 790 pte_lookup:
 791         CMPI    %r26, 0
 792         beq     1f                      /* fail quickly if pmap is invalid */
 793
 794 #ifdef __powerpc64__
 795         rldicl  %r21, %r31, (64 - PP2D_L_L), (64 - PP2D_L_NUM) /* pp2d offset */
 796         rldicl  %r25, %r31, (64 - PP2D_H_L), (64 - PP2D_H_NUM)
 797         rldimi  %r21, %r25, PP2D_L_NUM, (64 - (PP2D_L_NUM + PP2D_H_NUM))
 798         slwi    %r21, %r21, PP2D_ENTRY_SHIFT    /* multiply by pp2d entry size */
 799         ld      %r25, PM_PP2D(%r26)             /* pmap pm_pp2d[] address */
 800         ldx     %r25, %r25, %r21                /* get pdir address, i.e.  pmap->pm_pp2d[pp2d_idx] * */
 801
 802         cmpdi   %r25, 0
 803         beq 2f
 804
 805         rldicl  %r21, %r31, (64 - PDIR_L), (64 - PDIR_NUM)      /* pdir offset */
 806         slwi    %r21, %r21, PDIR_ENTRY_SHIFT    /* multiply by pdir entry size */
 807         ldx     %r25, %r25, %r21                /* get ptbl address, i.e.  pmap->pm_pp2d[pp2d_idx][pdir_idx] */
 808
 809         cmpdi   %r25, 0
 810         beq     2f
 811
 812         rldicl  %r21, %r31, (64 - PTBL_L), (64 - PTBL_NUM) /* ptbl offset */
 813         slwi    %r21, %r21, PTBL_ENTRY_SHIFT   /* multiply by pte entry size */
 814
 815 #else
 816         srwi    %r21, %r31, PDIR_SHIFT          /* pdir offset */
 817         slwi    %r21, %r21, PDIR_ENTRY_SHIFT    /* multiply by pdir entry size */
 818
 819         lwz     %r25, PM_PDIR(%r26)     /* pmap pm_dir[] address */
 820         /*
 821          * Get ptbl address, i.e. pmap->pm_pdir[pdir_idx]
 822          * This load may cause a Data TLB miss for non-kernel pmap!
 823          */
 824         lwzx    %r25, %r25, %r21        /* offset within pm_pdir[] table */
 825         cmpwi   %r25, 0
 826         beq     2f
 827
 828         lis     %r21, PTBL_MASK@h
 829         ori     %r21, %r21, PTBL_MASK@l
 830         and     %r21, %r21, %r31
 831
 832         /* ptbl offset, multiply by ptbl entry size */
 833         srwi    %r21, %r21, (PTBL_SHIFT - PTBL_ENTRY_SHIFT)
 834 #endif
 835
 836         add     %r25, %r25, %r21                /* address of pte entry */
 837         /*
 838          * Get pte->flags
 839          * This load may cause a Data TLB miss for non-kernel pmap!
 840          */
 841         lwz     %r21, PTE_FLAGS(%r25)
 842         andi.   %r21, %r21, PTE_VALID@l
 843         bne     2f
 844 1:
 845         li      %r25, 0
 846 2:
 847         blr
 848
 849 /*****************************************************************************
 850  *
 851  * Load MAS1-MAS3 registers with data, write TLB entry
 852  *
 853  * input:
 854  * r29 - mas0
 855  * r28 - mas1
 856  * r27 - mas2
 857  * r25 - pte
 858  *
 859  * output: none
 860  *
 861  * scratch regs: r21-r23
 862  *
 863  ****************************************************************************/
 864 tlb_fill_entry:
 865         /*
 866          * Update PTE flags: we have to do it atomically, as pmap_protect()
 867          * running on other CPUs could attempt to update the flags at the same
 868          * time.
 869          */
 870         li      %r23, PTE_FLAGS
 871 1:
 872         lwarx   %r21, %r23, %r25                /* get pte->flags */
 873         oris    %r21, %r21, PTE_REFERENCED@h    /* set referenced bit */
 874
 875         andi.   %r22, %r21, (PTE_SW | PTE_UW)@l /* check if writable */
 876         beq     2f
 877         ori     %r21, %r21, PTE_MODIFIED@l      /* set modified bit */
 878 2:
 879         stwcx.  %r21, %r23, %r25                /* write it back */
 880         bne-    1b
 881
 882         /* Update MAS2. */
 883         rlwimi  %r27, %r21, 13, 27, 30          /* insert WIMG bits from pte */
 884
 885         /* Setup MAS3 value in r23. */
 886         LOAD    %r23, PTE_RPN(%r25)             /* get pte->rpn */
 887 #ifdef __powerpc64__
 888         rldicr  %r22, %r23, 52, 51              /* extract MAS3 portion of RPN */
 889         rldicl  %r23, %r23, 20, 54              /* extract MAS7 portion of RPN */
 890
 891         rlwimi  %r22, %r21, 30, 26, 31          /* insert protection bits from pte */
 892 #else
 893         rlwinm  %r22, %r23, 20, 0, 11           /* extract MAS3 portion of RPN */
 894
 895         rlwimi  %r22, %r21, 30, 26, 31          /* insert protection bits from pte */
 896         rlwimi  %r22, %r21, 20, 12, 19          /* insert lower 8 RPN bits to MAS3 */
 897         rlwinm  %r23, %r23, 20, 24, 31          /* MAS7 portion of RPN */
 898 #endif
 899
 900         /* Load MAS registers. */
 901         mtspr   SPR_MAS0, %r29
 902         mtspr   SPR_MAS1, %r28
 903         mtspr   SPR_MAS2, %r27
 904         mtspr   SPR_MAS3, %r22
 905         mtspr   SPR_MAS7, %r23
 906
 907         mflr    %r21
 908         bl      zero_mas8
 909         mtlr    %r21
 910
 911         isync
 912         tlbwe
 913         isync
 914         msync
 915         blr
 916
 917 /*****************************************************************************
 918  * Instruction TLB miss interrupt
 919  *
 920  * Same notes as for the Data TLB miss
 921  ****************************************************************************/
 922 INTERRUPT(int_inst_tlb_error)
 923         TLB_PROLOG
 924         TLB_LOCK
 925
 926         mfsrr0  %r31                    /* faulting address */
 927
 928         /*
 929          * Save MAS0-MAS2 registers. There might be another tlb miss during pte
 930          * lookup overwriting current contents (which was hw filled).
 931          */
 932         mfspr   %r29, SPR_MAS0
 933         mfspr   %r28, SPR_MAS1
 934         mfspr   %r27, SPR_MAS2
 935
 936         mfsrr1  %r21
 937         mtcr    %r21
 938
 939         /* check MSR[PR] */
 940         bt      17, search_user_pmap
 941         b       search_kernel_pmap
 942
 943
 944         .globl  interrupt_vector_top
 945 interrupt_vector_top:
 946
 947 /*****************************************************************************
 948  * Debug interrupt
 949  ****************************************************************************/
 950 INTERRUPT(int_debug)
 951         STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1)
 952         FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG)
 953         bl      int_debug_int
 954         FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1)
 955         rfci
 956
 957 INTERRUPT(int_debug_ed)
 958         STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_DSRR0, SPR_DSRR1)
 959         FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG)
 960         bl      int_debug_int
 961         FRAME_LEAVE(SPR_DSRR0, SPR_DSRR1)
 962         rfdi
 963         /* .long 0x4c00004e */
 964
 965 /* Internal helper for debug interrupt handling. */
 966 /* Common code between e500v1/v2 and e500mc-based cores. */
 967 int_debug_int:
 968         mflr    %r14
 969         GET_CPUINFO(%r3)
 970         LOAD    %r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0)(%r3)
 971         bl      0f
 972         ADDR(interrupt_vector_base-.)
 973         ADDR(interrupt_vector_top-.)
 974 0:      mflr    %r5
 975         LOAD    %r4,0(%r5)      /* interrupt_vector_base in r4 */
 976         add     %r4,%r4,%r5
 977         CMPL    cr0, %r3, %r4
 978         blt     trap_common
 979         LOAD    %r4,WORD_SIZE(%r5)      /* interrupt_vector_top in r4 */
 980         add     %r4,%r4,%r5
 981         addi    %r4,%r4,4
 982         CMPL    cr0, %r3, %r4
 983         bge     trap_common
 984         /* Disable single-stepping for the interrupt handlers. */
 985         LOAD    %r3, FRAME_SRR1+CALLSIZE(%r1);
 986         rlwinm  %r3, %r3, 0, 23, 21
 987         STORE   %r3, FRAME_SRR1+CALLSIZE(%r1);
 988         /* Restore srr0 and srr1 as they could have been clobbered. */
 989         GET_CPUINFO(%r4)
 990         LOAD    %r3, (PC_BOOKE_CRITSAVE+BOOKE_CRITSAVE_SRR0)(%r4);
 991         mtspr   SPR_SRR0, %r3
 992         LOAD    %r4, (PC_BOOKE_CRITSAVE+BOOKE_CRITSAVE_SRR1)(%r4);
 993         mtspr   SPR_SRR1, %r4
 994         mtlr    %r14
 995         blr
 996
 997 /*****************************************************************************
 998  * Common trap code
 999  ****************************************************************************/
1000 trap_common:
1001         /* Call C trap dispatcher */
1002         GET_TOCBASE(%r2)
1003         addi    %r3, %r1, CALLSIZE
1004         bl      CNAME(powerpc_interrupt)
1005         TOC_RESTORE
1006
1007         .globl  CNAME(trapexit)         /* exported for db_backtrace use */
1008 CNAME(trapexit):
1009         /* disable interrupts */
1010         wrteei  0
1011
1012         /* Test AST pending - makes sense for user process only */
1013         LOAD    %r5, FRAME_SRR1+CALLSIZE(%r1)
1014         mtcr    %r5
1015         bf      17, 1f
1016
1017         GET_CPUINFO(%r3)
1018         LOAD    %r4, PC_CURTHREAD(%r3)
1019         lwz     %r4, TD_FLAGS(%r4)
1020         lis     %r5, (TDF_ASTPENDING | TDF_NEEDRESCHED)@h
1021         ori     %r5, %r5, (TDF_ASTPENDING | TDF_NEEDRESCHED)@l
1022         and.    %r4, %r4, %r5
1023         beq     1f
1024
1025         /* re-enable interrupts before calling ast() */
1026         wrteei  1
1027
1028         addi    %r3, %r1, CALLSIZE
1029         bl      CNAME(ast)
1030         TOC_RESTORE
1031         .globl  CNAME(asttrapexit)      /* db_backtrace code sentinel #2 */
1032 CNAME(asttrapexit):
1033         b       trapexit                /* test ast ret value ? */
1034 1:
1035         FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
1036         rfi
1037
1038
1039 #if defined(KDB)
1040 /*
1041  * Deliberate entry to dbtrap
1042  */
1043         /* .globl       CNAME(breakpoint)*/
1044 ASENTRY_NOPROF(breakpoint)
1045         mtsprg1 %r1
1046         mfmsr   %r3
1047         mtsrr1  %r3
1048         li      %r4, ~(PSL_EE | PSL_ME)@l
1049         oris    %r4, %r4, ~(PSL_EE | PSL_ME)@h
1050         and     %r3, %r3, %r4
1051         mtmsr   %r3                     /* disable interrupts */
1052         isync
1053         GET_CPUINFO(%r3)
1054         STORE   %r30, (PC_DBSAVE+CPUSAVE_R30)(%r3)
1055         STORE   %r31, (PC_DBSAVE+CPUSAVE_R31)(%r3)
1056
1057         mflr    %r31
1058         mtsrr0  %r31
1059
1060         mfdear  %r30
1061         mfesr   %r31
1062         STORE   %r30, (PC_DBSAVE+CPUSAVE_BOOKE_DEAR)(%r3)
1063         STORE   %r31, (PC_DBSAVE+CPUSAVE_BOOKE_ESR)(%r3)
1064
1065         mfsrr0  %r30
1066         mfsrr1  %r31
1067         STORE   %r30, (PC_DBSAVE+CPUSAVE_SRR0)(%r3)
1068         STORE   %r31, (PC_DBSAVE+CPUSAVE_SRR1)(%r3)
1069         isync
1070
1071         mfcr    %r30
1072
1073 /*
1074  * Now the kdb trap catching code.
1075  */
1076 dbtrap:
1077         FRAME_SETUP(SPR_SPRG1, PC_DBSAVE, EXC_DEBUG)
1078 /* Call C trap code: */
1079         GET_TOCBASE(%r2)
1080         addi    %r3, %r1, CALLSIZE
1081         bl      CNAME(db_trap_glue)
1082         TOC_RESTORE
1083         or.     %r3, %r3, %r3
1084         bne     dbleave
1085 /* This wasn't for KDB, so switch to real trap: */
1086         b       trap_common
1087
1088 dbleave:
1089         FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
1090         rfi
1091 #endif /* KDB */
1092
1093 #ifdef SMP
1094 ENTRY(tlb_lock)
1095         GET_CPUINFO(%r5)
1096         LOAD    %r5, PC_CURTHREAD(%r5)
1097 1:      LOADX   %r4, 0, %r3
1098         CMPI    %r4, TLB_UNLOCKED
1099         bne     1b
1100         STOREX  %r5, 0, %r3
1101         bne-    1b
1102         isync
1103         msync
1104         blr
1105
1106 ENTRY(tlb_unlock)
1107         isync
1108         msync
1109         li      %r4, TLB_UNLOCKED
1110         STORE   %r4, 0(%r3)
1111         isync
1112         msync
1113         blr
1114
1115 /*
1116  * TLB miss spin locks. For each CPU we have a reservation granule (32 bytes);
1117  * only a single word from this granule will actually be used as a spin lock
1118  * for mutual exclusion between TLB miss handler and pmap layer that
1119  * manipulates page table contents.
1120  */
1121         .data
1122         .align  5
1123 GLOBAL(tlb0_miss_locks)
1124         .space  RES_GRANULE * MAXCPU
1125 #endif