sys/powerpc/booke/trap_subr.S

   1 /*-
   2  * Copyright (C) 2006-2009 Semihalf, Rafal Jaworowski <raj@semihalf.com>
   3  * Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com>
   4  * Copyright (C) 2006 Juniper Networks, Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  * 3. The name of the author may not be used to endorse or promote products
  16  *    derived from this software without specific prior written permission.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  19  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  20  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN
  21  * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
  23  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  24  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  25  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  26  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  27  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28  */
  29 /*-
  30  * Copyright (C) 1995, 1996 Wolfgang Solfrank.
  31  * Copyright (C) 1995, 1996 TooLs GmbH.
  32  * All rights reserved.
  33  *
  34  * Redistribution and use in source and binary forms, with or without
  35  * modification, are permitted provided that the following conditions
  36  * are met:
  37  * 1. Redistributions of source code must retain the above copyright
  38  *    notice, this list of conditions and the following disclaimer.
  39  * 2. Redistributions in binary form must reproduce the above copyright
  40  *    notice, this list of conditions and the following disclaimer in the
  41  *    documentation and/or other materials provided with the distribution.
  42  * 3. All advertising materials mentioning features or use of this software
  43  *    must display the following acknowledgement:
  44  *      This product includes software developed by TooLs GmbH.
  45  * 4. The name of TooLs GmbH may not be used to endorse or promote products
  46  *    derived from this software without specific prior written permission.
  47  *
  48  * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
  49  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  50  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  51  * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  52  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  53  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  54  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  55  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  56  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  57  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  58  *
  59  *      from: $NetBSD: trap_subr.S,v 1.20 2002/04/22 23:20:08 kleink Exp $
  60  */
  61
  62 /*
  63  * NOTICE: This is not a standalone file.  to use it, #include it in
  64  * your port's locore.S, like so:
  65  *
  66  *      #include <powerpc/booke/trap_subr.S>
  67  */
  68
  69 /*
  70  * SPRG usage notes
  71  *
  72  * SPRG0 - pcpu pointer
  73  * SPRG1 - all interrupts except TLB miss, critical, machine check
  74  * SPRG2 - critical
  75  * SPRG3 - machine check
  76  * SPRG4-6 - scratch
  77  *
  78  */
  79
  80 /* Get the per-CPU data structure */
  81 #define GET_CPUINFO(r) mfsprg0 r
  82
  83 #define RES_GRANULE     64
  84 #define RES_LOCK        0       /* offset to the 'lock' word */
  85 #ifdef __powerpc64__
  86 #define RES_RECURSE     8       /* offset to the 'recurse' word */
  87 #else
  88 #define RES_RECURSE     4       /* offset to the 'recurse' word */
  89 #endif
  90
  91 /*
  92  * Standard interrupt prolog
  93  *
  94  * sprg_sp - SPRG{1-3} reg used to temporarily store the SP
  95  * savearea - temp save area (pc_{tempsave, disisave, critsave, mchksave})
  96  * isrr0-1 - save restore registers with CPU state at interrupt time (may be
  97  *           SRR0-1, CSRR0-1, MCSRR0-1
  98  *
  99  * 1. saves in the given savearea:
 100  *   - R30-31
 101  *   - DEAR, ESR
 102  *   - xSRR0-1
 103  *
 104  * 2. saves CR -> R30
 105  *
 106  * 3. switches to kstack if needed
 107  *
 108  * 4. notes:
 109  *   - R31 can be used as scratch register until a new frame is laid on
 110  *     the stack with FRAME_SETUP
 111  *
 112  *   - potential TLB miss: NO. Saveareas are always acessible via TLB1
 113  *     permanent entries, and within this prolog we do not dereference any
 114  *     locations potentially not in the TLB
 115  */
 116 #define STANDARD_PROLOG(sprg_sp, savearea, isrr0, isrr1)                \
 117         mtspr   sprg_sp, %r1;           /* Save SP */                   \
 118         GET_CPUINFO(%r1);               /* Per-cpu structure */         \
 119         STORE   %r30, (savearea+CPUSAVE_R30)(%r1);                      \
 120         STORE   %r31, (savearea+CPUSAVE_R31)(%r1);                      \
 121         mfspr   %r30, SPR_DEAR;                                         \
 122         mfspr   %r31, SPR_ESR;                                          \
 123         STORE   %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1);               \
 124         STORE   %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1);                \
 125         mfspr   %r30, isrr0;                                            \
 126         mfspr   %r31, isrr1;            /* MSR at interrupt time */     \
 127         STORE   %r30, (savearea+CPUSAVE_SRR0)(%r1);                     \
 128         STORE   %r31, (savearea+CPUSAVE_SRR1)(%r1);                     \
 129         isync;                                                          \
 130         mfspr   %r1, sprg_sp;           /* Restore SP */                \
 131         mfcr    %r30;                   /* Save CR */                   \
 132         /* switch to per-thread kstack if intr taken in user mode */    \
 133         mtcr    %r31;                   /* MSR at interrupt time  */    \
 134         bf      17, 1f;                                                 \
 135         GET_CPUINFO(%r1);               /* Per-cpu structure */         \
 136         LOAD    %r1, PC_CURPCB(%r1);    /* Per-thread kernel stack */   \
 137 1:
 138
 139 #define STANDARD_CRIT_PROLOG(sprg_sp, savearea, isrr0, isrr1)           \
 140         mtspr   sprg_sp, %r1;           /* Save SP */                   \
 141         GET_CPUINFO(%r1);               /* Per-cpu structure */         \
 142         STORE   %r30, (savearea+CPUSAVE_R30)(%r1);                      \
 143         STORE   %r31, (savearea+CPUSAVE_R31)(%r1);                      \
 144         mfspr   %r30, SPR_DEAR;                                         \
 145         mfspr   %r31, SPR_ESR;                                          \
 146         STORE   %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1);               \
 147         STORE   %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1);                \
 148         mfspr   %r30, isrr0;                                            \
 149         mfspr   %r31, isrr1;            /* MSR at interrupt time */     \
 150         STORE   %r30, (savearea+CPUSAVE_SRR0)(%r1);                     \
 151         STORE   %r31, (savearea+CPUSAVE_SRR1)(%r1);                     \
 152         mfspr   %r30, SPR_SRR0;                                         \
 153         mfspr   %r31, SPR_SRR1;         /* MSR at interrupt time */     \
 154         STORE   %r30, (savearea+BOOKE_CRITSAVE_SRR0)(%r1);              \
 155         STORE   %r31, (savearea+BOOKE_CRITSAVE_SRR1)(%r1);              \
 156         isync;                                                          \
 157         mfspr   %r1, sprg_sp;           /* Restore SP */                \
 158         mfcr    %r30;                   /* Save CR */                   \
 159         /* switch to per-thread kstack if intr taken in user mode */    \
 160         mtcr    %r31;                   /* MSR at interrupt time  */    \
 161         bf      17, 1f;                                                 \
 162         GET_CPUINFO(%r1);               /* Per-cpu structure */         \
 163         LOAD    %r1, PC_CURPCB(%r1);    /* Per-thread kernel stack */   \
 164 1:
 165
 166 /*
 167  * FRAME_SETUP assumes:
 168  *      SPRG{1-3}       SP at the time interrupt occurred
 169  *      savearea        r30-r31, DEAR, ESR, xSRR0-1
 170  *      r30             CR
 171  *      r31             scratch
 172  *      r1              kernel stack
 173  *
 174  * sprg_sp - SPRG reg containing SP at the time interrupt occurred
 175  * savearea - temp save
 176  * exc - exception number (EXC_xxx)
 177  *
 178  * 1. sets a new frame
 179  * 2. saves in the frame:
 180  *   - R0, R1 (SP at the time of interrupt), R2, LR, CR
 181  *   - R3-31 (R30-31 first restored from savearea)
 182  *   - XER, CTR, DEAR, ESR (from savearea), xSRR0-1
 183  *
 184  * Notes:
 185  * - potential TLB miss: YES, since we make dereferences to kstack, which
 186  *   can happen not covered (we can have up to two DTLB misses if fortunate
 187  *   enough i.e. when kstack crosses page boundary and both pages are
 188  *   untranslated)
 189  */
 190 #ifdef __powerpc64__
 191 #define SAVE_REGS(r)                                                    \
 192         std     %r3, FRAME_3+CALLSIZE(r);                               \
 193         std     %r4, FRAME_4+CALLSIZE(r);                               \
 194         std     %r5, FRAME_5+CALLSIZE(r);                               \
 195         std     %r6, FRAME_6+CALLSIZE(r);                               \
 196         std     %r7, FRAME_7+CALLSIZE(r);                               \
 197         std     %r8, FRAME_8+CALLSIZE(r);                               \
 198         std     %r9, FRAME_9+CALLSIZE(r);                               \
 199         std     %r10, FRAME_10+CALLSIZE(r);                             \
 200         std     %r11, FRAME_11+CALLSIZE(r);                             \
 201         std     %r12, FRAME_12+CALLSIZE(r);                             \
 202         std     %r13, FRAME_13+CALLSIZE(r);                             \
 203         std     %r14, FRAME_14+CALLSIZE(r);                             \
 204         std     %r15, FRAME_15+CALLSIZE(r);                             \
 205         std     %r16, FRAME_16+CALLSIZE(r);                             \
 206         std     %r17, FRAME_17+CALLSIZE(r);                             \
 207         std     %r18, FRAME_18+CALLSIZE(r);                             \
 208         std     %r19, FRAME_19+CALLSIZE(r);                             \
 209         std     %r20, FRAME_20+CALLSIZE(r);                             \
 210         std     %r21, FRAME_21+CALLSIZE(r);                             \
 211         std     %r22, FRAME_22+CALLSIZE(r);                             \
 212         std     %r23, FRAME_23+CALLSIZE(r);                             \
 213         std     %r24, FRAME_24+CALLSIZE(r);                             \
 214         std     %r25, FRAME_25+CALLSIZE(r);                             \
 215         std     %r26, FRAME_26+CALLSIZE(r);                             \
 216         std     %r27, FRAME_27+CALLSIZE(r);                             \
 217         std     %r28, FRAME_28+CALLSIZE(r);                             \
 218         std     %r29, FRAME_29+CALLSIZE(r);                             \
 219         std     %r30, FRAME_30+CALLSIZE(r);                             \
 220         std     %r31, FRAME_31+CALLSIZE(r)
 221 #define LD_REGS(r)                                                      \
 222         ld      %r3, FRAME_3+CALLSIZE(r);                               \
 223         ld      %r4, FRAME_4+CALLSIZE(r);                               \
 224         ld      %r5, FRAME_5+CALLSIZE(r);                               \
 225         ld      %r6, FRAME_6+CALLSIZE(r);                               \
 226         ld      %r7, FRAME_7+CALLSIZE(r);                               \
 227         ld      %r8, FRAME_8+CALLSIZE(r);                               \
 228         ld      %r9, FRAME_9+CALLSIZE(r);                               \
 229         ld      %r10, FRAME_10+CALLSIZE(r);                             \
 230         ld      %r11, FRAME_11+CALLSIZE(r);                             \
 231         ld      %r12, FRAME_12+CALLSIZE(r);                             \
 232         ld      %r13, FRAME_13+CALLSIZE(r);                             \
 233         ld      %r14, FRAME_14+CALLSIZE(r);                             \
 234         ld      %r15, FRAME_15+CALLSIZE(r);                             \
 235         ld      %r16, FRAME_16+CALLSIZE(r);                             \
 236         ld      %r17, FRAME_17+CALLSIZE(r);                             \
 237         ld      %r18, FRAME_18+CALLSIZE(r);                             \
 238         ld      %r19, FRAME_19+CALLSIZE(r);                             \
 239         ld      %r20, FRAME_20+CALLSIZE(r);                             \
 240         ld      %r21, FRAME_21+CALLSIZE(r);                             \
 241         ld      %r22, FRAME_22+CALLSIZE(r);                             \
 242         ld      %r23, FRAME_23+CALLSIZE(r);                             \
 243         ld      %r24, FRAME_24+CALLSIZE(r);                             \
 244         ld      %r25, FRAME_25+CALLSIZE(r);                             \
 245         ld      %r26, FRAME_26+CALLSIZE(r);                             \
 246         ld      %r27, FRAME_27+CALLSIZE(r);                             \
 247         ld      %r28, FRAME_28+CALLSIZE(r);                             \
 248         ld      %r29, FRAME_29+CALLSIZE(r);                             \
 249         ld      %r30, FRAME_30+CALLSIZE(r);                             \
 250         ld      %r31, FRAME_31+CALLSIZE(r)
 251 #else
 252 #define SAVE_REGS(r)                                                    \
 253         stmw    %r3,  FRAME_3+CALLSIZE(r)
 254 #define LD_REGS(r)                                                      \
 255         lmw     %r3,  FRAME_3+CALLSIZE(r)
 256 #endif
 257 #define FRAME_SETUP(sprg_sp, savearea, exc)                             \
 258         mfspr   %r31, sprg_sp;          /* get saved SP */              \
 259         /* establish a new stack frame and put everything on it */      \
 260         STU     %r31, -(FRAMELEN+REDZONE)(%r1);                         \
 261         STORE   %r0, FRAME_0+CALLSIZE(%r1);     /* save r0 in the trapframe */  \
 262         STORE   %r31, FRAME_1+CALLSIZE(%r1);    /* save SP   "     " */ \
 263         STORE   %r2, FRAME_2+CALLSIZE(%r1);     /* save r2   "     " */ \
 264         mflr    %r31;                                                   \
 265         STORE   %r31, FRAME_LR+CALLSIZE(%r1);   /* save LR   "     " */ \
 266         STORE   %r30, FRAME_CR+CALLSIZE(%r1);   /* save CR   "     " */ \
 267         GET_CPUINFO(%r2);                                               \
 268         LOAD    %r30, (savearea+CPUSAVE_R30)(%r2); /* get saved r30 */  \
 269         LOAD    %r31, (savearea+CPUSAVE_R31)(%r2); /* get saved r31 */  \
 270         /* save R3-31 */                                                \
 271         SAVE_REGS(%r1);                                                 \
 272         /* save DEAR, ESR */                                            \
 273         LOAD    %r28, (savearea+CPUSAVE_BOOKE_DEAR)(%r2);               \
 274         LOAD    %r29, (savearea+CPUSAVE_BOOKE_ESR)(%r2);                \
 275         STORE   %r28, FRAME_BOOKE_DEAR+CALLSIZE(%r1);                   \
 276         STORE   %r29, FRAME_BOOKE_ESR+CALLSIZE(%r1);                    \
 277         /* save XER, CTR, exc number */                                 \
 278         mfxer   %r3;                                                    \
 279         mfctr   %r4;                                                    \
 280         STORE   %r3, FRAME_XER+CALLSIZE(%r1);                           \
 281         STORE   %r4, FRAME_CTR+CALLSIZE(%r1);                           \
 282         li      %r5, exc;                                               \
 283         STORE   %r5, FRAME_EXC+CALLSIZE(%r1);                           \
 284         /* save DBCR0 */                                                \
 285         mfspr   %r3, SPR_DBCR0;                                         \
 286         STORE   %r3, FRAME_BOOKE_DBCR0+CALLSIZE(%r1);                   \
 287         /* save xSSR0-1 */                                              \
 288         LOAD    %r30, (savearea+CPUSAVE_SRR0)(%r2);                     \
 289         LOAD    %r31, (savearea+CPUSAVE_SRR1)(%r2);                     \
 290         STORE   %r30, FRAME_SRR0+CALLSIZE(%r1);                         \
 291         STORE   %r31, FRAME_SRR1+CALLSIZE(%r1);                         \
 292         LOAD    THREAD_REG, PC_CURTHREAD(%r2);                          \
 293
 294 /*
 295  *
 296  * isrr0-1 - save restore registers to restore CPU state to (may be
 297  *           SRR0-1, CSRR0-1, MCSRR0-1
 298  *
 299  * Notes:
 300  *  - potential TLB miss: YES. The deref'd kstack may be not covered
 301  */
 302 #define FRAME_LEAVE(isrr0, isrr1)                                       \
 303         wrteei 0;                                                       \
 304         /* restore CTR, XER, LR, CR */                                  \
 305         LOAD    %r4, FRAME_CTR+CALLSIZE(%r1);                           \
 306         LOAD    %r5, FRAME_XER+CALLSIZE(%r1);                           \
 307         LOAD    %r6, FRAME_LR+CALLSIZE(%r1);                            \
 308         LOAD    %r7, FRAME_CR+CALLSIZE(%r1);                            \
 309         mtctr   %r4;                                                    \
 310         mtxer   %r5;                                                    \
 311         mtlr    %r6;                                                    \
 312         mtcr    %r7;                                                    \
 313         /* restore DBCR0 */                                             \
 314         LOAD    %r4, FRAME_BOOKE_DBCR0+CALLSIZE(%r1);                   \
 315         mtspr   SPR_DBCR0, %r4;                                         \
 316         /* restore xSRR0-1 */                                           \
 317         LOAD    %r30, FRAME_SRR0+CALLSIZE(%r1);                         \
 318         LOAD    %r31, FRAME_SRR1+CALLSIZE(%r1);                         \
 319         mtspr   isrr0, %r30;                                            \
 320         mtspr   isrr1, %r31;                                            \
 321         /* restore R2-31, SP */                                         \
 322         LD_REGS(%r1);                                                   \
 323         LOAD    %r2, FRAME_2+CALLSIZE(%r1);                             \
 324         LOAD    %r0, FRAME_0+CALLSIZE(%r1);                             \
 325         LOAD    %r1, FRAME_1+CALLSIZE(%r1);                             \
 326         isync
 327
 328 /*
 329  * TLB miss prolog
 330  *
 331  * saves LR, CR, SRR0-1, R20-31 in the TLBSAVE area
 332  *
 333  * Notes:
 334  *  - potential TLB miss: NO. It is crucial that we do not generate a TLB
 335  *    miss within the TLB prolog itself!
 336  *  - TLBSAVE is always translated
 337  */
 338 #ifdef __powerpc64__
 339 #define TLB_SAVE_REGS(br)                                               \
 340         std     %r20, (TLBSAVE_BOOKE_R20)(br);                          \
 341         std     %r21, (TLBSAVE_BOOKE_R21)(br);                          \
 342         std     %r22, (TLBSAVE_BOOKE_R22)(br);                          \
 343         std     %r23, (TLBSAVE_BOOKE_R23)(br);                          \
 344         std     %r24, (TLBSAVE_BOOKE_R24)(br);                          \
 345         std     %r25, (TLBSAVE_BOOKE_R25)(br);                          \
 346         std     %r26, (TLBSAVE_BOOKE_R26)(br);                          \
 347         std     %r27, (TLBSAVE_BOOKE_R27)(br);                          \
 348         std     %r28, (TLBSAVE_BOOKE_R28)(br);                          \
 349         std     %r29, (TLBSAVE_BOOKE_R29)(br);                          \
 350         std     %r30, (TLBSAVE_BOOKE_R30)(br);                          \
 351         std     %r31, (TLBSAVE_BOOKE_R31)(br);
 352 #define TLB_RESTORE_REGS(br)                                            \
 353         ld      %r20, (TLBSAVE_BOOKE_R20)(br);                          \
 354         ld      %r21, (TLBSAVE_BOOKE_R21)(br);                          \
 355         ld      %r22, (TLBSAVE_BOOKE_R22)(br);                          \
 356         ld      %r23, (TLBSAVE_BOOKE_R23)(br);                          \
 357         ld      %r24, (TLBSAVE_BOOKE_R24)(br);                          \
 358         ld      %r25, (TLBSAVE_BOOKE_R25)(br);                          \
 359         ld      %r26, (TLBSAVE_BOOKE_R26)(br);                          \
 360         ld      %r27, (TLBSAVE_BOOKE_R27)(br);                          \
 361         ld      %r28, (TLBSAVE_BOOKE_R28)(br);                          \
 362         ld      %r29, (TLBSAVE_BOOKE_R29)(br);                          \
 363         ld      %r30, (TLBSAVE_BOOKE_R30)(br);                          \
 364         ld      %r31, (TLBSAVE_BOOKE_R31)(br);
 365 #define TLB_NEST(outr,inr)                                              \
 366         rlwinm  outr, inr, 7, 23, 24;   /* 8 x TLBSAVE_LEN */
 367 #else
 368 #define TLB_SAVE_REGS(br)                                               \
 369         stmw    %r20, TLBSAVE_BOOKE_R20(br)
 370 #define TLB_RESTORE_REGS(br)                                            \
 371         lmw     %r20, TLBSAVE_BOOKE_R20(br)
 372 #define TLB_NEST(outr,inr)                                              \
 373         rlwinm  outr, inr, 6, 24, 25;   /* 4 x TLBSAVE_LEN */
 374 #endif
 375 #define TLB_PROLOG                                                      \
 376         mtspr   SPR_SPRG4, %r1;                 /* Save SP */           \
 377         mtspr   SPR_SPRG5, %r28;                                        \
 378         mtspr   SPR_SPRG6, %r29;                                        \
 379         /* calculate TLB nesting level and TLBSAVE instance address */  \
 380         GET_CPUINFO(%r1);               /* Per-cpu structure */         \
 381         LOAD    %r28, PC_BOOKE_TLB_LEVEL(%r1);                          \
 382         TLB_NEST(%r29,%r28);                                            \
 383         addi    %r28, %r28, 1;                                          \
 384         STORE   %r28, PC_BOOKE_TLB_LEVEL(%r1);                          \
 385         addi    %r29, %r29, PC_BOOKE_TLBSAVE@l;                         \
 386         add     %r1, %r1, %r29;         /* current TLBSAVE ptr */       \
 387                                                                         \
 388         /* save R20-31 */                                               \
 389         mfspr   %r28, SPR_SPRG5;                                        \
 390         mfspr   %r29, SPR_SPRG6;                                        \
 391         TLB_SAVE_REGS(%r1);                     \
 392         /* save LR, CR */                                               \
 393         mflr    %r30;                                                   \
 394         mfcr    %r31;                                                   \
 395         STORE   %r30, (TLBSAVE_BOOKE_LR)(%r1);                          \
 396         STORE   %r31, (TLBSAVE_BOOKE_CR)(%r1);                          \
 397         /* save SRR0-1 */                                               \
 398         mfsrr0  %r30;           /* execution addr at interrupt time */  \
 399         mfsrr1  %r31;           /* MSR at interrupt time*/              \
 400         STORE   %r30, (TLBSAVE_BOOKE_SRR0)(%r1);        /* save SRR0 */ \
 401         STORE   %r31, (TLBSAVE_BOOKE_SRR1)(%r1);        /* save SRR1 */ \
 402         isync;                                                          \
 403         mfspr   %r1, SPR_SPRG4
 404
 405 /*
 406  * restores LR, CR, SRR0-1, R20-31 from the TLBSAVE area
 407  *
 408  * same notes as for the TLB_PROLOG
 409  */
 410 #define TLB_RESTORE                                                     \
 411         mtspr   SPR_SPRG4, %r1;                 /* Save SP */           \
 412         GET_CPUINFO(%r1);               /* Per-cpu structure */         \
 413         /* calculate TLB nesting level and TLBSAVE instance addr */     \
 414         LOAD    %r28, PC_BOOKE_TLB_LEVEL(%r1);                          \
 415         subi    %r28, %r28, 1;                                          \
 416         STORE   %r28, PC_BOOKE_TLB_LEVEL(%r1);                          \
 417         TLB_NEST(%r29,%r28);                                            \
 418         addi    %r29, %r29, PC_BOOKE_TLBSAVE@l;                         \
 419         add     %r1, %r1, %r29;                                         \
 420                                                                         \
 421         /* restore LR, CR */                                            \
 422         LOAD    %r30, (TLBSAVE_BOOKE_LR)(%r1);                          \
 423         LOAD    %r31, (TLBSAVE_BOOKE_CR)(%r1);                          \
 424         mtlr    %r30;                                                   \
 425         mtcr    %r31;                                                   \
 426         /* restore SRR0-1 */                                            \
 427         LOAD    %r30, (TLBSAVE_BOOKE_SRR0)(%r1);                        \
 428         LOAD    %r31, (TLBSAVE_BOOKE_SRR1)(%r1);                        \
 429         mtsrr0  %r30;                                                   \
 430         mtsrr1  %r31;                                                   \
 431         /* restore R20-31 */                                            \
 432         TLB_RESTORE_REGS(%r1);                                          \
 433         mfspr   %r1, SPR_SPRG4
 434
 435 #ifdef SMP
 436 #define TLB_LOCK                                                        \
 437         GET_CPUINFO(%r20);                                              \
 438         LOAD    %r21, PC_CURTHREAD(%r20);                               \
 439         LOAD    %r22, PC_BOOKE_TLB_LOCK(%r20);                          \
 440                                                                         \
 441 1:      LOADX   %r23, 0, %r22;                                          \
 442         CMPI    %r23, TLB_UNLOCKED;                                     \
 443         beq     2f;                                                     \
 444                                                                         \
 445         /* check if this is recursion */                                \
 446         CMPL    cr0, %r21, %r23;                                        \
 447         bne-    1b;                                                     \
 448                                                                         \
 449 2:      /* try to acquire lock */                                       \
 450         STOREX  %r21, 0, %r22;                                          \
 451         bne-    1b;                                                     \
 452                                                                         \
 453         /* got it, update recursion counter */                          \
 454         lwz     %r21, RES_RECURSE(%r22);                                \
 455         addi    %r21, %r21, 1;                                          \
 456         stw     %r21, RES_RECURSE(%r22);                                \
 457         isync;                                                          \
 458         msync
 459
 460 #define TLB_UNLOCK                                                      \
 461         GET_CPUINFO(%r20);                                              \
 462         LOAD    %r21, PC_CURTHREAD(%r20);                               \
 463         LOAD    %r22, PC_BOOKE_TLB_LOCK(%r20);                          \
 464                                                                         \
 465         /* update recursion counter */                                  \
 466         lwz     %r23, RES_RECURSE(%r22);                                \
 467         subi    %r23, %r23, 1;                                          \
 468         stw     %r23, RES_RECURSE(%r22);                                \
 469                                                                         \
 470         cmplwi  %r23, 0;                                                \
 471         bne     1f;                                                     \
 472         isync;                                                          \
 473         msync;                                                          \
 474                                                                         \
 475         /* release the lock */                                          \
 476         li      %r23, TLB_UNLOCKED;                                     \
 477         STORE   %r23, 0(%r22);                                          \
 478 1:      isync;                                                          \
 479         msync
 480 #else
 481 #define TLB_LOCK
 482 #define TLB_UNLOCK
 483 #endif  /* SMP */
 484
 485 #define INTERRUPT(label)                                                \
 486         .globl  label;                                                  \
 487         .align  5;                                                      \
 488         CNAME(label):
 489
 490 /*
 491  * Interrupt handling routines in BookE can be flexibly placed and do not have
 492  * to live in pre-defined vectors location. Note they need to be TLB-mapped at
 493  * all times in order to be able to handle exceptions. We thus arrange for
 494  * them to be part of kernel text which is always TLB-accessible.
 495  *
 496  * The interrupt handling routines have to be 16 bytes aligned: we align them
 497  * to 32 bytes (cache line length) which supposedly performs better.
 498  *
 499  */
 500         .text
 501         .globl CNAME(interrupt_vector_base)
 502         .align 5
 503 interrupt_vector_base:
 504 /*****************************************************************************
 505  * Catch-all handler to handle uninstalled IVORs
 506  ****************************************************************************/
 507 INTERRUPT(int_unknown)
 508         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 509         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_RSVD)
 510         b       trap_common
 511
 512 /*****************************************************************************
 513  * Critical input interrupt
 514  ****************************************************************************/
 515 INTERRUPT(int_critical_input)
 516         STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1)
 517         FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_CRIT)
 518         GET_TOCBASE(%r2)
 519         addi    %r3, %r1, CALLSIZE
 520         bl      CNAME(powerpc_interrupt)
 521         TOC_RESTORE
 522         FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1)
 523         rfci
 524
 525
 526 /*****************************************************************************
 527  * Machine check interrupt
 528  ****************************************************************************/
 529 INTERRUPT(int_machine_check)
 530         STANDARD_PROLOG(SPR_SPRG3, PC_BOOKE_MCHKSAVE, SPR_MCSRR0, SPR_MCSRR1)
 531         FRAME_SETUP(SPR_SPRG3, PC_BOOKE_MCHKSAVE, EXC_MCHK)
 532         GET_TOCBASE(%r2)
 533         addi    %r3, %r1, CALLSIZE
 534         bl      CNAME(powerpc_interrupt)
 535         TOC_RESTORE
 536         FRAME_LEAVE(SPR_MCSRR0, SPR_MCSRR1)
 537         rfmci
 538
 539
 540 /*****************************************************************************
 541  * Data storage interrupt
 542  ****************************************************************************/
 543 INTERRUPT(int_data_storage)
 544         STANDARD_PROLOG(SPR_SPRG1, PC_DISISAVE, SPR_SRR0, SPR_SRR1)
 545         FRAME_SETUP(SPR_SPRG1, PC_DISISAVE, EXC_DSI)
 546         b       trap_common
 547
 548
 549 /*****************************************************************************
 550  * Instruction storage interrupt
 551  ****************************************************************************/
 552 INTERRUPT(int_instr_storage)
 553         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 554         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ISI)
 555         b       trap_common
 556
 557
 558 /*****************************************************************************
 559  * External input interrupt
 560  ****************************************************************************/
 561 INTERRUPT(int_external_input)
 562         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 563         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_EXI)
 564         b       trap_common
 565
 566
 567 INTERRUPT(int_alignment)
 568         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 569         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ALI)
 570         b       trap_common
 571
 572
 573 INTERRUPT(int_program)
 574         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 575         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_PGM)
 576         b       trap_common
 577
 578
 579 INTERRUPT(int_fpu)
 580         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 581         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FPU)
 582         b       trap_common
 583
 584
 585 /*****************************************************************************
 586  * System call
 587  ****************************************************************************/
 588 INTERRUPT(int_syscall)
 589         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 590         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SC)
 591         b       trap_common
 592
 593
 594 /*****************************************************************************
 595  * Decrementer interrupt
 596  ****************************************************************************/
 597 INTERRUPT(int_decrementer)
 598         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 599         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_DECR)
 600         b       trap_common
 601
 602
 603 /*****************************************************************************
 604  * Fixed interval timer
 605  ****************************************************************************/
 606 INTERRUPT(int_fixed_interval_timer)
 607         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 608         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FIT)
 609         b       trap_common
 610
 611
 612 /*****************************************************************************
 613  * Watchdog interrupt
 614  ****************************************************************************/
 615 INTERRUPT(int_watchdog)
 616         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 617         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_WDOG)
 618         b       trap_common
 619
 620
 621 /*****************************************************************************
 622  * Altivec Unavailable interrupt
 623  ****************************************************************************/
 624 INTERRUPT(int_vec)
 625         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 626         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VEC)
 627         b       trap_common
 628
 629
 630 /*****************************************************************************
 631  * Altivec Assist interrupt
 632  ****************************************************************************/
 633 INTERRUPT(int_vecast)
 634         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 635         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VECAST_E)
 636         b       trap_common
 637
 638
 639 #ifdef __SPE__
 640 /*****************************************************************************
 641  * Floating point Assist interrupt
 642  ****************************************************************************/
 643 INTERRUPT(int_spe_fpdata)
 644         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 645         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SPFPD)
 646         addi    %r3, %r1, CALLSIZE
 647         bl      spe_handle_fpdata
 648         FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
 649         rfi
 650
 651 INTERRUPT(int_spe_fpround)
 652         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 653         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SPFPR)
 654         addi    %r3, %r1, CALLSIZE
 655         bl      spe_handle_fpround
 656         FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
 657         rfi
 658 #endif
 659
 660
 661 #ifdef HWPMC_HOOKS
 662 /*****************************************************************************
 663  * PMC Interrupt
 664  ****************************************************************************/
 665 INTERRUPT(int_performance_counter)
 666         STANDARD_PROLOG(SPR_SPRG3, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
 667         FRAME_SETUP(SPR_SPRG3, PC_TEMPSAVE, EXC_PERF)
 668         b       trap_common
 669 #endif
 670
 671
 672 /*****************************************************************************
 673  * Data TLB miss interrupt
 674  *
 675  * There can be nested TLB misses - while handling a TLB miss we reference
 676  * data structures that may be not covered by translations. We support up to
 677  * TLB_NESTED_MAX-1 nested misses.
 678  *
 679  * Registers use:
 680  *      r31 - dear
 681  *      r30 - unused
 682  *      r29 - saved mas0
 683  *      r28 - saved mas1
 684  *      r27 - saved mas2
 685  *      r26 - pmap address
 686  *      r25 - pte address
 687  *
 688  *      r20:r23 - scratch registers
 689  ****************************************************************************/
 690 INTERRUPT(int_data_tlb_error)
 691         TLB_PROLOG
 692         TLB_LOCK
 693
 694         mfspr   %r31, SPR_DEAR
 695
 696         /*
 697          * Save MAS0-MAS2 registers. There might be another tlb miss during
 698          * pte lookup overwriting current contents (which was hw filled).
 699          */
 700         mfspr   %r29, SPR_MAS0
 701         mfspr   %r28, SPR_MAS1
 702         mfspr   %r27, SPR_MAS2
 703
 704         /* Check faulting address. */
 705         LOAD_ADDR(%r21, VM_MAXUSER_ADDRESS)
 706         CMPL    cr0, %r31, %r21
 707         blt     search_user_pmap
 708
 709         /* If it's kernel address, allow only supervisor mode misses. */
 710         mfsrr1  %r21
 711         mtcr    %r21
 712         bt      17, search_failed       /* check MSR[PR] */
 713
 714 #ifdef __powerpc64__
 715         srdi    %r21, %r31, 48
 716         cmpldi  cr0, %r21, VM_MIN_KERNEL_ADDRESS@highest
 717 #else
 718         lis     %r21, VM_MIN_KERNEL_ADDRESS@h
 719         cmplw   cr0, %r31, %r21
 720 #endif
 721         blt     search_failed
 722
 723 search_kernel_pmap:
 724         /* Load r26 with kernel_pmap address */
 725         bl      1f
 726 #ifdef __powerpc64__
 727         .llong kernel_pmap_store-.
 728 #else
 729         .long kernel_pmap_store-.
 730 #endif
 731 1:      mflr    %r21
 732         LOAD    %r26, 0(%r21)
 733         add     %r26, %r21, %r26        /* kernel_pmap_store in r26 */
 734
 735         /* Force kernel tid, set TID to 0 in MAS1. */
 736         li      %r21, 0
 737         rlwimi  %r28, %r21, 0, 8, 15    /* clear TID bits */
 738
 739 tlb_miss_handle:
 740         /* This may result in nested tlb miss. */
 741         bl      pte_lookup              /* returns PTE address in R25 */
 742
 743         CMPI    %r25, 0                 /* pte found? */
 744         beq     search_failed
 745
 746         /* Finish up, write TLB entry. */
 747         bl      tlb_fill_entry
 748
 749 tlb_miss_return:
 750         TLB_UNLOCK
 751         TLB_RESTORE
 752         rfi
 753
 754 search_user_pmap:
 755         /* Load r26 with current user space process pmap */
 756         GET_CPUINFO(%r26)
 757         LOAD    %r26, PC_CURPMAP(%r26)
 758
 759         b       tlb_miss_handle
 760
 761 search_failed:
 762         /*
 763          * Whenever we don't find a TLB mapping in PT, set a TLB0 entry with
 764          * the faulting virtual address anyway, but put a fake RPN and no
 765          * access rights. This should cause a following {D,I}SI exception.
 766          */
 767         lis     %r23, 0xffff0000@h      /* revoke all permissions */
 768
 769         /* Load MAS registers. */
 770         mtspr   SPR_MAS0, %r29
 771         mtspr   SPR_MAS1, %r28
 772         mtspr   SPR_MAS2, %r27
 773         mtspr   SPR_MAS3, %r23
 774
 775         li      %r23, 0
 776         mtspr   SPR_MAS7, %r23
 777
 778         isync
 779         tlbwe
 780         msync
 781         isync
 782         b       tlb_miss_return
 783
 784 /*****************************************************************************
 785  *
 786  * Return pte address that corresponds to given pmap/va.  If there is no valid
 787  * entry return 0.
 788  *
 789  * input: r26 - pmap
 790  * input: r31 - dear
 791  * output: r25 - pte address
 792  *
 793  * scratch regs used: r21
 794  *
 795  ****************************************************************************/
 796 pte_lookup:
 797         CMPI    %r26, 0
 798         beq     1f                      /* fail quickly if pmap is invalid */
 799
 800 #ifdef __powerpc64__
 801         rldicl  %r21, %r31, (64 - PG_ROOT_L), (64 - PG_ROOT_NUM) /* pp2d offset */
 802         slwi    %r21, %r21, PG_ROOT_ENTRY_SHIFT /* multiply by pp2d entry size */
 803         ld      %r25, PM_ROOT(%r26)             /* pmap pm_pp2d[] address */
 804         ldx     %r25, %r25, %r21                /* get pdir address, i.e.  pmap->pm_pp2d[pp2d_idx] * */
 805
 806         cmpdi   %r25, 0
 807         beq 2f
 808
 809         rldicl  %r21, %r31, (64 - PDIR_L1_L), (64 - PDIR_L1_NUM) /* pp2d offset */
 810         slwi    %r21, %r21, PDIR_L1_ENTRY_SHIFT /* multiply by pp2d entry size */
 811         ldx     %r25, %r25, %r21                /* get pdir address, i.e.  pmap->pm_pp2d[pp2d_idx] * */
 812
 813         cmpdi   %r25, 0
 814         beq 2f
 815
 816         rldicl  %r21, %r31, (64 - PDIR_L), (64 - PDIR_NUM)      /* pdir offset */
 817         slwi    %r21, %r21, PDIR_ENTRY_SHIFT    /* multiply by pdir entry size */
 818         ldx     %r25, %r25, %r21                /* get ptbl address, i.e.  pmap->pm_pp2d[pp2d_idx][pdir_idx] */
 819
 820         cmpdi   %r25, 0
 821         beq     2f
 822
 823         rldicl  %r21, %r31, (64 - PTBL_L), (64 - PTBL_NUM) /* ptbl offset */
 824         slwi    %r21, %r21, PTBL_ENTRY_SHIFT   /* multiply by pte entry size */
 825
 826 #else
 827         srwi    %r21, %r31, PDIR_SHIFT          /* pdir offset */
 828         slwi    %r21, %r21, PDIR_ENTRY_SHIFT    /* multiply by pdir entry size */
 829
 830         lwz     %r25, PM_PDIR(%r26)     /* pmap pm_dir[] address */
 831         /*
 832          * Get ptbl address, i.e. pmap->pm_pdir[pdir_idx]
 833          * This load may cause a Data TLB miss for non-kernel pmap!
 834          */
 835         lwzx    %r25, %r25, %r21        /* offset within pm_pdir[] table */
 836         cmpwi   %r25, 0
 837         beq     2f
 838
 839         lis     %r21, PTBL_MASK@h
 840         ori     %r21, %r21, PTBL_MASK@l
 841         and     %r21, %r21, %r31
 842
 843         /* ptbl offset, multiply by ptbl entry size */
 844         srwi    %r21, %r21, (PTBL_SHIFT - PTBL_ENTRY_SHIFT)
 845 #endif
 846
 847         add     %r25, %r25, %r21                /* address of pte entry */
 848         /*
 849          * Get pte->flags
 850          * This load may cause a Data TLB miss for non-kernel pmap!
 851          */
 852         lwz     %r21, PTE_FLAGS(%r25)
 853         andi.   %r21, %r21, PTE_VALID@l
 854         bne     2f
 855 1:
 856         li      %r25, 0
 857 2:
 858         blr
 859
 860 /*****************************************************************************
 861  *
 862  * Load MAS1-MAS3 registers with data, write TLB entry
 863  *
 864  * input:
 865  * r29 - mas0
 866  * r28 - mas1
 867  * r27 - mas2
 868  * r25 - pte
 869  *
 870  * output: none
 871  *
 872  * scratch regs: r21-r23
 873  *
 874  ****************************************************************************/
 875 tlb_fill_entry:
 876         /*
 877          * Update PTE flags: we have to do it atomically, as pmap_protect()
 878          * running on other CPUs could attempt to update the flags at the same
 879          * time.
 880          */
 881         li      %r23, PTE_FLAGS
 882 1:
 883         lwarx   %r21, %r23, %r25                /* get pte->flags */
 884         oris    %r21, %r21, PTE_REFERENCED@h    /* set referenced bit */
 885
 886         andi.   %r22, %r21, (PTE_SW | PTE_UW)@l /* check if writable */
 887         beq     2f
 888         ori     %r21, %r21, PTE_MODIFIED@l      /* set modified bit */
 889 2:
 890         stwcx.  %r21, %r23, %r25                /* write it back */
 891         bne-    1b
 892
 893         /* Update MAS2. */
 894         rlwimi  %r27, %r21, 13, 27, 30          /* insert WIMG bits from pte */
 895
 896         /* Setup MAS3 value in r23. */
 897         LOAD    %r23, PTE_RPN(%r25)             /* get pte->rpn */
 898 #ifdef __powerpc64__
 899         rldicr  %r22, %r23, 52, 51              /* extract MAS3 portion of RPN */
 900         rldicl  %r23, %r23, 20, 54              /* extract MAS7 portion of RPN */
 901
 902         rlwimi  %r22, %r21, 30, 26, 31          /* insert protection bits from pte */
 903 #else
 904         rlwinm  %r22, %r23, 20, 0, 11           /* extract MAS3 portion of RPN */
 905
 906         rlwimi  %r22, %r21, 30, 26, 31          /* insert protection bits from pte */
 907         rlwimi  %r22, %r21, 20, 12, 19          /* insert lower 8 RPN bits to MAS3 */
 908         rlwinm  %r23, %r23, 20, 24, 31          /* MAS7 portion of RPN */
 909 #endif
 910
 911         /* Load MAS registers. */
 912         mtspr   SPR_MAS0, %r29
 913         mtspr   SPR_MAS1, %r28
 914         mtspr   SPR_MAS2, %r27
 915         mtspr   SPR_MAS3, %r22
 916         mtspr   SPR_MAS7, %r23
 917
 918         isync
 919         tlbwe
 920         isync
 921         msync
 922         blr
 923
 924 /*****************************************************************************
 925  * Instruction TLB miss interrupt
 926  *
 927  * Same notes as for the Data TLB miss
 928  ****************************************************************************/
 929 INTERRUPT(int_inst_tlb_error)
 930         TLB_PROLOG
 931         TLB_LOCK
 932
 933         mfsrr0  %r31                    /* faulting address */
 934
 935         /*
 936          * Save MAS0-MAS2 registers. There might be another tlb miss during pte
 937          * lookup overwriting current contents (which was hw filled).
 938          */
 939         mfspr   %r29, SPR_MAS0
 940         mfspr   %r28, SPR_MAS1
 941         mfspr   %r27, SPR_MAS2
 942
 943         mfsrr1  %r21
 944         mtcr    %r21
 945
 946         /* check MSR[PR] */
 947         bt      17, search_user_pmap
 948         b       search_kernel_pmap
 949
 950
 951         .globl  interrupt_vector_top
 952 interrupt_vector_top:
 953
 954 /*****************************************************************************
 955  * Debug interrupt
 956  ****************************************************************************/
 957 INTERRUPT(int_debug)
 958         STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1)
 959         FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG)
 960         bl      int_debug_int
 961         FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1)
 962         rfci
 963
 964 INTERRUPT(int_debug_ed)
 965         STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_DSRR0, SPR_DSRR1)
 966         FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG)
 967         bl      int_debug_int
 968         FRAME_LEAVE(SPR_DSRR0, SPR_DSRR1)
 969         rfdi
 970         /* .long 0x4c00004e */
 971
 972 /* Internal helper for debug interrupt handling. */
 973 /* Common code between e500v1/v2 and e500mc-based cores. */
 974 int_debug_int:
 975         mflr    %r14
 976         GET_CPUINFO(%r3)
 977         LOAD    %r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0)(%r3)
 978         bl      0f
 979         ADDR(interrupt_vector_base-.)
 980         ADDR(interrupt_vector_top-.)
 981 0:      mflr    %r5
 982         LOAD    %r4,0(%r5)      /* interrupt_vector_base in r4 */
 983         add     %r4,%r4,%r5
 984         CMPL    cr0, %r3, %r4
 985         blt     trap_common
 986         LOAD    %r4,WORD_SIZE(%r5)      /* interrupt_vector_top in r4 */
 987         add     %r4,%r4,%r5
 988         addi    %r4,%r4,4
 989         CMPL    cr0, %r3, %r4
 990         bge     trap_common
 991         /* Disable single-stepping for the interrupt handlers. */
 992         LOAD    %r3, FRAME_SRR1+CALLSIZE(%r1);
 993         rlwinm  %r3, %r3, 0, 23, 21
 994         STORE   %r3, FRAME_SRR1+CALLSIZE(%r1);
 995         /* Restore srr0 and srr1 as they could have been clobbered. */
 996         GET_CPUINFO(%r4)
 997         LOAD    %r3, (PC_BOOKE_CRITSAVE+BOOKE_CRITSAVE_SRR0)(%r4);
 998         mtspr   SPR_SRR0, %r3
 999         LOAD    %r4, (PC_BOOKE_CRITSAVE+BOOKE_CRITSAVE_SRR1)(%r4);
1000         mtspr   SPR_SRR1, %r4
1001         mtlr    %r14
1002         blr
1003
1004 /*****************************************************************************
1005  * Common trap code
1006  ****************************************************************************/
1007 trap_common:
1008         /* Call C trap dispatcher */
1009         GET_TOCBASE(%r2)
1010         addi    %r3, %r1, CALLSIZE
1011         bl      CNAME(powerpc_interrupt)
1012         TOC_RESTORE
1013
1014         .globl  CNAME(trapexit)         /* exported for db_backtrace use */
1015 CNAME(trapexit):
1016         /* disable interrupts */
1017         wrteei  0
1018
1019         /* Test AST pending - makes sense for user process only */
1020         LOAD    %r5, FRAME_SRR1+CALLSIZE(%r1)
1021         mtcr    %r5
1022         bf      17, 1f
1023
1024         GET_CPUINFO(%r3)
1025         LOAD    %r4, PC_CURTHREAD(%r3)
1026         lwz     %r4, TD_AST(%r4)
1027         cmpwi   %r4, 0
1028         beq     1f
1029
1030         /* re-enable interrupts before calling ast() */
1031         wrteei  1
1032
1033         addi    %r3, %r1, CALLSIZE
1034         bl      CNAME(ast)
1035         TOC_RESTORE
1036         .globl  CNAME(asttrapexit)      /* db_backtrace code sentinel #2 */
1037 CNAME(asttrapexit):
1038         b       trapexit                /* test ast ret value ? */
1039 1:
1040         FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
1041         rfi
1042
1043
1044 #if defined(KDB)
1045 /*
1046  * Deliberate entry to dbtrap
1047  */
1048         /* .globl       CNAME(breakpoint)*/
1049 ASENTRY_NOPROF(breakpoint)
1050         mtsprg1 %r1
1051         mfmsr   %r3
1052         mtsrr1  %r3
1053         li      %r4, ~(PSL_EE | PSL_ME)@l
1054         oris    %r4, %r4, ~(PSL_EE | PSL_ME)@h
1055         and     %r3, %r3, %r4
1056         mtmsr   %r3                     /* disable interrupts */
1057         isync
1058         GET_CPUINFO(%r3)
1059         STORE   %r30, (PC_DBSAVE+CPUSAVE_R30)(%r3)
1060         STORE   %r31, (PC_DBSAVE+CPUSAVE_R31)(%r3)
1061
1062         mflr    %r31
1063         mtsrr0  %r31
1064
1065         mfspr   %r30, SPR_DEAR
1066         mfspr   %r31, SPR_ESR
1067         STORE   %r30, (PC_DBSAVE+CPUSAVE_BOOKE_DEAR)(%r3)
1068         STORE   %r31, (PC_DBSAVE+CPUSAVE_BOOKE_ESR)(%r3)
1069
1070         mfsrr0  %r30
1071         mfsrr1  %r31
1072         STORE   %r30, (PC_DBSAVE+CPUSAVE_SRR0)(%r3)
1073         STORE   %r31, (PC_DBSAVE+CPUSAVE_SRR1)(%r3)
1074         isync
1075
1076         mfcr    %r30
1077
1078 /*
1079  * Now the kdb trap catching code.
1080  */
1081 dbtrap:
1082         FRAME_SETUP(SPR_SPRG1, PC_DBSAVE, EXC_DEBUG)
1083 /* Call C trap code: */
1084         GET_TOCBASE(%r2)
1085         addi    %r3, %r1, CALLSIZE
1086         bl      CNAME(db_trap_glue)
1087         TOC_RESTORE
1088         or.     %r3, %r3, %r3
1089         bne     dbleave
1090 /* This wasn't for KDB, so switch to real trap: */
1091         b       trap_common
1092
1093 dbleave:
1094         FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
1095         rfi
1096 ASEND(breakpoint)
1097 #endif /* KDB */
1098
1099 #ifdef SMP
1100 ENTRY(tlb_lock)
1101         GET_CPUINFO(%r5)
1102         LOAD    %r5, PC_CURTHREAD(%r5)
1103 1:      LOADX   %r4, 0, %r3
1104         CMPI    %r4, TLB_UNLOCKED
1105         bne     1b
1106         STOREX  %r5, 0, %r3
1107         bne-    1b
1108         isync
1109         msync
1110         blr
1111 END(tlb_lock)
1112
1113 ENTRY(tlb_unlock)
1114         isync
1115         msync
1116         li      %r4, TLB_UNLOCKED
1117         STORE   %r4, 0(%r3)
1118         isync
1119         msync
1120         blr
1121 END(tlb_unlock)
1122
1123 /*
1124  * TLB miss spin locks. For each CPU we have a reservation granule (32 bytes);
1125  * only a single word from this granule will actually be used as a spin lock
1126  * for mutual exclusion between TLB miss handler and pmap layer that
1127  * manipulates page table contents.
1128  */
1129         .data
1130         .align  5
1131 GLOBAL(tlb0_miss_locks)
1132         .space  RES_GRANULE * MAXCPU
1133 #endif