sys/ia64/ia64/exception.S

   1 /*-
   2  * Copyright (c) 2003,2004 Marcel Moolenaar
   3  * Copyright (c) 2000 Doug Rabson
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <machine/asm.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_xtrace.h"
  32
  33 #include <machine/pte.h>
  34 #include <assym.s>
  35
  36 /*
  37  * Nested TLB restart tokens. These are used by the
  38  * nested TLB handler for jumping back to the code
  39  * where the nested TLB was caused.
  40  */
  41 #define NTLBRT_SAVE     0x12c12c
  42 #define NTLBRT_RESTORE  0x12c12d
  43
  44 /*
  45  * ar.k7 = kernel memory stack
  46  * ar.k6 = kernel register stack
  47  * ar.k5 = EPC gateway page
  48  * ar.k4 = PCPU data
  49  */
  50
  51         .section .ivt.data, "aw"
  52
  53         .global ia64_kptdir
  54 ia64_kptdir:    data8   0
  55
  56 #ifdef EXCEPTION_TRACING
  57
  58         .global xtrace, xhead
  59 xtrace: .space  1024*5*8
  60 xhead:  data8   xtrace
  61
  62 #define XTRACE(offset)                          \
  63 {       .mmi ;                                  \
  64         mov     r24=ar.itc ;                    \
  65         mov     r25=cr.iip ;                    \
  66         mov     r27=offset ;                    \
  67 } ;                                             \
  68 {       .mlx ;                                  \
  69         mov     r28=cr.ifa ;                    \
  70         movl    r29=xhead ;;                    \
  71 } ;                                             \
  72 {       .mmi ;                                  \
  73         ld8     r29=[r29] ;;                    \
  74         st8     [r29]=r24,8 ;                   \
  75         nop     0 ;;                            \
  76 } ;                                             \
  77 {       .mmi ;                                  \
  78         st8     [r29]=r27,8 ;;                  \
  79         mov     r24=cr.isr ;                    \
  80         add     r27=8,r29 ;;                    \
  81 } ;                                             \
  82 {       .mmi ;                                  \
  83         st8     [r29]=r25,16 ;;                 \
  84         st8     [r27]=r28,16 ;                  \
  85         mov     r25=pr ;;                       \
  86 } ;                                             \
  87 {       .mlx ;                                  \
  88         st8     [r29]=r24 ;                     \
  89         movl    r28=xhead ;;                    \
  90 } ;                                             \
  91 {       .mii ;                                  \
  92         cmp.eq  p15,p0=r27,r28 ;                \
  93         addl    r29=1024*5*8,r0 ;;              \
  94 (p15)   sub     r27=r28,r29 ;;                  \
  95 } ;                                             \
  96 {       .mmi ;                                  \
  97         st8     [r28]=r27 ;                     \
  98         nop     0 ;                             \
  99         mov     pr=r25,0x1ffff ;;               \
 100 }
 101
 102 #else
 103
 104 #define XTRACE(offset)
 105
 106 #endif
 107
 108         .section .ivt.text, "ax"
 109
 110 /*
 111  * exception_save: save interrupted state
 112  *
 113  * Arguments:
 114  *      r16     address of bundle that contains the branch. The
 115  *              return address will be the next bundle.
 116  *      r17     the value to save as ifa in the trapframe. This
 117  *              normally is cr.ifa, but some interruptions set
 118  *              set cr.iim and not cr.ifa.
 119  *
 120  * Returns:
 121  *      p15     interrupted from user stack
 122  *      p14     interrupted from kernel stack
 123  *      p13     interrupted from user backing store
 124  *      p12     interrupted from kernel backing store
 125  *      p11     interrupts were enabled
 126  *      p10     interrupts were disabled
 127  */
 128 ENTRY_NOPROFILE(exception_save, 0)
 129 {       .mii
 130         mov             r20=ar.unat
 131         extr.u          r31=sp,61,3
 132         mov             r18=pr
 133         ;;
 134 }
 135 {       .mmi
 136         cmp.le          p14,p15=IA64_VM_MINKERN_REGION,r31
 137         ;;
 138 (p15)   mov             r23=ar.k7               // kernel memory stack
 139 (p14)   mov             r23=sp
 140         ;;
 141 }
 142 {       .mii
 143         mov             r21=ar.rsc
 144         add             r30=-SIZEOF_TRAPFRAME,r23
 145         ;;
 146         dep             r30=0,r30,0,10
 147         ;;
 148 }
 149 {       .mmi
 150         mov             ar.rsc=0
 151         mov             r22=cr.iip
 152         addl            r29=NTLBRT_SAVE,r0      // 22-bit restart token.
 153         ;;
 154 }
 155
 156         /*
 157          * We have a 1KB aligned trapframe, pointed to by r30. We can't
 158          * reliably write to the trapframe using virtual addressing, due
 159          * to the fact that TC entries we depend on can be removed by:
 160          * 1.  ptc.g instructions issued by other threads/cores/CPUs, or
 161          * 2.  TC modifications in another thread on the same core.
 162          * When our TC entry gets removed, we get nested TLB faults and
 163          * since no state is saved, we can only deal with those when
 164          * explicitly coded and expected.
 165          * As such, we switch to physical addressing and account for the
 166          * fact that the tpa instruction can cause a nested TLB fault.
 167          * Since the data nested TLB fault does not preserve any state,
 168          * we have to be careful what we clobber. Consequently, we have
 169          * to be careful what we use here. Below a list of registers that
 170          * are considered alive:
 171          *      r16,r17=arguments
 172          *      r18=pr, r19=length, r20=unat, r21=rsc, r22=iip, r23=TOS
 173          *      r29=restart token
 174          *      r30=trapframe pointers
 175          *      p14,p15=memory stack switch
 176          */
 177 exception_save_restart:
 178         tpa             r24=r30                 // Nested TLB fault possible
 179         sub             r19=r23,r30
 180         nop             0
 181         ;;
 182
 183         rsm             psr.dt
 184         add             r29=16,r19              // Clobber restart token
 185         mov             r30=r24
 186         ;;
 187         srlz.d
 188         add             r31=8,r24
 189         ;;
 190
 191         // r18=pr, r19=length, r20=unat, r21=rsc, r22=iip, r23=TOS
 192         // r29=delta
 193 {       .mmi
 194         st8             [r30]=r19,16            // length
 195         st8             [r31]=r0,16             // flags
 196         ;;
 197 }
 198 {       .mmi
 199         st8.spill       [r30]=sp,16             // sp
 200         st8             [r31]=r20,16            // unat
 201         sub             sp=r23,r29
 202         ;;
 203 }
 204 {       .mmi
 205         mov             r19=ar.rnat
 206         mov             r20=ar.bspstore
 207         mov             r23=rp
 208         ;;
 209 }
 210         // r18=pr, r19=rnat, r20=bspstore, r21=rsc, r22=iip, r23=rp
 211         // r24=pfs
 212 {       .mmi
 213         st8             [r30]=r23,16            // rp
 214         st8             [r31]=r18,16            // pr
 215         mov             r24=ar.pfs
 216         ;;
 217 }
 218 {       .mmb
 219         st8             [r30]=r24,16            // pfs
 220         st8             [r31]=r20,16            // bspstore
 221         cover
 222         ;;
 223 }
 224 {       .mmi
 225         mov             r18=ar.fpsr
 226         mov             r23=cr.ipsr
 227         extr.u          r24=r20,61,3
 228         ;;
 229 }
 230         // r18=fpsr, r19=rnat, r20=bspstore, r21=rsc, r22=iip, r23=ipsr
 231 {       .mmi
 232         st8             [r30]=r19,16            // rnat
 233         st8             [r31]=r0,16             // __spare
 234         cmp.le          p12,p13=IA64_VM_MINKERN_REGION,r24
 235         ;;
 236 }
 237 {       .mmi
 238         st8.spill       [r30]=r13,16            // tp
 239         st8             [r31]=r21,16            // rsc
 240         tbit.nz         p11,p10=r23,14          // p11=interrupts enabled
 241         ;;
 242 }
 243 {       .mmi
 244 (p13)   mov             r21=ar.k6               // kernel register stack
 245         ;;
 246         st8             [r30]=r18,16            // fpsr
 247 (p13)   dep             r20=r20,r21,0,9         // align dirty registers
 248         ;;
 249 }
 250         // r19=rnat, r20=bspstore, r22=iip, r23=ipsr
 251 {       .mmi
 252         st8             [r31]=r23,16            // psr
 253 (p13)   mov             ar.bspstore=r20
 254         nop             0
 255         ;;
 256 }
 257 {       .mmi
 258 (p13)   mov             ar.rnat=r19
 259         mov             r18=ar.bsp
 260         nop             0
 261         ;;
 262 }
 263 {       .mmi
 264         mov             r19=cr.ifs
 265         st8.spill       [r30]=gp,16             // gp
 266         sub             r18=r18,r20
 267         ;;
 268 }
 269         // r18=ndirty, r19=ifs, r22=iip
 270 {       .mmi
 271         st8             [r31]=r18,16            // ndirty
 272         st8             [r30]=r19,16            // cfm
 273         nop             0
 274         ;;
 275 }
 276 {       .mmi
 277         mov             r18=cr.isr
 278         st8             [r31]=r22,16            // iip
 279         add             r29=16,r30
 280         ;;
 281 }
 282 {       .mmi
 283         st8             [r30]=r17,24            // ifa
 284         st8             [r31]=r18,24            // isr
 285         nop             0
 286         ;;
 287 }
 288 {       .mmi
 289         .mem.offset     0,0
 290         st8.spill       [r30]=r2,16             // r2
 291         .mem.offset     8,0
 292         st8.spill       [r31]=r3,16             // r3
 293         add             r2=9*8,r29
 294         ;;
 295 }
 296 {       .mmi
 297         .mem.offset     0,0
 298         st8.spill       [r30]=r8,16             // r8
 299         .mem.offset     8,0
 300         st8.spill       [r31]=r9,16             // r9
 301         add             r3=8,r2
 302         ;;
 303 }
 304 {       .mmi
 305         .mem.offset     0,0
 306         st8.spill       [r30]=r10,16            // r10
 307         .mem.offset     8,0
 308         st8.spill       [r31]=r11,16            // r11
 309         add             r8=16,r16
 310         ;;
 311 }
 312 {       .mmi
 313         .mem.offset     0,0
 314         st8.spill       [r30]=r14               // r14
 315         .mem.offset     8,0
 316         st8.spill       [r31]=r15               // r15
 317         mov             r9=r29
 318 }
 319 {       .mmb
 320         mov             r10=ar.csd
 321         mov             r11=ar.ssd
 322         bsw.1
 323         ;;
 324 }
 325 {       .mmi
 326         .mem.offset     0,0
 327         st8.spill       [r2]=r16,16             // r16
 328         .mem.offset     8,0
 329         st8.spill       [r3]=r17,16             // r17
 330         mov             r14=b6
 331         ;;
 332 }
 333 {       .mmi
 334         .mem.offset     0,0
 335         st8.spill       [r2]=r18,16             // r18
 336         .mem.offset     8,0
 337         st8.spill       [r3]=r19,16             // r19
 338         mov             r15=b7
 339         ;;
 340 }
 341 {       .mmi
 342         .mem.offset     0,0
 343         st8.spill       [r2]=r20,16             // r20
 344         .mem.offset     8,0
 345         st8.spill       [r3]=r21,16             // r21
 346         mov             b7=r8
 347         ;;
 348 }
 349 {       .mmi
 350         .mem.offset     0,0
 351         st8.spill       [r2]=r22,16             // r22
 352         .mem.offset     8,0
 353         st8.spill       [r3]=r23,16             // r23
 354         ;;
 355 }
 356
 357         .mem.offset     0,0
 358         st8.spill       [r2]=r24,16             // r24
 359         .mem.offset     8,0
 360         st8.spill       [r3]=r25,16             // r25
 361         ;;
 362         .mem.offset     0,0
 363         st8.spill       [r2]=r26,16             // r26
 364         .mem.offset     8,0
 365         st8.spill       [r3]=r27,16             // r27
 366         ;;
 367         .mem.offset     0,0
 368         st8.spill       [r2]=r28,16             // r28
 369         .mem.offset     8,0
 370         st8.spill       [r3]=r29,16             // r29
 371         ;;
 372         .mem.offset     0,0
 373         st8.spill       [r2]=r30,16             // r30
 374         .mem.offset     8,0
 375         st8.spill       [r3]=r31,16             // r31
 376         ;;
 377
 378 {       .mmi
 379         st8             [r2]=r14,16             // b6
 380         mov             r17=ar.unat
 381         nop             0
 382         ;;
 383 }
 384 {       .mmi
 385         st8             [r3]=r15,16             // b7
 386         mov             r16=ar.ccv
 387         nop             0
 388         ;;
 389 }
 390 {       .mmi
 391         st8             [r2]=r16,16             // ccv
 392         st8             [r3]=r10,16             // csd
 393         nop             0
 394         ;;
 395 }
 396 {       .mmi
 397         st8             [r2]=r11,24             // ssd
 398         st8             [r9]=r17
 399         nop             0
 400         ;;
 401 }
 402
 403         stf.spill       [r3]=f6,32              // f6
 404         stf.spill       [r2]=f7,32              // f7
 405         ;;
 406         stf.spill       [r3]=f8,32              // f8
 407         stf.spill       [r2]=f9,32              // f9
 408         ;;
 409         stf.spill       [r3]=f10,32             // f10
 410         stf.spill       [r2]=f11,32             // f11
 411         ;;
 412         stf.spill       [r3]=f12,32             // f12
 413         stf.spill       [r2]=f13,32             // f13
 414         ;;
 415         stf.spill       [r3]=f14                // f14
 416         stf.spill       [r2]=f15                // f15
 417         ;;
 418 {       .mmi
 419         mov             ar.rsc=3
 420         mov             r13=ar.k4
 421         nop             0
 422         ;;
 423 }
 424 {       .mlx
 425         ssm             psr.dt|psr.ic|psr.dfh
 426         movl            gp=__gp
 427         ;;
 428 }
 429 {       .mib
 430         srlz.d
 431         nop             0
 432         br.sptk         b7
 433         ;;
 434 }
 435 END(exception_save)
 436
 437 /*
 438  * exception_restore:   restore interrupted state
 439  *
 440  * Arguments:
 441  *      sp+16   trapframe pointer
 442  */
 443 ENTRY_NOPROFILE(exception_restore, 0)
 444 {       .mmi
 445         rsm             psr.i
 446         add             sp=16,sp
 447         nop             0
 448         ;;
 449 }
 450
 451         // The next instruction can fault. Let it be...
 452         tpa             r9=sp
 453         ;;
 454         rsm             psr.dt|psr.ic
 455         add             r8=SIZEOF_SPECIAL+16,r9
 456         ;;
 457         srlz.d
 458         add             r2=SIZEOF_TRAPFRAME-16,r9
 459         add             r3=SIZEOF_TRAPFRAME-32,r9
 460         ;;
 461
 462 {       .mmi
 463         ldf.fill        f15=[r2],-32            // f15
 464         ldf.fill        f14=[r3],-32            // f14
 465         nop             0
 466         ;;
 467 }
 468 {       .mmi
 469         ldf.fill        f13=[r2],-32            // f13
 470         ldf.fill        f12=[r3],-32            // f12
 471         nop             0
 472         ;;
 473 }
 474 {       .mmi
 475         ldf.fill        f11=[r2],-32            // f11
 476         ldf.fill        f10=[r3],-32            // f10
 477         nop             0
 478         ;;
 479 }
 480 {       .mmi
 481         ldf.fill        f9=[r2],-32             // f9
 482         ldf.fill        f8=[r3],-32             // f8
 483         nop             0
 484         ;;
 485 }
 486 {       .mmi
 487         ldf.fill        f7=[r2],-24             // f7
 488         ldf.fill        f6=[r3],-16             // f6
 489         nop             0
 490         ;;
 491 }
 492 {       .mmi
 493         ld8             r8=[r8]                 // unat (after)
 494         ;;
 495         mov             ar.unat=r8
 496         nop             0
 497         ;;
 498 }
 499
 500         ld8             r10=[r2],-16            // ssd
 501         ld8             r11=[r3],-16            // csd
 502         ;;
 503         mov             ar.ssd=r10
 504         mov             ar.csd=r11
 505
 506         ld8             r14=[r2],-16            // ccv
 507         ld8             r15=[r3],-16            // b7
 508         ;;
 509
 510 {       .mmi
 511         mov             ar.ccv=r14
 512         ld8             r8=[r2],-16             // b6
 513         mov             b7=r15
 514         ;;
 515 }
 516 {       .mmi
 517         ld8.fill        r31=[r3],-16            // r31
 518         ld8.fill        r30=[r2],-16            // r30
 519         mov             b6=r8
 520         ;;
 521 }
 522
 523         ld8.fill        r29=[r3],-16            // r29
 524         ld8.fill        r28=[r2],-16            // r28
 525         ;;
 526         ld8.fill        r27=[r3],-16            // r27
 527         ld8.fill        r26=[r2],-16            // r26
 528         ;;
 529         ld8.fill        r25=[r3],-16            // r25
 530         ld8.fill        r24=[r2],-16            // r24
 531         ;;
 532         ld8.fill        r23=[r3],-16            // r23
 533         ld8.fill        r22=[r2],-16            // r22
 534         ;;
 535         ld8.fill        r21=[r3],-16            // r21
 536         ld8.fill        r20=[r2],-16            // r20
 537         ;;
 538         ld8.fill        r19=[r3],-16            // r19
 539         ld8.fill        r18=[r2],-16            // r18
 540         ;;
 541
 542 {       .mmb
 543         ld8.fill        r17=[r3],-16            // r17
 544         ld8.fill        r16=[r2],-16            // r16
 545         bsw.0
 546         ;;
 547 }
 548 {       .mii
 549         ld8             r16=[r9]                // tf_length
 550         add             r31=16,r9
 551         add             r30=24,r9
 552 }
 553 {       .mmi
 554         ld8.fill        r15=[r3],-16            // r15
 555         ld8.fill        r14=[r2],-16            // r14
 556         nop             0
 557         ;;
 558 }
 559 {       .mmi
 560         ld8.fill        r11=[r3],-16            // r11
 561         ld8.fill        r10=[r2],-16            // r10
 562         add             r16=r16,sp              // ar.k7
 563         ;;
 564 }
 565 {       .mmi
 566         ld8.fill        r9=[r3],-16             // r9
 567         ld8.fill        r8=[r2],-16             // r8
 568         nop             0
 569         ;;
 570 }
 571 {       .mmi
 572         ld8.fill        r3=[r3]                 // r3
 573         ld8.fill        r2=[r2]                 // r2
 574         nop             0
 575         ;;
 576 }
 577
 578         ld8.fill        sp=[r31],16             // sp
 579         ld8             r17=[r30],16            // unat
 580         ;;
 581         ld8             r29=[r31],16            // rp
 582         ld8             r18=[r30],16            // pr
 583         ;;
 584         ld8             r28=[r31],16            // pfs
 585         ld8             r20=[r30],24            // bspstore
 586         mov             rp=r29
 587         ;;
 588         ld8             r21=[r31],24            // rnat
 589         mov             ar.pfs=r28
 590         ;;
 591         ld8.fill        r26=[r30],16            // tp
 592         ld8             r22=[r31],16            // rsc
 593         ;;
 594
 595 {       .mmi
 596         ld8             r23=[r30],16            // fpsr
 597         ld8             r24=[r31],16            // psr
 598         extr.u          r28=r20,61,3
 599         ;;
 600 }
 601 {       .mmi
 602         ld8.fill        r1=[r30],16             // gp
 603         ld8             r27=[r31],16            // ndirty
 604         cmp.le          p14,p15=IA64_VM_MINKERN_REGION,r28
 605         ;;
 606 }
 607 {       .mmi
 608         ld8             r25=[r30]               // cfm
 609         ld8             r19=[r31]               // ip
 610         nop             0
 611         ;;
 612 }
 613 {       .mii
 614         // Switch register stack
 615         alloc           r30=ar.pfs,0,0,0,0      // discard current frame
 616         shl             r31=r27,16              // value for ar.rsc
 617 (p15)   mov             r13=r26
 618         ;;
 619 }
 620         // The loadrs can fault if the backing store is not currently
 621         // mapped. We assured forward progress by getting everything we
 622         // need from the trapframe so that we don't care if the CPU
 623         // purges that translation when it needs to insert a new one for
 624         // the backing store.
 625 {       .mmi
 626         mov             ar.rsc=r31              // setup for loadrs
 627         mov             ar.k7=r16
 628         addl            r29=NTLBRT_RESTORE,r0   // 22-bit restart token
 629         ;;
 630 }
 631
 632         ssm             psr.dt
 633         ;;
 634         srlz.d
 635
 636 exception_restore_restart:
 637 {       .mmi
 638         mov             r30=ar.bspstore
 639         ;;
 640         loadrs                                  // load user regs
 641         mov             r29=0                   // Clobber restart token
 642         ;;
 643 }
 644 {       .mmi
 645         mov             r31=ar.bspstore
 646         ;;
 647         mov             ar.bspstore=r20
 648         dep             r31=0,r31,0,13          // 8KB aligned
 649         ;;
 650 }
 651 {       .mmi
 652         mov             ar.k6=r31
 653         mov             ar.rnat=r21
 654         nop             0
 655         ;;
 656 }
 657 {       .mmi
 658         mov             ar.unat=r17
 659         mov             cr.iip=r19
 660         nop             0
 661 }
 662 {       .mmi
 663         mov             cr.ipsr=r24
 664         mov             cr.ifs=r25
 665         mov             pr=r18,0x1ffff
 666         ;;
 667 }
 668 {       .mmb
 669         mov             ar.rsc=r22
 670         mov             ar.fpsr=r23
 671         rfi
 672         ;;
 673 }
 674 END(exception_restore)
 675
 676 /*
 677  * Call exception_save_regs to preserve the interrupted state in a
 678  * trapframe. Note that we don't use a call instruction because we
 679  * must be careful not to lose track of the RSE state. We then call
 680  * trap() with the value of _n_ as an argument to handle the
 681  * exception. We arrange for trap() to return to exception_restore
 682  * which will restore the interrupted state before executing an rfi to
 683  * resume it.
 684  */
 685 #define CALL(_func_, _n_, _ifa_)                \
 686 {       .mib ;                                  \
 687         mov             r17=_ifa_ ;             \
 688         mov             r16=ip ;                \
 689         br.sptk         exception_save ;;       \
 690 } ;                                             \
 691 {       .mmi ;                                  \
 692         alloc           r15=ar.pfs,0,0,2,0 ;;   \
 693 (p11)   ssm             psr.i ;                 \
 694         mov             out0=_n_ ;;             \
 695 } ;                                             \
 696 {       .mib ;                                  \
 697 (p11)   srlz.d ;                                \
 698         add             out1=16,sp ;            \
 699         br.call.sptk    rp=_func_ ;;            \
 700 } ;                                             \
 701 {       .mib ;                                  \
 702         nop             0 ;                     \
 703         nop             0 ;                     \
 704         br.sptk         exception_restore ;;    \
 705 }
 706
 707 #define IVT_ENTRY(name, offset)                 \
 708         .org    ia64_vector_table + offset;     \
 709         .global ivt_##name;                     \
 710         .proc   ivt_##name;                     \
 711         .prologue;                              \
 712         .unwabi @svr4, 'I';                     \
 713         .save   rp, r0;                         \
 714         .body;                                  \
 715 ivt_##name:                                     \
 716         XTRACE(offset)
 717
 718 #define IVT_END(name)                           \
 719         .endp   ivt_##name
 720
 721 #ifdef COMPAT_FREEBSD32
 722 #define IA32_TRAP       ia32_trap
 723 #else
 724 #define IA32_TRAP       trap
 725 #endif
 726
 727 /*
 728  * The IA64 Interrupt Vector Table (IVT) contains 20 slots with 64
 729  * bundles per vector and 48 slots with 16 bundles per vector.
 730  */
 731
 732         .section .ivt, "ax"
 733
 734         .align  32768
 735         .global ia64_vector_table
 736         .size   ia64_vector_table, 32768
 737 ia64_vector_table:
 738
 739 IVT_ENTRY(VHPT_Translation, 0x0000)
 740         CALL(trap, 0, cr.ifa)
 741 IVT_END(VHPT_Translation)
 742
 743 IVT_ENTRY(Instruction_TLB, 0x0400)
 744         mov     r16=cr.ifa
 745         mov     r17=pr
 746         ;;
 747         thash   r18=r16
 748         ttag    r19=r16
 749         ;;
 750         add     r21=16,r18              // tag
 751         add     r20=24,r18              // collision chain
 752         ;;
 753         ld8     r21=[r21]               // check VHPT tag
 754         ld8     r20=[r20]               // bucket head
 755         ;;
 756         cmp.ne  p15,p0=r21,r19
 757 (p15)   br.dpnt.few 1f
 758         ;;
 759         ld8     r21=[r18]               // read pte
 760         ;;
 761         itc.i   r21                     // insert pte
 762         mov     pr=r17,0x1ffff
 763         ;;
 764         rfi                             // done
 765         ;;
 766 1:      rsm     psr.dt                  // turn off data translations
 767         dep     r20=0,r20,61,3          // convert vhpt ptr to physical
 768         ;;
 769         srlz.d                          // serialize
 770         ld8     r20=[r20]               // first entry
 771         ;;
 772 2:      cmp.eq  p15,p0=r0,r20           // done?
 773 (p15)   br.cond.spnt.few 9f             // bail if done
 774         ;;
 775         add     r21=16,r20              // tag location
 776         ;;
 777         ld8     r21=[r21]               // read tag
 778         ;;
 779         cmp.ne  p15,p0=r21,r19          // compare tags
 780 (p15)   br.cond.sptk.few 3f             // if not, read next in chain
 781         ;;
 782         ld8     r21=[r20]               // read pte
 783         mov     r22=PTE_ACCESSED
 784         ;;
 785         or      r21=r21,r22
 786         ;;
 787         st8     [r20]=r21,8
 788         ;;
 789         ld8     r22=[r20]               // read rest of pte
 790         ;;
 791         dep     r18=0,r18,61,3          // convert vhpt ptr to physical
 792         ;;
 793         add     r20=16,r18              // address of tag
 794         ;;
 795         ld8.acq r23=[r20]               // read old tag
 796         ;;
 797         dep     r23=-1,r23,63,1         // set ti bit
 798         ;;
 799         st8.rel [r20]=r23               // store old tag + ti
 800         ;;
 801         mf                              // make sure everyone sees
 802         ;;
 803         st8     [r18]=r21,8             // store pte
 804         ;;
 805         st8     [r18]=r22,8
 806         ;;
 807         st8.rel [r18]=r19               // store new tag
 808         ;;
 809         itc.i   r21                     // and place in TLB
 810         ssm     psr.dt
 811         ;;
 812         srlz.d
 813         mov     pr=r17,0x1ffff          // restore predicates
 814         rfi
 815         ;;
 816 3:      add     r20=24,r20              // next in chain
 817         ;;
 818         ld8     r20=[r20]               // read chain
 819         br.sptk 2b                      // loop
 820         ;;
 821 9:      ssm     psr.dt
 822         mov     pr=r17,0x1ffff          // restore predicates
 823         ;;
 824         srlz.d
 825         ;;
 826         CALL(trap, 20, cr.ifa)          // Page Not Present trap
 827 IVT_END(Instruction_TLB)
 828
 829 IVT_ENTRY(Data_TLB, 0x0800)
 830         mov     r16=cr.ifa
 831         mov     r17=pr
 832         ;;
 833         thash   r18=r16
 834         ttag    r19=r16
 835         ;;
 836         add     r21=16,r18              // tag
 837         add     r20=24,r18              // collision chain
 838         ;;
 839         ld8     r21=[r21]               // check VHPT tag
 840         ld8     r20=[r20]               // bucket head
 841         ;;
 842         cmp.ne  p15,p0=r21,r19
 843 (p15)   br.dpnt.few 1f
 844         ;;
 845         ld8     r21=[r18]               // read pte
 846         ;;
 847         itc.d   r21                     // insert pte
 848         mov     pr=r17,0x1ffff
 849         ;;
 850         rfi                             // done
 851         ;;
 852 1:      rsm     psr.dt                  // turn off data translations
 853         dep     r20=0,r20,61,3          // convert vhpt ptr to physical
 854         ;;
 855         srlz.d                          // serialize
 856         ld8     r20=[r20]               // first entry
 857         ;;
 858 2:      cmp.eq  p15,p0=r0,r20           // done?
 859 (p15)   br.cond.spnt.few 9f             // bail if done
 860         ;;
 861         add     r21=16,r20              // tag location
 862         ;;
 863         ld8     r21=[r21]               // read tag
 864         ;;
 865         cmp.ne  p15,p0=r21,r19          // compare tags
 866 (p15)   br.cond.sptk.few 3f             // if not, read next in chain
 867         ;;
 868         ld8     r21=[r20]               // read pte
 869         mov     r22=PTE_ACCESSED
 870         ;;
 871         or      r21=r21,r22
 872         ;;
 873         st8     [r20]=r21,8
 874         ;;
 875         ld8     r22=[r20]               // read rest of pte
 876         ;;
 877         dep     r18=0,r18,61,3          // convert vhpt ptr to physical
 878         ;;
 879         add     r20=16,r18              // address of tag
 880         ;;
 881         ld8.acq r23=[r20]               // read old tag
 882         ;;
 883         dep     r23=-1,r23,63,1         // set ti bit
 884         ;;
 885         st8.rel [r20]=r23               // store old tag + ti
 886         ;;
 887         mf                              // make sure everyone sees
 888         ;;
 889         st8     [r18]=r21,8             // store pte
 890         ;;
 891         st8     [r18]=r22,8
 892         ;;
 893         st8.rel [r18]=r19               // store new tag
 894         ;;
 895         itc.d   r21                     // and place in TLB
 896         ssm     psr.dt
 897         ;;
 898         srlz.d
 899         mov     pr=r17,0x1ffff          // restore predicates
 900         rfi
 901         ;;
 902 3:      add     r20=24,r20              // next in chain
 903         ;;
 904         ld8     r20=[r20]               // read chain
 905         br.sptk 2b                      // loop
 906         ;;
 907 9:      ssm     psr.dt
 908         mov     pr=r17,0x1ffff          // restore predicates
 909         ;;
 910         srlz.d
 911         ;;
 912         CALL(trap, 20, cr.ifa)          // Page Not Present trap
 913 IVT_END(Data_TLB)
 914
 915 IVT_ENTRY(Alternate_Instruction_TLB, 0x0c00)
 916         mov     r16=cr.ifa              // where did it happen
 917         mov     r18=pr                  // save predicates
 918         ;;
 919         extr.u  r17=r16,61,3            // get region number
 920         mov     r19=PTE_PRESENT+PTE_ACCESSED+PTE_DIRTY+PTE_PL_KERN+PTE_AR_RWX
 921         ;;
 922         cmp.eq  p13,p0=IA64_PBVM_RR,r17         // RR4?
 923 (p13)   br.cond.sptk.few        4f
 924         ;;
 925         cmp.ge  p13,p0=5,r17            // RR0-RR5?
 926         cmp.eq  p14,p15=7,r17           // RR7?
 927 (p13)   br.cond.spnt.few        9f
 928         ;;
 929 (p14)   add     r19=PTE_MA_WB,r19
 930 (p15)   add     r19=PTE_MA_UC,r19
 931         dep     r17=0,r16,50,14         // clear bits above PPN
 932         ;;
 933 1:      dep     r16=r19,r17,0,12        // put pte bits in 0..11
 934         ;;
 935         itc.i   r16
 936         mov     pr=r18,0x1ffff          // restore predicates
 937         ;;
 938         rfi
 939         ;;
 940 4:
 941         add     r19=PTE_MA_WB,r19
 942         movl    r17=IA64_PBVM_BASE
 943         ;;
 944         sub     r17=r16,r17
 945         movl    r16=IA64_PBVM_PGTBL
 946         ;;
 947         extr.u  r17=r17,IA64_PBVM_PAGE_SHIFT,61-IA64_PBVM_PAGE_SHIFT
 948         ;;
 949         shladd  r16=r17,3,r16
 950         ;;
 951         ld8     r17=[r16]
 952         br.sptk 1b
 953         ;;
 954 9:      mov     pr=r18,0x1ffff          // restore predicates
 955         CALL(trap, 3, cr.ifa)
 956 IVT_END(Alternate_Instruction_TLB)
 957
 958 IVT_ENTRY(Alternate_Data_TLB, 0x1000)
 959         mov     r16=cr.ifa              // where did it happen
 960         mov     r18=pr                  // save predicates
 961         ;;
 962         extr.u  r17=r16,61,3            // get region number
 963         mov     r19=PTE_PRESENT+PTE_ACCESSED+PTE_DIRTY+PTE_PL_KERN+PTE_AR_RWX
 964         ;;
 965         cmp.eq  p13,p0=IA64_PBVM_RR,r17         // RR4?
 966 (p13)   br.cond.sptk.few        4f
 967         ;;
 968         cmp.ge  p13,p0=5,r17            // RR0-RR5?
 969         cmp.eq  p14,p15=7,r17           // RR7?
 970 (p13)   br.cond.spnt.few        9f
 971         ;;
 972 (p14)   add     r19=PTE_MA_WB,r19
 973 (p15)   add     r19=PTE_MA_UC,r19
 974         dep     r17=0,r16,50,14         // clear bits above PPN
 975         ;;
 976 1:      dep     r16=r19,r17,0,12        // put pte bits in 0..11
 977         ;;
 978         itc.d   r16
 979         mov     pr=r18,0x1ffff          // restore predicates
 980         ;;
 981         rfi
 982         ;;
 983 4:
 984         add     r19=PTE_MA_WB,r19
 985         movl    r17=IA64_PBVM_BASE
 986         ;;
 987         sub     r17=r16,r17
 988         movl    r16=IA64_PBVM_PGTBL
 989         ;;
 990         extr.u  r17=r17,IA64_PBVM_PAGE_SHIFT,61-IA64_PBVM_PAGE_SHIFT
 991         ;;
 992         shladd  r16=r17,3,r16
 993         ;;
 994         ld8     r17=[r16]
 995         br.sptk 1b
 996         ;;
 997 9:      mov     pr=r18,0x1ffff          // restore predicates
 998         CALL(trap, 4, cr.ifa)
 999 IVT_END(Alternate_Data_TLB)
1000
1001 IVT_ENTRY(Data_Nested_TLB, 0x1400)
1002         // See exception_save_restart and exception_restore_restart for the
1003         // contexts that may cause a data nested TLB. We can only use the
1004         // banked general registers and predicates, but don't use:
1005         //      p14 & p15       -       Set in exception save
1006         //      r16 & r17       -       Arguments to exception save
1007         //      r30             -       Faulting address (modulo page size)
1008         // We assume r30 has the virtual addresses that relate to the data
1009         // nested TLB fault. The address does not have to be exact, as long
1010         // as it's in the same page. We use physical addressing to avoid
1011         // double nested faults. Since all virtual addresses we encounter
1012         // here are direct mapped region 7 addresses, we have no problem
1013         // constructing physical addresses.
1014
1015 {       .mmi
1016         mov             cr.ifa=r30
1017         mov             r26=rr[r30]
1018         extr.u          r27=r30,61,3
1019         ;;
1020 }
1021 {       .mii
1022         nop             0
1023         dep             r26=0,r26,0,2
1024         cmp.eq          p12,p13=7,r27
1025         ;;
1026 }
1027 {       .mii
1028         mov             cr.itir=r26
1029 (p12)   dep             r28=0,r30,61,3
1030 (p13)   extr.u          r28=r30,3*PAGE_SHIFT-8, PAGE_SHIFT-3    // dir L0 index
1031         ;;
1032 }
1033 {       .mlx
1034 (p12)   add             r28=PTE_PRESENT+PTE_ACCESSED+PTE_DIRTY+PTE_PL_KERN+PTE_AR_RWX+PTE_MA_WB,r28
1035 (p13)   movl            r27=ia64_kptdir
1036         ;;
1037 }
1038 {       .mib
1039 (p13)   ld8             r27=[r27]
1040 (p13)   extr.u          r26=r30,2*PAGE_SHIFT-5, PAGE_SHIFT-3    // dir L1 index
1041 (p12)   br.cond.spnt.few 1f
1042         ;;
1043 }
1044 {       .mmi
1045         rsm             psr.dt
1046         ;;
1047         srlz.d
1048         dep             r27=0,r27,61,3
1049         ;;
1050 }
1051 {       .mmi
1052         shladd          r27=r28,3,r27
1053         ;;
1054         ld8             r27=[r27]                               // dir L1 page
1055         extr.u          r28=r30,PAGE_SHIFT,PAGE_SHIFT-5         // pte index
1056         ;;
1057 }
1058 {       .mii
1059         shladd          r27=r26,3,r27
1060         shl             r28=r28,5
1061         ;;
1062         dep             r27=0,r27,61,3
1063         ;;
1064 }
1065         ld8             r27=[r27]                               // pte page
1066         ;;
1067         add             r27=r28,r27
1068         ;;
1069         dep             r27=0,r27,61,3
1070         ;;
1071         ld8             r28=[r27]                               // pte
1072         ;;
1073         or              r28=PTE_DIRTY+PTE_ACCESSED,r28
1074         ;;
1075         st8             [r27]=r28
1076         ;;
1077         ssm             psr.dt
1078         ;;
1079 1:
1080 {       .mmi
1081         itc.d           r28
1082         ;;
1083         addl            r26=NTLBRT_SAVE,r0
1084         addl            r27=NTLBRT_RESTORE,r0
1085         ;;
1086 }
1087 {       .mmi
1088         srlz.d
1089         cmp.eq          p12,p0=r29,r26
1090         cmp.eq          p13,p0=r29,r27
1091         ;;
1092 }
1093 {       .mbb
1094         nop             0
1095 (p12)   br.cond.sptk.few        exception_save_restart
1096 (p13)   br.cond.sptk.few        exception_restore_restart
1097         ;;
1098 }
1099
1100 {       .mlx
1101         mov             r26=ar.bsp
1102         movl            r29=kstack
1103         ;;
1104 }
1105 {       .mlx
1106         mov             r28=sp
1107         movl            r27=kstack_top
1108         ;;
1109 }
1110 {       .mmi
1111         add             sp=-16,r27
1112         ;;
1113         mov             r27=ar.bspstore
1114         nop             0
1115         ;;
1116 }
1117         mov             ar.rsc=0
1118         dep             r29=r27,r29,0,9
1119         ;;
1120         mov             ar.bspstore=r29
1121         ;;
1122         CALL(trap, 5, r30)
1123 IVT_END(Data_Nested_TLB)
1124
1125 IVT_ENTRY(Instruction_Key_Miss, 0x1800)
1126         CALL(trap, 6, cr.ifa)
1127 IVT_END(Instruction_Key_Miss)
1128
1129 IVT_ENTRY(Data_Key_Miss, 0x1c00)
1130         CALL(trap, 7, cr.ifa)
1131 IVT_END(Data_Key_Miss)
1132
1133 IVT_ENTRY(Dirty_Bit, 0x2000)
1134         mov     r16=cr.ifa
1135         mov     r17=pr
1136         ;;
1137         thash   r18=r16
1138         ;;
1139         ttag    r19=r16
1140         add     r20=24,r18              // collision chain
1141         ;;
1142         ld8     r20=[r20]               // bucket head
1143         ;;
1144         rsm     psr.dt                  // turn off data translations
1145         dep     r20=0,r20,61,3          // convert vhpt ptr to physical
1146         ;;
1147         srlz.d                          // serialize
1148         ld8     r20=[r20]               // first entry
1149         ;;
1150 1:      cmp.eq  p15,p0=r0,r20           // done?
1151 (p15)   br.cond.spnt.few 9f             // bail if done
1152         ;;
1153         add     r21=16,r20              // tag location
1154         ;;
1155         ld8     r21=[r21]               // read tag
1156         ;;
1157         cmp.ne  p15,p0=r21,r19          // compare tags
1158 (p15)   br.cond.sptk.few 2f             // if not, read next in chain
1159         ;;
1160         ld8     r21=[r20]               // read pte
1161         mov     r22=PTE_DIRTY+PTE_ACCESSED
1162         ;;
1163         or      r21=r22,r21             // set dirty & access bit
1164         ;;
1165         st8     [r20]=r21,8             // store back
1166         ;;
1167         ld8     r22=[r20]               // read rest of pte
1168         ;;
1169         dep     r18=0,r18,61,3          // convert vhpt ptr to physical
1170         ;;
1171         add     r20=16,r18              // address of tag
1172         ;;
1173         ld8.acq r23=[r20]               // read old tag
1174         ;;
1175         dep     r23=-1,r23,63,1         // set ti bit
1176         ;;
1177         st8.rel [r20]=r23               // store old tag + ti
1178         ;;
1179         mf                              // make sure everyone sees
1180         ;;
1181         st8     [r18]=r21,8             // store pte
1182         ;;
1183         st8     [r18]=r22,8
1184         ;;
1185         st8.rel [r18]=r19               // store new tag
1186         ;;
1187         itc.d   r21                     // and place in TLB
1188         ssm     psr.dt
1189         ;;
1190         srlz.d
1191         mov     pr=r17,0x1ffff          // restore predicates
1192         rfi
1193         ;;
1194 2:      add     r20=24,r20              // next in chain
1195         ;;
1196         ld8     r20=[r20]               // read chain
1197         br.sptk 1b                      // loop
1198         ;;
1199 9:      ssm     psr.dt
1200         mov     pr=r17,0x1ffff          // restore predicates
1201         ;;
1202         srlz.d
1203         ;;
1204         CALL(trap, 8, cr.ifa)                   // die horribly
1205 IVT_END(Dirty_Bit)
1206
1207 IVT_ENTRY(Instruction_Access_Bit, 0x2400)
1208         mov     r16=cr.ifa
1209         mov     r17=pr
1210         ;;
1211         thash   r18=r16
1212         ;;
1213         ttag    r19=r16
1214         add     r20=24,r18              // collision chain
1215         ;;
1216         ld8     r20=[r20]               // bucket head
1217         ;;
1218         rsm     psr.dt                  // turn off data translations
1219         dep     r20=0,r20,61,3          // convert vhpt ptr to physical
1220         ;;
1221         srlz.d                          // serialize
1222         ld8     r20=[r20]               // first entry
1223         ;;
1224 1:      cmp.eq  p15,p0=r0,r20           // done?
1225 (p15)   br.cond.spnt.few 9f             // bail if done
1226         ;;
1227         add     r21=16,r20              // tag location
1228         ;;
1229         ld8     r21=[r21]               // read tag
1230         ;;
1231         cmp.ne  p15,p0=r21,r19          // compare tags
1232 (p15)   br.cond.sptk.few 2f             // if not, read next in chain
1233         ;;
1234         ld8     r21=[r20]               // read pte
1235         mov     r22=PTE_ACCESSED
1236         ;;
1237         or      r21=r22,r21             // set accessed bit
1238         ;;
1239         st8     [r20]=r21,8             // store back
1240         ;;
1241         ld8     r22=[r20]               // read rest of pte
1242         ;;
1243         dep     r18=0,r18,61,3          // convert vhpt ptr to physical
1244         ;;
1245         add     r20=16,r18              // address of tag
1246         ;;
1247         ld8.acq r23=[r20]               // read old tag
1248         ;;
1249         dep     r23=-1,r23,63,1         // set ti bit
1250         ;;
1251         st8.rel [r20]=r23               // store old tag + ti
1252         ;;
1253         mf                              // make sure everyone sees
1254         ;;
1255         st8     [r18]=r21,8             // store pte
1256         ;;
1257         st8     [r18]=r22,8
1258         ;;
1259         st8.rel [r18]=r19               // store new tag
1260         ;;
1261         itc.i   r21                     // and place in TLB
1262         ssm     psr.dt
1263         ;;
1264         srlz.d
1265         mov     pr=r17,0x1ffff          // restore predicates
1266         rfi                             // walker will retry the access
1267         ;;
1268 2:      add     r20=24,r20              // next in chain
1269         ;;
1270         ld8     r20=[r20]               // read chain
1271         br.sptk 1b                      // loop
1272         ;;
1273 9:      ssm     psr.dt
1274         mov     pr=r17,0x1ffff          // restore predicates
1275         ;;
1276         srlz.d
1277         ;;
1278         CALL(trap, 9, cr.ifa)
1279 IVT_END(Instruction_Access_Bit)
1280
1281 IVT_ENTRY(Data_Access_Bit, 0x2800)
1282         mov     r16=cr.ifa
1283         mov     r17=pr
1284         ;;
1285         thash   r18=r16
1286         ;;
1287         ttag    r19=r16
1288         add     r20=24,r18              // collision chain
1289         ;;
1290         ld8     r20=[r20]               // bucket head
1291         ;;
1292         rsm     psr.dt                  // turn off data translations
1293         dep     r20=0,r20,61,3          // convert vhpt ptr to physical
1294         ;;
1295         srlz.d                          // serialize
1296         ld8     r20=[r20]               // first entry
1297         ;;
1298 1:      cmp.eq  p15,p0=r0,r20           // done?
1299 (p15)   br.cond.spnt.few 9f             // bail if done
1300         ;;
1301         add     r21=16,r20              // tag location
1302         ;;
1303         ld8     r21=[r21]               // read tag
1304         ;;
1305         cmp.ne  p15,p0=r21,r19          // compare tags
1306 (p15)   br.cond.sptk.few 2f             // if not, read next in chain
1307         ;;
1308         ld8     r21=[r20]               // read pte
1309         mov     r22=PTE_ACCESSED
1310         ;;
1311         or      r21=r22,r21             // set accessed bit
1312         ;;
1313         st8     [r20]=r21,8             // store back
1314         ;;
1315         ld8     r22=[r20]               // read rest of pte
1316         ;;
1317         dep     r18=0,r18,61,3          // convert vhpt ptr to physical
1318         ;;
1319         add     r20=16,r18              // address of tag
1320         ;;
1321         ld8.acq r23=[r20]               // read old tag
1322         ;;
1323         dep     r23=-1,r23,63,1         // set ti bit
1324         ;;
1325         st8.rel [r20]=r23               // store old tag + ti
1326         ;;
1327         mf                              // make sure everyone sees
1328         ;;
1329         st8     [r18]=r21,8             // store pte
1330         ;;
1331         st8     [r18]=r22,8
1332         ;;
1333         st8.rel [r18]=r19               // store new tag
1334         ;;
1335         itc.d   r21                     // and place in TLB
1336         ssm     psr.dt
1337         ;;
1338         srlz.d
1339         mov     pr=r17,0x1ffff          // restore predicates
1340         rfi                             // walker will retry the access
1341         ;;
1342 2:      add     r20=24,r20              // next in chain
1343         ;;
1344         ld8     r20=[r20]               // read chain
1345         br.sptk 1b                      // loop
1346         ;;
1347 9:      ssm     psr.dt
1348         mov     pr=r17,0x1ffff          // restore predicates
1349         ;;
1350         srlz.d
1351         ;;
1352         CALL(trap, 10, cr.ifa)
1353 IVT_END(Data_Access_Bit)
1354
1355 IVT_ENTRY(Break_Instruction, 0x2c00)
1356 {       .mib
1357         mov             r17=cr.iim
1358         mov             r16=ip
1359         br.sptk         exception_save
1360         ;;
1361 }
1362 {       .mmi
1363         alloc           r15=ar.pfs,0,0,2,0
1364         ;;
1365 (p11)   ssm             psr.i
1366         mov             out0=11
1367         ;;
1368 }
1369 {       .mmi
1370         flushrs
1371         ;;
1372 (p11)   srlz.d
1373         add             out1=16,sp
1374 }
1375 {       .mib
1376         nop             0
1377         nop             0
1378         br.call.sptk    rp=trap
1379         ;;
1380 }
1381 {       .mib
1382         nop             0
1383         nop             0
1384         br.sptk         exception_restore
1385         ;;
1386 }
1387 IVT_END(Break_Instruction)
1388
1389 IVT_ENTRY(External_Interrupt, 0x3000)
1390 {       .mib
1391         mov             r17=ar.itc      // Put the ITC in the trapframe.
1392         mov             r16=ip
1393         br.sptk         exception_save
1394         ;;
1395 }
1396 {       .mmi
1397         alloc           r15=ar.pfs,0,0,1,0
1398         nop             0
1399         nop             0
1400         ;;
1401 }
1402 {       .mib
1403         add             out0=16,sp
1404         nop             0
1405         br.call.sptk    rp=ia64_handle_intr
1406         ;;
1407 }
1408 {       .mib
1409         nop             0
1410         nop             0
1411         br.sptk         exception_restore
1412         ;;
1413 }
1414 IVT_END(External_Interrupt)
1415
1416 IVT_ENTRY(Reserved_3400, 0x3400)
1417         CALL(trap, 13, cr.ifa)
1418 IVT_END(Reserved_3400)
1419
1420 IVT_ENTRY(Reserved_3800, 0x3800)
1421         CALL(trap, 14, cr.ifa)
1422 IVT_END(Reserved_3800)
1423
1424 IVT_ENTRY(Reserved_3c00, 0x3c00)
1425         CALL(trap, 15, cr.ifa)
1426 IVT_END(Reserved_3c00)
1427
1428 IVT_ENTRY(Reserved_4000, 0x4000)
1429         CALL(trap, 16, cr.ifa)
1430 IVT_END(Reserved_4000)
1431
1432 IVT_ENTRY(Reserved_4400, 0x4400)
1433         CALL(trap, 17, cr.ifa)
1434 IVT_END(Reserved_4400)
1435
1436 IVT_ENTRY(Reserved_4800, 0x4800)
1437         CALL(trap, 18, cr.ifa)
1438 IVT_END(Reserved_4800)
1439
1440 IVT_ENTRY(Reserved_4c00, 0x4c00)
1441         CALL(trap, 19, cr.ifa)
1442 IVT_END(Reserved_4c00)
1443
1444 IVT_ENTRY(Page_Not_Present, 0x5000)
1445         CALL(trap, 20, cr.ifa)
1446 IVT_END(Page_Not_Present)
1447
1448 IVT_ENTRY(Key_Permission, 0x5100)
1449         CALL(trap, 21, cr.ifa)
1450 IVT_END(Key_Permission)
1451
1452 IVT_ENTRY(Instruction_Access_Rights, 0x5200)
1453         CALL(trap, 22, cr.ifa)
1454 IVT_END(Instruction_Access_Rights)
1455
1456 IVT_ENTRY(Data_Access_Rights, 0x5300)
1457         CALL(trap, 23, cr.ifa)
1458 IVT_END(Data_Access_Rights)
1459
1460 IVT_ENTRY(General_Exception, 0x5400)
1461         CALL(trap, 24, cr.ifa)
1462 IVT_END(General_Exception)
1463
1464 IVT_ENTRY(Disabled_FP_Register, 0x5500)
1465         CALL(trap, 25, cr.ifa)
1466 IVT_END(Disabled_FP_Register)
1467
1468 IVT_ENTRY(NaT_Consumption, 0x5600)
1469         CALL(trap, 26, cr.ifa)
1470 IVT_END(NaT_Consumption)
1471
1472 IVT_ENTRY(Speculation, 0x5700)
1473         CALL(trap, 27, cr.iim)
1474 IVT_END(Speculation)
1475
1476 IVT_ENTRY(Reserved_5800, 0x5800)
1477         CALL(trap, 28, cr.ifa)
1478 IVT_END(Reserved_5800)
1479
1480 IVT_ENTRY(Debug, 0x5900)
1481         CALL(trap, 29, cr.ifa)
1482 IVT_END(Debug)
1483
1484 IVT_ENTRY(Unaligned_Reference, 0x5a00)
1485         CALL(trap, 30, cr.ifa)
1486 IVT_END(Unaligned_Reference)
1487
1488 IVT_ENTRY(Unsupported_Data_Reference, 0x5b00)
1489         CALL(trap, 31, cr.ifa)
1490 IVT_END(Unsupported_Data_Reference)
1491
1492 IVT_ENTRY(Floating_Point_Fault, 0x5c00)
1493         CALL(trap, 32, cr.ifa)
1494 IVT_END(Floating_Point_Fault)
1495
1496 IVT_ENTRY(Floating_Point_Trap, 0x5d00)
1497         CALL(trap, 33, cr.ifa)
1498 IVT_END(Floating_Point_Trap)
1499
1500 IVT_ENTRY(Lower_Privilege_Transfer_Trap, 0x5e00)
1501         CALL(trap, 34, cr.ifa)
1502 IVT_END(Lower_Privilege_Transfer_Trap)
1503
1504 IVT_ENTRY(Taken_Branch_Trap, 0x5f00)
1505         CALL(trap, 35, cr.ifa)
1506 IVT_END(Taken_Branch_Trap)
1507
1508 IVT_ENTRY(Single_Step_Trap, 0x6000)
1509         CALL(trap, 36, cr.ifa)
1510 IVT_END(Single_Step_Trap)
1511
1512 IVT_ENTRY(Reserved_6100, 0x6100)
1513         CALL(trap, 37, cr.ifa)
1514 IVT_END(Reserved_6100)
1515
1516 IVT_ENTRY(Reserved_6200, 0x6200)
1517         CALL(trap, 38, cr.ifa)
1518 IVT_END(Reserved_6200)
1519
1520 IVT_ENTRY(Reserved_6300, 0x6300)
1521         CALL(trap, 39, cr.ifa)
1522 IVT_END(Reserved_6300)
1523
1524 IVT_ENTRY(Reserved_6400, 0x6400)
1525         CALL(trap, 40, cr.ifa)
1526 IVT_END(Reserved_6400)
1527
1528 IVT_ENTRY(Reserved_6500, 0x6500)
1529         CALL(trap, 41, cr.ifa)
1530 IVT_END(Reserved_6500)
1531
1532 IVT_ENTRY(Reserved_6600, 0x6600)
1533         CALL(trap, 42, cr.ifa)
1534 IVT_END(Reserved_6600)
1535
1536 IVT_ENTRY(Reserved_6700, 0x6700)
1537         CALL(trap, 43, cr.ifa)
1538 IVT_END(Reserved_6700)
1539
1540 IVT_ENTRY(Reserved_6800, 0x6800)
1541         CALL(trap, 44, cr.ifa)
1542 IVT_END(Reserved_6800)
1543
1544 IVT_ENTRY(IA_32_Exception, 0x6900)
1545         CALL(IA32_TRAP, 45, cr.ifa)
1546 IVT_END(IA_32_Exception)
1547
1548 IVT_ENTRY(IA_32_Intercept, 0x6a00)
1549         CALL(IA32_TRAP, 46, cr.iim)
1550 IVT_END(IA_32_Intercept)
1551
1552 IVT_ENTRY(IA_32_Interrupt, 0x6b00)
1553         CALL(IA32_TRAP, 47, cr.ifa)
1554 IVT_END(IA_32_Interrupt)
1555
1556 IVT_ENTRY(Reserved_6c00, 0x6c00)
1557         CALL(trap, 48, cr.ifa)
1558 IVT_END(Reserved_6c00)
1559
1560 IVT_ENTRY(Reserved_6d00, 0x6d00)
1561         CALL(trap, 49, cr.ifa)
1562 IVT_END(Reserved_6d00)
1563
1564 IVT_ENTRY(Reserved_6e00, 0x6e00)
1565         CALL(trap, 50, cr.ifa)
1566 IVT_END(Reserved_6e00)
1567
1568 IVT_ENTRY(Reserved_6f00, 0x6f00)
1569         CALL(trap, 51, cr.ifa)
1570 IVT_END(Reserved_6f00)
1571
1572 IVT_ENTRY(Reserved_7000, 0x7000)
1573         CALL(trap, 52, cr.ifa)
1574 IVT_END(Reserved_7000)
1575
1576 IVT_ENTRY(Reserved_7100, 0x7100)
1577         CALL(trap, 53, cr.ifa)
1578 IVT_END(Reserved_7100)
1579
1580 IVT_ENTRY(Reserved_7200, 0x7200)
1581         CALL(trap, 54, cr.ifa)
1582 IVT_END(Reserved_7200)
1583
1584 IVT_ENTRY(Reserved_7300, 0x7300)
1585         CALL(trap, 55, cr.ifa)
1586 IVT_END(Reserved_7300)
1587
1588 IVT_ENTRY(Reserved_7400, 0x7400)
1589         CALL(trap, 56, cr.ifa)
1590 IVT_END(Reserved_7400)
1591
1592 IVT_ENTRY(Reserved_7500, 0x7500)
1593         CALL(trap, 57, cr.ifa)
1594 IVT_END(Reserved_7500)
1595
1596 IVT_ENTRY(Reserved_7600, 0x7600)
1597         CALL(trap, 58, cr.ifa)
1598 IVT_END(Reserved_7600)
1599
1600 IVT_ENTRY(Reserved_7700, 0x7700)
1601         CALL(trap, 59, cr.ifa)
1602 IVT_END(Reserved_7700)
1603
1604 IVT_ENTRY(Reserved_7800, 0x7800)
1605         CALL(trap, 60, cr.ifa)
1606 IVT_END(Reserved_7800)
1607
1608 IVT_ENTRY(Reserved_7900, 0x7900)
1609         CALL(trap, 61, cr.ifa)
1610 IVT_END(Reserved_7900)
1611
1612 IVT_ENTRY(Reserved_7a00, 0x7a00)
1613         CALL(trap, 62, cr.ifa)
1614 IVT_END(Reserved_7a00)
1615
1616 IVT_ENTRY(Reserved_7b00, 0x7b00)
1617         CALL(trap, 63, cr.ifa)
1618 IVT_END(Reserved_7b00)
1619
1620 IVT_ENTRY(Reserved_7c00, 0x7c00)
1621         CALL(trap, 64, cr.ifa)
1622 IVT_END(Reserved_7c00)
1623
1624 IVT_ENTRY(Reserved_7d00, 0x7d00)
1625         CALL(trap, 65, cr.ifa)
1626 IVT_END(Reserved_7d00)
1627
1628 IVT_ENTRY(Reserved_7e00, 0x7e00)
1629         CALL(trap, 66, cr.ifa)
1630 IVT_END(Reserved_7e00)
1631
1632 IVT_ENTRY(Reserved_7f00, 0x7f00)
1633         CALL(trap, 67, cr.ifa)
1634 IVT_END(Reserved_7f00)