2 * Copyright (c) 2003,2004 Marcel Moolenaar
3 * Copyright (c) 2000 Doug Rabson
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 #include <machine/asm.h>
29 __FBSDID("$FreeBSD$");
31 #include "opt_xtrace.h"
33 #include <machine/pte.h>
37 * Nested TLB restart tokens. These are used by the
38 * nested TLB handler for jumping back to the code
39 * where the nested TLB was caused.
41 #define NTLBRT_SAVE 0x12c12c
42 #define NTLBRT_RESTORE 0x12c12d
45 * ar.k7 = kernel memory stack
46 * ar.k6 = kernel register stack
47 * ar.k5 = EPC gateway page
51 .section .ivt.data, "aw"
56 #ifdef EXCEPTION_TRACING
59 xtrace: .space 1024*5*8
62 #define XTRACE(offset) \
92 cmp.eq p15,p0=r27,r28 ; \
93 addl r29=1024*5*8,r0 ;; \
94 (p15) sub r27=r28,r29 ;; \
99 mov pr=r25,0x1ffff ;; \
104 #define XTRACE(offset)
108 .section .ivt.text, "ax"
111 * exception_save: save interrupted state
114 * r16 address of bundle that contains the branch. The
115 * return address will be the next bundle.
116 * r17 the value to save as ifa in the trapframe. This
117 * normally is cr.ifa, but some interruptions set
118 * set cr.iim and not cr.ifa.
121 * p15 interrupted from user stack
122 * p14 interrupted from kernel stack
123 * p13 interrupted from user backing store
124 * p12 interrupted from kernel backing store
125 * p11 interrupts were enabled
126 * p10 interrupts were disabled
128 ENTRY_NOPROFILE(exception_save, 0)
136 cmp.le p14,p15=IA64_VM_MINKERN_REGION,r31
138 (p15) mov r23=ar.k7 // kernel memory stack
144 add r30=-SIZEOF_TRAPFRAME,r23
152 addl r29=NTLBRT_SAVE,r0 // 22-bit restart token.
157 * We have a 1KB aligned trapframe, pointed to by r30. We can't
158 * reliably write to the trapframe using virtual addressing, due
159 * to the fact that TC entries we depend on can be removed by:
160 * 1. ptc.g instructions issued by other threads/cores/CPUs, or
161 * 2. TC modifications in another thread on the same core.
162 * When our TC entry gets removed, we get nested TLB faults and
163 * since no state is saved, we can only deal with those when
164 * explicitly coded and expected.
165 * As such, we switch to physical addressing and account for the
166 * fact that the tpa instruction can cause a nested TLB fault.
167 * Since the data nested TLB fault does not preserve any state,
168 * we have to be careful what we clobber. Consequently, we have
169 * to be careful what we use here. Below a list of registers that
170 * are considered alive:
172 * r18=pr, r19=length, r20=unat, r21=rsc, r22=iip, r23=TOS
174 * r30=trapframe pointers
175 * p14,p15=memory stack switch
177 exception_save_restart:
178 tpa r24=r30 // Nested TLB fault possible
184 add r29=16,r19 // Clobber restart token
191 // r18=pr, r19=length, r20=unat, r21=rsc, r22=iip, r23=TOS
194 st8 [r30]=r19,16 // length
195 st8 [r31]=r0,16 // flags
199 st8.spill [r30]=sp,16 // sp
200 st8 [r31]=r20,16 // unat
210 // r18=pr, r19=rnat, r20=bspstore, r21=rsc, r22=iip, r23=rp
213 st8 [r30]=r23,16 // rp
214 st8 [r31]=r18,16 // pr
219 st8 [r30]=r24,16 // pfs
220 st8 [r31]=r20,16 // bspstore
230 // r18=fpsr, r19=rnat, r20=bspstore, r21=rsc, r22=iip, r23=ipsr
232 st8 [r30]=r19,16 // rnat
233 st8 [r31]=r0,16 // __spare
234 cmp.le p12,p13=IA64_VM_MINKERN_REGION,r24
238 st8.spill [r30]=r13,16 // tp
239 st8 [r31]=r21,16 // rsc
240 tbit.nz p11,p10=r23,14 // p11=interrupts enabled
244 (p13) mov r21=ar.k6 // kernel register stack
246 st8 [r30]=r18,16 // fpsr
247 (p13) dep r20=r20,r21,0,9 // align dirty registers
250 // r19=rnat, r20=bspstore, r22=iip, r23=ipsr
252 st8 [r31]=r23,16 // psr
253 (p13) mov ar.bspstore=r20
258 (p13) mov ar.rnat=r19
265 st8.spill [r30]=gp,16 // gp
269 // r18=ndirty, r19=ifs, r22=iip
271 st8 [r31]=r18,16 // ndirty
272 st8 [r30]=r19,16 // cfm
278 st8 [r31]=r22,16 // iip
283 st8 [r30]=r17,24 // ifa
284 st8 [r31]=r18,24 // isr
290 st8.spill [r30]=r2,16 // r2
292 st8.spill [r31]=r3,16 // r3
298 st8.spill [r30]=r8,16 // r8
300 st8.spill [r31]=r9,16 // r9
306 st8.spill [r30]=r10,16 // r10
308 st8.spill [r31]=r11,16 // r11
314 st8.spill [r30]=r14 // r14
316 st8.spill [r31]=r15 // r15
327 st8.spill [r2]=r16,16 // r16
329 st8.spill [r3]=r17,16 // r17
335 st8.spill [r2]=r18,16 // r18
337 st8.spill [r3]=r19,16 // r19
343 st8.spill [r2]=r20,16 // r20
345 st8.spill [r3]=r21,16 // r21
351 st8.spill [r2]=r22,16 // r22
353 st8.spill [r3]=r23,16 // r23
358 st8.spill [r2]=r24,16 // r24
360 st8.spill [r3]=r25,16 // r25
363 st8.spill [r2]=r26,16 // r26
365 st8.spill [r3]=r27,16 // r27
368 st8.spill [r2]=r28,16 // r28
370 st8.spill [r3]=r29,16 // r29
373 st8.spill [r2]=r30,16 // r30
375 st8.spill [r3]=r31,16 // r31
379 st8 [r2]=r14,16 // b6
385 st8 [r3]=r15,16 // b7
391 st8 [r2]=r16,16 // ccv
392 st8 [r3]=r10,16 // csd
397 st8 [r2]=r11,24 // ssd
403 stf.spill [r3]=f6,32 // f6
404 stf.spill [r2]=f7,32 // f7
406 stf.spill [r3]=f8,32 // f8
407 stf.spill [r2]=f9,32 // f9
409 stf.spill [r3]=f10,32 // f10
410 stf.spill [r2]=f11,32 // f11
412 stf.spill [r3]=f12,32 // f12
413 stf.spill [r2]=f13,32 // f13
415 stf.spill [r3]=f14 // f14
416 stf.spill [r2]=f15 // f15
425 ssm psr.dt|psr.ic|psr.dfh
438 * exception_restore: restore interrupted state
441 * sp+16 trapframe pointer
443 ENTRY_NOPROFILE(exception_restore, 0)
451 // The next instruction can fault. Let it be...
455 add r8=SIZEOF_SPECIAL+16,r9
458 add r2=SIZEOF_TRAPFRAME-16,r9
459 add r3=SIZEOF_TRAPFRAME-32,r9
463 ldf.fill f15=[r2],-32 // f15
464 ldf.fill f14=[r3],-32 // f14
469 ldf.fill f13=[r2],-32 // f13
470 ldf.fill f12=[r3],-32 // f12
475 ldf.fill f11=[r2],-32 // f11
476 ldf.fill f10=[r3],-32 // f10
481 ldf.fill f9=[r2],-32 // f9
482 ldf.fill f8=[r3],-32 // f8
487 ldf.fill f7=[r2],-24 // f7
488 ldf.fill f6=[r3],-16 // f6
493 ld8 r8=[r8] // unat (after)
500 ld8 r10=[r2],-16 // ssd
501 ld8 r11=[r3],-16 // csd
506 ld8 r14=[r2],-16 // ccv
507 ld8 r15=[r3],-16 // b7
512 ld8 r8=[r2],-16 // b6
517 ld8.fill r31=[r3],-16 // r31
518 ld8.fill r30=[r2],-16 // r30
523 ld8.fill r29=[r3],-16 // r29
524 ld8.fill r28=[r2],-16 // r28
526 ld8.fill r27=[r3],-16 // r27
527 ld8.fill r26=[r2],-16 // r26
529 ld8.fill r25=[r3],-16 // r25
530 ld8.fill r24=[r2],-16 // r24
532 ld8.fill r23=[r3],-16 // r23
533 ld8.fill r22=[r2],-16 // r22
535 ld8.fill r21=[r3],-16 // r21
536 ld8.fill r20=[r2],-16 // r20
538 ld8.fill r19=[r3],-16 // r19
539 ld8.fill r18=[r2],-16 // r18
543 ld8.fill r17=[r3],-16 // r17
544 ld8.fill r16=[r2],-16 // r16
549 ld8 r16=[r9] // tf_length
554 ld8.fill r15=[r3],-16 // r15
555 ld8.fill r14=[r2],-16 // r14
560 ld8.fill r11=[r3],-16 // r11
561 ld8.fill r10=[r2],-16 // r10
562 add r16=r16,sp // ar.k7
566 ld8.fill r9=[r3],-16 // r9
567 ld8.fill r8=[r2],-16 // r8
572 ld8.fill r3=[r3] // r3
573 ld8.fill r2=[r2] // r2
578 ld8.fill sp=[r31],16 // sp
579 ld8 r17=[r30],16 // unat
581 ld8 r29=[r31],16 // rp
582 ld8 r18=[r30],16 // pr
584 ld8 r28=[r31],16 // pfs
585 ld8 r20=[r30],24 // bspstore
588 ld8 r21=[r31],24 // rnat
591 ld8.fill r26=[r30],16 // tp
592 ld8 r22=[r31],16 // rsc
596 ld8 r23=[r30],16 // fpsr
597 ld8 r24=[r31],16 // psr
602 ld8.fill r1=[r30],16 // gp
603 ld8 r27=[r31],16 // ndirty
604 cmp.le p14,p15=IA64_VM_MINKERN_REGION,r28
614 // Switch register stack
615 alloc r30=ar.pfs,0,0,0,0 // discard current frame
616 shl r31=r27,16 // value for ar.rsc
620 // The loadrs can fault if the backing store is not currently
621 // mapped. We assured forward progress by getting everything we
622 // need from the trapframe so that we don't care if the CPU
623 // purges that translation when it needs to insert a new one for
624 // the backing store.
626 mov ar.rsc=r31 // setup for loadrs
628 addl r29=NTLBRT_RESTORE,r0 // 22-bit restart token
636 exception_restore_restart:
640 loadrs // load user regs
641 mov r29=0 // Clobber restart token
648 dep r31=0,r31,0,13 // 8KB aligned
674 END(exception_restore)
677 * Call exception_save_regs to preserve the interrupted state in a
678 * trapframe. Note that we don't use a call instruction because we
679 * must be careful not to lose track of the RSE state. We then call
680 * trap() with the value of _n_ as an argument to handle the
681 * exception. We arrange for trap() to return to exception_restore
682 * which will restore the interrupted state before executing an rfi to
685 #define CALL(_func_, _n_, _ifa_) \
689 br.sptk exception_save ;; \
692 alloc r15=ar.pfs,0,0,2,0 ;; \
699 br.call.sptk rp=_func_ ;; \
704 br.sptk exception_restore ;; \
707 #define IVT_ENTRY(name, offset) \
708 .org ia64_vector_table + offset; \
709 .global ivt_##name; \
712 .unwabi @svr4, 'I'; \
718 #define IVT_END(name) \
721 #ifdef COMPAT_FREEBSD32
722 #define IA32_TRAP ia32_trap
724 #define IA32_TRAP trap
728 * The IA64 Interrupt Vector Table (IVT) contains 20 slots with 64
729 * bundles per vector and 48 slots with 16 bundles per vector.
735 .global ia64_vector_table
736 .size ia64_vector_table, 32768
739 IVT_ENTRY(VHPT_Translation, 0x0000)
740 CALL(trap, 0, cr.ifa)
741 IVT_END(VHPT_Translation)
743 IVT_ENTRY(Instruction_TLB, 0x0400)
750 add r21=16,r18 // tag
751 add r20=24,r18 // collision chain
753 ld8 r21=[r21] // check VHPT tag
754 ld8 r20=[r20] // bucket head
756 cmp.ne p15,p0=r21,r19
759 ld8 r21=[r18] // read pte
761 itc.i r21 // insert pte
766 1: rsm psr.dt // turn off data translations
767 dep r20=0,r20,61,3 // convert vhpt ptr to physical
770 ld8 r20=[r20] // first entry
772 2: cmp.eq p15,p0=r0,r20 // done?
773 (p15) br.cond.spnt.few 9f // bail if done
775 add r21=16,r20 // tag location
777 ld8 r21=[r21] // read tag
779 cmp.ne p15,p0=r21,r19 // compare tags
780 (p15) br.cond.sptk.few 3f // if not, read next in chain
782 ld8 r21=[r20] // read pte
789 ld8 r22=[r20] // read rest of pte
791 dep r18=0,r18,61,3 // convert vhpt ptr to physical
793 add r20=16,r18 // address of tag
795 ld8.acq r23=[r20] // read old tag
797 dep r23=-1,r23,63,1 // set ti bit
799 st8.rel [r20]=r23 // store old tag + ti
801 mf // make sure everyone sees
803 st8 [r18]=r21,8 // store pte
807 st8.rel [r18]=r19 // store new tag
809 itc.i r21 // and place in TLB
813 mov pr=r17,0x1ffff // restore predicates
816 3: add r20=24,r20 // next in chain
818 ld8 r20=[r20] // read chain
822 mov pr=r17,0x1ffff // restore predicates
826 CALL(trap, 20, cr.ifa) // Page Not Present trap
827 IVT_END(Instruction_TLB)
829 IVT_ENTRY(Data_TLB, 0x0800)
836 add r21=16,r18 // tag
837 add r20=24,r18 // collision chain
839 ld8 r21=[r21] // check VHPT tag
840 ld8 r20=[r20] // bucket head
842 cmp.ne p15,p0=r21,r19
845 ld8 r21=[r18] // read pte
847 itc.d r21 // insert pte
852 1: rsm psr.dt // turn off data translations
853 dep r20=0,r20,61,3 // convert vhpt ptr to physical
856 ld8 r20=[r20] // first entry
858 2: cmp.eq p15,p0=r0,r20 // done?
859 (p15) br.cond.spnt.few 9f // bail if done
861 add r21=16,r20 // tag location
863 ld8 r21=[r21] // read tag
865 cmp.ne p15,p0=r21,r19 // compare tags
866 (p15) br.cond.sptk.few 3f // if not, read next in chain
868 ld8 r21=[r20] // read pte
875 ld8 r22=[r20] // read rest of pte
877 dep r18=0,r18,61,3 // convert vhpt ptr to physical
879 add r20=16,r18 // address of tag
881 ld8.acq r23=[r20] // read old tag
883 dep r23=-1,r23,63,1 // set ti bit
885 st8.rel [r20]=r23 // store old tag + ti
887 mf // make sure everyone sees
889 st8 [r18]=r21,8 // store pte
893 st8.rel [r18]=r19 // store new tag
895 itc.d r21 // and place in TLB
899 mov pr=r17,0x1ffff // restore predicates
902 3: add r20=24,r20 // next in chain
904 ld8 r20=[r20] // read chain
908 mov pr=r17,0x1ffff // restore predicates
912 CALL(trap, 20, cr.ifa) // Page Not Present trap
915 IVT_ENTRY(Alternate_Instruction_TLB, 0x0c00)
916 mov r16=cr.ifa // where did it happen
917 mov r18=pr // save predicates
919 extr.u r17=r16,61,3 // get region number
920 mov r19=PTE_PRESENT+PTE_ACCESSED+PTE_DIRTY+PTE_PL_KERN+PTE_AR_RWX
922 cmp.eq p13,p0=IA64_PBVM_RR,r17 // RR4?
923 (p13) br.cond.sptk.few 4f
925 cmp.ge p13,p0=5,r17 // RR0-RR5?
926 cmp.eq p14,p15=7,r17 // RR7?
927 (p13) br.cond.spnt.few 9f
929 (p14) add r19=PTE_MA_WB,r19
930 (p15) add r19=PTE_MA_UC,r19
931 dep r17=0,r16,50,14 // clear bits above PPN
933 1: dep r16=r19,r17,0,12 // put pte bits in 0..11
936 mov pr=r18,0x1ffff // restore predicates
941 add r19=PTE_MA_WB,r19
942 movl r17=IA64_PBVM_BASE
945 movl r16=IA64_PBVM_PGTBL
947 extr.u r17=r17,IA64_PBVM_PAGE_SHIFT,61-IA64_PBVM_PAGE_SHIFT
954 9: mov pr=r18,0x1ffff // restore predicates
955 CALL(trap, 3, cr.ifa)
956 IVT_END(Alternate_Instruction_TLB)
958 IVT_ENTRY(Alternate_Data_TLB, 0x1000)
959 mov r16=cr.ifa // where did it happen
960 mov r18=pr // save predicates
962 extr.u r17=r16,61,3 // get region number
963 mov r19=PTE_PRESENT+PTE_ACCESSED+PTE_DIRTY+PTE_PL_KERN+PTE_AR_RWX
965 cmp.eq p13,p0=IA64_PBVM_RR,r17 // RR4?
966 (p13) br.cond.sptk.few 4f
968 cmp.ge p13,p0=5,r17 // RR0-RR5?
969 cmp.eq p14,p15=7,r17 // RR7?
970 (p13) br.cond.spnt.few 9f
972 (p14) add r19=PTE_MA_WB,r19
973 (p15) add r19=PTE_MA_UC,r19
974 dep r17=0,r16,50,14 // clear bits above PPN
976 1: dep r16=r19,r17,0,12 // put pte bits in 0..11
979 mov pr=r18,0x1ffff // restore predicates
984 add r19=PTE_MA_WB,r19
985 movl r17=IA64_PBVM_BASE
988 movl r16=IA64_PBVM_PGTBL
990 extr.u r17=r17,IA64_PBVM_PAGE_SHIFT,61-IA64_PBVM_PAGE_SHIFT
997 9: mov pr=r18,0x1ffff // restore predicates
998 CALL(trap, 4, cr.ifa)
999 IVT_END(Alternate_Data_TLB)
1001 IVT_ENTRY(Data_Nested_TLB, 0x1400)
1002 // See exception_save_restart and exception_restore_restart for the
1003 // contexts that may cause a data nested TLB. We can only use the
1004 // banked general registers and predicates, but don't use:
1005 // p14 & p15 - Set in exception save
1006 // r16 & r17 - Arguments to exception save
1007 // r30 - Faulting address (modulo page size)
1008 // We assume r30 has the virtual addresses that relate to the data
1009 // nested TLB fault. The address does not have to be exact, as long
1010 // as it's in the same page. We use physical addressing to avoid
1011 // double nested faults. Since all virtual addresses we encounter
1012 // here are direct mapped region 7 addresses, we have no problem
1013 // constructing physical addresses.
1024 cmp.eq p12,p13=7,r27
1029 (p12) dep r28=0,r30,61,3
1030 (p13) extr.u r28=r30,3*PAGE_SHIFT-8, PAGE_SHIFT-3 // dir L0 index
1034 (p12) add r28=PTE_PRESENT+PTE_ACCESSED+PTE_DIRTY+PTE_PL_KERN+PTE_AR_RWX+PTE_MA_WB,r28
1035 (p13) movl r27=ia64_kptdir
1040 (p13) extr.u r26=r30,2*PAGE_SHIFT-5, PAGE_SHIFT-3 // dir L1 index
1041 (p12) br.cond.spnt.few 1f
1052 shladd r27=r28,3,r27
1054 ld8 r27=[r27] // dir L1 page
1055 extr.u r28=r30,PAGE_SHIFT,PAGE_SHIFT-5 // pte index
1059 shladd r27=r26,3,r27
1065 ld8 r27=[r27] // pte page
1071 ld8 r28=[r27] // pte
1073 or r28=PTE_DIRTY+PTE_ACCESSED,r28
1083 addl r26=NTLBRT_SAVE,r0
1084 addl r27=NTLBRT_RESTORE,r0
1089 cmp.eq p12,p0=r29,r26
1090 cmp.eq p13,p0=r29,r27
1095 (p12) br.cond.sptk.few exception_save_restart
1096 (p13) br.cond.sptk.few exception_restore_restart
1123 IVT_END(Data_Nested_TLB)
1125 IVT_ENTRY(Instruction_Key_Miss, 0x1800)
1126 CALL(trap, 6, cr.ifa)
1127 IVT_END(Instruction_Key_Miss)
1129 IVT_ENTRY(Data_Key_Miss, 0x1c00)
1130 CALL(trap, 7, cr.ifa)
1131 IVT_END(Data_Key_Miss)
1133 IVT_ENTRY(Dirty_Bit, 0x2000)
1140 add r20=24,r18 // collision chain
1142 ld8 r20=[r20] // bucket head
1144 rsm psr.dt // turn off data translations
1145 dep r20=0,r20,61,3 // convert vhpt ptr to physical
1148 ld8 r20=[r20] // first entry
1150 1: cmp.eq p15,p0=r0,r20 // done?
1151 (p15) br.cond.spnt.few 9f // bail if done
1153 add r21=16,r20 // tag location
1155 ld8 r21=[r21] // read tag
1157 cmp.ne p15,p0=r21,r19 // compare tags
1158 (p15) br.cond.sptk.few 2f // if not, read next in chain
1160 ld8 r21=[r20] // read pte
1161 mov r22=PTE_DIRTY+PTE_ACCESSED
1163 or r21=r22,r21 // set dirty & access bit
1165 st8 [r20]=r21,8 // store back
1167 ld8 r22=[r20] // read rest of pte
1169 dep r18=0,r18,61,3 // convert vhpt ptr to physical
1171 add r20=16,r18 // address of tag
1173 ld8.acq r23=[r20] // read old tag
1175 dep r23=-1,r23,63,1 // set ti bit
1177 st8.rel [r20]=r23 // store old tag + ti
1179 mf // make sure everyone sees
1181 st8 [r18]=r21,8 // store pte
1185 st8.rel [r18]=r19 // store new tag
1187 itc.d r21 // and place in TLB
1191 mov pr=r17,0x1ffff // restore predicates
1194 2: add r20=24,r20 // next in chain
1196 ld8 r20=[r20] // read chain
1200 mov pr=r17,0x1ffff // restore predicates
1204 CALL(trap, 8, cr.ifa) // die horribly
1207 IVT_ENTRY(Instruction_Access_Bit, 0x2400)
1214 add r20=24,r18 // collision chain
1216 ld8 r20=[r20] // bucket head
1218 rsm psr.dt // turn off data translations
1219 dep r20=0,r20,61,3 // convert vhpt ptr to physical
1222 ld8 r20=[r20] // first entry
1224 1: cmp.eq p15,p0=r0,r20 // done?
1225 (p15) br.cond.spnt.few 9f // bail if done
1227 add r21=16,r20 // tag location
1229 ld8 r21=[r21] // read tag
1231 cmp.ne p15,p0=r21,r19 // compare tags
1232 (p15) br.cond.sptk.few 2f // if not, read next in chain
1234 ld8 r21=[r20] // read pte
1235 mov r22=PTE_ACCESSED
1237 or r21=r22,r21 // set accessed bit
1239 st8 [r20]=r21,8 // store back
1241 ld8 r22=[r20] // read rest of pte
1243 dep r18=0,r18,61,3 // convert vhpt ptr to physical
1245 add r20=16,r18 // address of tag
1247 ld8.acq r23=[r20] // read old tag
1249 dep r23=-1,r23,63,1 // set ti bit
1251 st8.rel [r20]=r23 // store old tag + ti
1253 mf // make sure everyone sees
1255 st8 [r18]=r21,8 // store pte
1259 st8.rel [r18]=r19 // store new tag
1261 itc.i r21 // and place in TLB
1265 mov pr=r17,0x1ffff // restore predicates
1266 rfi // walker will retry the access
1268 2: add r20=24,r20 // next in chain
1270 ld8 r20=[r20] // read chain
1274 mov pr=r17,0x1ffff // restore predicates
1278 CALL(trap, 9, cr.ifa)
1279 IVT_END(Instruction_Access_Bit)
1281 IVT_ENTRY(Data_Access_Bit, 0x2800)
1288 add r20=24,r18 // collision chain
1290 ld8 r20=[r20] // bucket head
1292 rsm psr.dt // turn off data translations
1293 dep r20=0,r20,61,3 // convert vhpt ptr to physical
1296 ld8 r20=[r20] // first entry
1298 1: cmp.eq p15,p0=r0,r20 // done?
1299 (p15) br.cond.spnt.few 9f // bail if done
1301 add r21=16,r20 // tag location
1303 ld8 r21=[r21] // read tag
1305 cmp.ne p15,p0=r21,r19 // compare tags
1306 (p15) br.cond.sptk.few 2f // if not, read next in chain
1308 ld8 r21=[r20] // read pte
1309 mov r22=PTE_ACCESSED
1311 or r21=r22,r21 // set accessed bit
1313 st8 [r20]=r21,8 // store back
1315 ld8 r22=[r20] // read rest of pte
1317 dep r18=0,r18,61,3 // convert vhpt ptr to physical
1319 add r20=16,r18 // address of tag
1321 ld8.acq r23=[r20] // read old tag
1323 dep r23=-1,r23,63,1 // set ti bit
1325 st8.rel [r20]=r23 // store old tag + ti
1327 mf // make sure everyone sees
1329 st8 [r18]=r21,8 // store pte
1333 st8.rel [r18]=r19 // store new tag
1335 itc.d r21 // and place in TLB
1339 mov pr=r17,0x1ffff // restore predicates
1340 rfi // walker will retry the access
1342 2: add r20=24,r20 // next in chain
1344 ld8 r20=[r20] // read chain
1348 mov pr=r17,0x1ffff // restore predicates
1352 CALL(trap, 10, cr.ifa)
1353 IVT_END(Data_Access_Bit)
1355 IVT_ENTRY(Break_Instruction, 0x2c00)
1359 br.sptk exception_save
1363 alloc r15=ar.pfs,0,0,2,0
1378 br.call.sptk rp=trap
1384 br.sptk exception_restore
1387 IVT_END(Break_Instruction)
1389 IVT_ENTRY(External_Interrupt, 0x3000)
1391 mov r17=ar.itc // Put the ITC in the trapframe.
1393 br.sptk exception_save
1397 alloc r15=ar.pfs,0,0,1,0
1405 br.call.sptk rp=ia64_handle_intr
1411 br.sptk exception_restore
1414 IVT_END(External_Interrupt)
1416 IVT_ENTRY(Reserved_3400, 0x3400)
1417 CALL(trap, 13, cr.ifa)
1418 IVT_END(Reserved_3400)
1420 IVT_ENTRY(Reserved_3800, 0x3800)
1421 CALL(trap, 14, cr.ifa)
1422 IVT_END(Reserved_3800)
1424 IVT_ENTRY(Reserved_3c00, 0x3c00)
1425 CALL(trap, 15, cr.ifa)
1426 IVT_END(Reserved_3c00)
1428 IVT_ENTRY(Reserved_4000, 0x4000)
1429 CALL(trap, 16, cr.ifa)
1430 IVT_END(Reserved_4000)
1432 IVT_ENTRY(Reserved_4400, 0x4400)
1433 CALL(trap, 17, cr.ifa)
1434 IVT_END(Reserved_4400)
1436 IVT_ENTRY(Reserved_4800, 0x4800)
1437 CALL(trap, 18, cr.ifa)
1438 IVT_END(Reserved_4800)
1440 IVT_ENTRY(Reserved_4c00, 0x4c00)
1441 CALL(trap, 19, cr.ifa)
1442 IVT_END(Reserved_4c00)
1444 IVT_ENTRY(Page_Not_Present, 0x5000)
1445 CALL(trap, 20, cr.ifa)
1446 IVT_END(Page_Not_Present)
1448 IVT_ENTRY(Key_Permission, 0x5100)
1449 CALL(trap, 21, cr.ifa)
1450 IVT_END(Key_Permission)
1452 IVT_ENTRY(Instruction_Access_Rights, 0x5200)
1453 CALL(trap, 22, cr.ifa)
1454 IVT_END(Instruction_Access_Rights)
1456 IVT_ENTRY(Data_Access_Rights, 0x5300)
1457 CALL(trap, 23, cr.ifa)
1458 IVT_END(Data_Access_Rights)
1460 IVT_ENTRY(General_Exception, 0x5400)
1461 CALL(trap, 24, cr.ifa)
1462 IVT_END(General_Exception)
1464 IVT_ENTRY(Disabled_FP_Register, 0x5500)
1465 CALL(trap, 25, cr.ifa)
1466 IVT_END(Disabled_FP_Register)
1468 IVT_ENTRY(NaT_Consumption, 0x5600)
1469 CALL(trap, 26, cr.ifa)
1470 IVT_END(NaT_Consumption)
1472 IVT_ENTRY(Speculation, 0x5700)
1473 CALL(trap, 27, cr.iim)
1474 IVT_END(Speculation)
1476 IVT_ENTRY(Reserved_5800, 0x5800)
1477 CALL(trap, 28, cr.ifa)
1478 IVT_END(Reserved_5800)
1480 IVT_ENTRY(Debug, 0x5900)
1481 CALL(trap, 29, cr.ifa)
1484 IVT_ENTRY(Unaligned_Reference, 0x5a00)
1485 CALL(trap, 30, cr.ifa)
1486 IVT_END(Unaligned_Reference)
1488 IVT_ENTRY(Unsupported_Data_Reference, 0x5b00)
1489 CALL(trap, 31, cr.ifa)
1490 IVT_END(Unsupported_Data_Reference)
1492 IVT_ENTRY(Floating_Point_Fault, 0x5c00)
1493 CALL(trap, 32, cr.ifa)
1494 IVT_END(Floating_Point_Fault)
1496 IVT_ENTRY(Floating_Point_Trap, 0x5d00)
1497 CALL(trap, 33, cr.ifa)
1498 IVT_END(Floating_Point_Trap)
1500 IVT_ENTRY(Lower_Privilege_Transfer_Trap, 0x5e00)
1501 CALL(trap, 34, cr.ifa)
1502 IVT_END(Lower_Privilege_Transfer_Trap)
1504 IVT_ENTRY(Taken_Branch_Trap, 0x5f00)
1505 CALL(trap, 35, cr.ifa)
1506 IVT_END(Taken_Branch_Trap)
1508 IVT_ENTRY(Single_Step_Trap, 0x6000)
1509 CALL(trap, 36, cr.ifa)
1510 IVT_END(Single_Step_Trap)
1512 IVT_ENTRY(Reserved_6100, 0x6100)
1513 CALL(trap, 37, cr.ifa)
1514 IVT_END(Reserved_6100)
1516 IVT_ENTRY(Reserved_6200, 0x6200)
1517 CALL(trap, 38, cr.ifa)
1518 IVT_END(Reserved_6200)
1520 IVT_ENTRY(Reserved_6300, 0x6300)
1521 CALL(trap, 39, cr.ifa)
1522 IVT_END(Reserved_6300)
1524 IVT_ENTRY(Reserved_6400, 0x6400)
1525 CALL(trap, 40, cr.ifa)
1526 IVT_END(Reserved_6400)
1528 IVT_ENTRY(Reserved_6500, 0x6500)
1529 CALL(trap, 41, cr.ifa)
1530 IVT_END(Reserved_6500)
1532 IVT_ENTRY(Reserved_6600, 0x6600)
1533 CALL(trap, 42, cr.ifa)
1534 IVT_END(Reserved_6600)
1536 IVT_ENTRY(Reserved_6700, 0x6700)
1537 CALL(trap, 43, cr.ifa)
1538 IVT_END(Reserved_6700)
1540 IVT_ENTRY(Reserved_6800, 0x6800)
1541 CALL(trap, 44, cr.ifa)
1542 IVT_END(Reserved_6800)
1544 IVT_ENTRY(IA_32_Exception, 0x6900)
1545 CALL(IA32_TRAP, 45, cr.ifa)
1546 IVT_END(IA_32_Exception)
1548 IVT_ENTRY(IA_32_Intercept, 0x6a00)
1549 CALL(IA32_TRAP, 46, cr.iim)
1550 IVT_END(IA_32_Intercept)
1552 IVT_ENTRY(IA_32_Interrupt, 0x6b00)
1553 CALL(IA32_TRAP, 47, cr.ifa)
1554 IVT_END(IA_32_Interrupt)
1556 IVT_ENTRY(Reserved_6c00, 0x6c00)
1557 CALL(trap, 48, cr.ifa)
1558 IVT_END(Reserved_6c00)
1560 IVT_ENTRY(Reserved_6d00, 0x6d00)
1561 CALL(trap, 49, cr.ifa)
1562 IVT_END(Reserved_6d00)
1564 IVT_ENTRY(Reserved_6e00, 0x6e00)
1565 CALL(trap, 50, cr.ifa)
1566 IVT_END(Reserved_6e00)
1568 IVT_ENTRY(Reserved_6f00, 0x6f00)
1569 CALL(trap, 51, cr.ifa)
1570 IVT_END(Reserved_6f00)
1572 IVT_ENTRY(Reserved_7000, 0x7000)
1573 CALL(trap, 52, cr.ifa)
1574 IVT_END(Reserved_7000)
1576 IVT_ENTRY(Reserved_7100, 0x7100)
1577 CALL(trap, 53, cr.ifa)
1578 IVT_END(Reserved_7100)
1580 IVT_ENTRY(Reserved_7200, 0x7200)
1581 CALL(trap, 54, cr.ifa)
1582 IVT_END(Reserved_7200)
1584 IVT_ENTRY(Reserved_7300, 0x7300)
1585 CALL(trap, 55, cr.ifa)
1586 IVT_END(Reserved_7300)
1588 IVT_ENTRY(Reserved_7400, 0x7400)
1589 CALL(trap, 56, cr.ifa)
1590 IVT_END(Reserved_7400)
1592 IVT_ENTRY(Reserved_7500, 0x7500)
1593 CALL(trap, 57, cr.ifa)
1594 IVT_END(Reserved_7500)
1596 IVT_ENTRY(Reserved_7600, 0x7600)
1597 CALL(trap, 58, cr.ifa)
1598 IVT_END(Reserved_7600)
1600 IVT_ENTRY(Reserved_7700, 0x7700)
1601 CALL(trap, 59, cr.ifa)
1602 IVT_END(Reserved_7700)
1604 IVT_ENTRY(Reserved_7800, 0x7800)
1605 CALL(trap, 60, cr.ifa)
1606 IVT_END(Reserved_7800)
1608 IVT_ENTRY(Reserved_7900, 0x7900)
1609 CALL(trap, 61, cr.ifa)
1610 IVT_END(Reserved_7900)
1612 IVT_ENTRY(Reserved_7a00, 0x7a00)
1613 CALL(trap, 62, cr.ifa)
1614 IVT_END(Reserved_7a00)
1616 IVT_ENTRY(Reserved_7b00, 0x7b00)
1617 CALL(trap, 63, cr.ifa)
1618 IVT_END(Reserved_7b00)
1620 IVT_ENTRY(Reserved_7c00, 0x7c00)
1621 CALL(trap, 64, cr.ifa)
1622 IVT_END(Reserved_7c00)
1624 IVT_ENTRY(Reserved_7d00, 0x7d00)
1625 CALL(trap, 65, cr.ifa)
1626 IVT_END(Reserved_7d00)
1628 IVT_ENTRY(Reserved_7e00, 0x7e00)
1629 CALL(trap, 66, cr.ifa)
1630 IVT_END(Reserved_7e00)
1632 IVT_ENTRY(Reserved_7f00, 0x7f00)
1633 CALL(trap, 67, cr.ifa)
1634 IVT_END(Reserved_7f00)