2 * Copyright (c) 2006 Kip Macy <kmacy@FreeBSD.org>
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 #include <machine/asm.h>
31 #include "opt_compat.h"
33 #include "opt_simulator.h"
34 #include "opt_trap_trace.h"
36 #include <machine/asi.h>
37 #include <machine/asmacros.h>
38 #include <machine/frame.h>
39 #include <machine/fsr.h>
40 #include <machine/hypervisorvar.h>
41 #include <machine/intr_machdep.h>
42 #include <machine/ktr.h>
43 #include <machine/mmu.h>
44 #include <machine/pcb.h>
45 #include <machine/pstate.h>
46 #include <machine/trap.h>
47 #include <machine/tstate.h>
48 #include <machine/utrap.h>
49 #include <machine/wstate.h>
55 #define SPILL_FILL_MAGIC_TRAP_ON nop
56 #define SPILL_FILL_MAGIC_TRAP_OFF nop
65 .globl trap_conversion
72 mov CONS_PUTCHAR, %o5 ; \
78 * Atomically set the reference bit in a tte.
80 #define TTE_SET_BIT(r1, r2, r3, bit) \
81 add r1, TTE_DATA, r1 ; \
84 casxa [r1] ASI_N, r2, r3 ; \
89 #define TTE_SET_REF(r1, r2, r3) TTE_SET_BIT(r1, r2, r3, TD_REF)
90 #define TTE_SET_W(r1, r2, r3) TTE_SET_BIT(r1, r2, r3, TD_W)
94 * Macros for spilling and filling live windows.
95 * Here we use the more complicated [regaddr] format which requires
96 * us to interleave setting the globals in order to be able to use
97 * imm_asi - we don't ever implicitly assume kernel context as in
98 * Solaris' spill/fill handlers so that we have the option of using
99 * block initializing stores - twin doubleword loads could also be
100 * advantageous but will require an additional macro
105 #define SPILL(storer, base, size) \
106 storer %l0, [base + (0 * size)] ; \
107 storer %l1, [base + (1 * size)] ; \
108 storer %l2, [base + (2 * size)] ; \
109 storer %l3, [base + (3 * size)] ; \
110 storer %l4, [base + (4 * size)] ; \
111 storer %l5, [base + (5 * size)] ; \
112 storer %l6, [base + (6 * size)] ; \
113 storer %l7, [base + (7 * size)] ; \
114 storer %i0, [base + (8 * size)] ; \
115 storer %i1, [base + (9 * size)] ; \
116 storer %i2, [base + (10 * size)] ; \
117 storer %i3, [base + (11 * size)] ; \
118 storer %i4, [base + (12 * size)] ; \
119 storer %i5, [base + (13 * size)] ; \
120 storer %i6, [base + (14 * size)] ; \
121 storer %i7, [base + (15 * size)]
123 #define SPILL_ASI(storer, bias, size, asi) \
125 storer %l0, [%sp + %g1]asi ;\
126 mov size + bias, %g2 ;\
127 storer %l1, [%sp + %g2]asi ;\
128 mov (2 * size) + bias, %g3 ;\
129 storer %l2, [%sp + %g3]asi ;\
130 mov (3 * size) + bias, %g4 ;\
131 storer %l3, [%sp + %g4]asi ;\
132 add %sp, (4 * size), %g5 ;\
133 storer %l4, [%g5 + %g1]asi ;\
134 storer %l5, [%g5 + %g2]asi ;\
135 storer %l6, [%g5 + %g3]asi ;\
136 storer %l7, [%g5 + %g4]asi ;\
137 add %g5, (4 * size), %g5 ;\
138 storer %i0, [%g5 + %g1]asi ;\
139 storer %i1, [%g5 + %g2]asi ;\
140 storer %i2, [%g5 + %g3]asi ;\
141 storer %i3, [%g5 + %g4]asi ;\
142 add %g5, (4 * size), %g5 ;\
143 storer %i4, [%g5 + %g1]asi ;\
144 storer %i5, [%g5 + %g2]asi ;\
145 storer %i6, [%g5 + %g3]asi ;\
146 storer %i7, [%g5 + %g4]asi
148 /* 16 instructions */
149 #define SPILL_ASI_64 \
150 stxa %l0, [%g1 + 0x0]%asi ;\
151 stxa %i0, [%g1 + 0x40]%asi ;\
152 stxa %l1, [%g1 + 0x8]%asi ;\
153 stxa %l2, [%g1 + 0x10]%asi ;\
154 stxa %l3, [%g1 + 0x18]%asi ;\
155 stxa %l4, [%g1 + 0x20]%asi ;\
156 stxa %l5, [%g1 + 0x28]%asi ;\
157 stxa %l6, [%g1 + 0x30]%asi ;\
158 stxa %l7, [%g1 + 0x38]%asi ;\
159 stxa %i1, [%g1 + 0x48]%asi ;\
160 stxa %i2, [%g1 + 0x50]%asi ;\
161 stxa %i3, [%g1 + 0x58]%asi ;\
162 stxa %i4, [%g1 + 0x60]%asi ;\
163 stxa %i5, [%g1 + 0x68]%asi ;\
164 stxa %i6, [%g1 + 0x70]%asi ;\
165 stxa %i7, [%g1 + 0x78]%asi
167 /* 23 instructions */
168 #define FILL(loader, bias, size, asi) \
170 loader [%sp + %g1]asi, %l0 ;\
171 mov size + bias, %g2 ;\
172 loader [%sp + %g2]asi, %l1 ;\
173 mov (2 * size) + bias, %g3 ;\
174 loader [%sp + %g3]asi, %l2 ;\
175 mov (3 * size) + bias, %g4 ;\
176 loader [%sp + %g4]asi, %l3 ;\
177 add %sp, (4 * size), %g5 ;\
178 loader [%g5 + %g1]asi, %l4 ;\
179 loader [%g5 + %g2]asi, %l5 ;\
180 loader [%g5 + %g3]asi, %l6 ;\
181 loader [%g5 + %g4]asi, %l7 ;\
182 add %g5, (4 * size), %g5 ;\
183 loader [%g5 + %g1]asi, %i0 ;\
184 loader [%g5 + %g2]asi, %i1 ;\
185 loader [%g5 + %g3]asi, %i2 ;\
186 loader [%g5 + %g4]asi, %i3 ;\
187 add %g5, (4 * size), %g5 ;\
188 loader [%g5 + %g1]asi, %i4 ;\
189 loader [%g5 + %g2]asi, %i5 ;\
190 loader [%g5 + %g3]asi, %i6 ;\
191 loader [%g5 + %g4]asi, %i7
193 #define SPILL_ASI_SET(storer, size) \
194 storer %l0, [%g1 + (0 * size)]%asi ;\
195 storer %l1, [%g1 + (1 * size)]%asi ;\
196 storer %l2, [%g1 + (2 * size)]%asi ;\
197 storer %l3, [%g1 + (3 * size)]%asi ;\
198 storer %l4, [%g1 + (4 * size)]%asi ;\
199 storer %l5, [%g1 + (5 * size)]%asi ;\
200 storer %l6, [%g1 + (6 * size)]%asi ;\
201 storer %l7, [%g1 + (7 * size)]%asi ;\
202 storer %i0, [%g1 + (8 * size)]%asi ;\
203 storer %i1, [%g1 + (9 * size)]%asi ;\
204 storer %i2, [%g1 + (10 * size)]%asi ;\
205 storer %i3, [%g1 + (11 * size)]%asi ;\
206 storer %i4, [%g1 + (12 * size)]%asi ;\
207 storer %i5, [%g1 + (13 * size)]%asi ;\
208 storer %i6, [%g1 + (14 * size)]%asi ;\
209 storer %i7, [%g1 + (15 * size)]%asi
211 /* 16 instructions */
212 #define FILL_ASI_SET(loader, size) \
213 loader [%g1 + 0x0]%asi, %l0 ;\
214 loader [%g1 + (size * 1)]%asi, %l1 ;\
215 loader [%g1 + (size * 2)]%asi, %l2 ;\
216 loader [%g1 + (size * 3)]%asi, %l3 ;\
217 loader [%g1 + (size * 4)]%asi, %l4 ;\
218 loader [%g1 + (size * 5)]%asi, %l5 ;\
219 loader [%g1 + (size * 6)]%asi, %l6 ;\
220 loader [%g1 + (size * 7)]%asi, %l7 ;\
221 loader [%g1 + (size * 8)]%asi, %i0 ;\
222 loader [%g1 + (size * 9)]%asi, %i1 ;\
223 loader [%g1 + (size * 10)]%asi, %i2 ;\
224 loader [%g1 + (size * 11)]%asi, %i3 ;\
225 loader [%g1 + (size * 12)]%asi, %i4 ;\
226 loader [%g1 + (size * 13)]%asi, %i5 ;\
227 loader [%g1 + (size * 14)]%asi, %i6 ;\
228 loader [%g1 + (size * 15)]%asi, %i7
232 prefetch [%g1 + 0x40], #one_read ;\
233 ldda [%g1 + 0]%asi, %l0 ;\
234 ldda [%g1 + 0x10]%asi, %l2 ;\
235 ldda [%g1 + 0x20]%asi, %l4 ;\
236 ldda [%g1 + 0x30]%asi, %l6 ;\
237 ldda [%g1 + 0x40]%asi, %i0 ;\
238 ldda [%g1 + 0x50]%asi, %i2 ;\
239 ldda [%g1 + 0x60]%asi, %i4 ;\
240 ldda [%g1 + 0x70]%asi, %i6
242 #include <sun4v/sun4v/wbuf.S>
244 * Clean window traps occur when %cleanwin is zero to ensure that data
245 * is not leaked between address spaces in registers.
265 wrpr %l7, 0, %cleanwin
274 .macro tl0_setup type
283 * Generic trap type. Call trap() with the specified type.
291 * This is used to suck up the massive swaths of reserved trap types.
292 * Generates count "reserved" trap vectors.
295 .macro tl0_reserved count
302 .macro tl1_setup type
315 .macro tl1_reserved count
322 GET_MMFSA_SCRATCH(%g1)
323 mov MMFSA_D_ADDR, %g2
324 ldxa [%g1 + %g2]ASI_REAL, %g3
328 mov T_INSTRUCTION_EXCEPTION, %g2
334 GET_MMFSA_SCRATCH(%g1)
335 mov MMFSA_I_TYPE, %g2
336 mov MMFSA_I_ADDR, %g3
338 ldxa [%g1 + %g2]ASI_REAL, %g4
339 ba,pt %xcc, tsb_miss_handler
340 ldxa [%g1 + %g3]ASI_REAL, %g5
345 GET_MMFSA_SCRATCH(%g1)
346 mov MMFSA_D_ADDR, %g2
347 ba,pt %xcc, data_excptn_fault
348 ldxa [%g1 + %g2]ASI_REAL, %g3
352 ENTRY(data_excptn_fault)
354 ldxa [%g1 + %g7]ASI_REAL, %g4
355 sllx %g4, TRAP_CTX_SHIFT, %g4
356 or %g4, T_DATA_EXCEPTION, %g2
360 END(data_excptn_fault)
363 GET_MMFSA_SCRATCH(%g1)
364 mov MMFSA_D_TYPE, %g2
365 mov MMFSA_D_ADDR, %g3
367 ldxa [%g1 + %g2]ASI_REAL, %g4
368 ba,pt %xcc, tsb_miss_handler
369 ldxa [%g1 + %g3]ASI_REAL, %g5
374 GET_MMFSA_SCRATCH(%g1)
375 mov MMFSA_D_ADDR, %g3
377 ba,pt %xcc, tsb_miss_handler
378 ldxa [%g1 + %g3]ASI_REAL, %g5
383 GET_MMFSA_SCRATCH(%g1)
384 mov MMFSA_D_ADDR, %g3
386 ba,pt %xcc, align_fault
387 ldxa [%g1 + %g3]ASI_REAL, %g3
392 ldxa [%g1 + %g7]ASI_REAL, %g4
393 sllx %g4, TRAP_CTX_SHIFT, %g4
394 or %g4, T_MEM_ADDRESS_NOT_ALIGNED, %g2
401 ba,a,pt %xcc, cpu_mondo
406 ba,a,pt %xcc, dev_mondo
410 .macro resumable_error
417 mov T_RESUMABLE_ERROR, %g2
421 .macro nonresumable_error
426 mov T_NONRESUMABLE_ERROR, %g2
431 #define ALIGN_128 .align 128
433 #define LOOKASIDE #Lookaside
435 #ifdef USE_FAST_SPILLFILL
436 #define spill_64bit_asi(asival, asival_unaligned, target) \
437 wr %g0, asival, %asi ; \
438 add %sp, SPOFF, %g1 ; \
444 ba,a,pt %xcc, fault_64bit_##target ; \
447 #define spill_64clean(asival, asival_unaligned, target) \
448 wr %g0, asival, %asi ; \
449 add %sp, SPOFF, %g1 ; \
453 mov WSTATE_USER64, %g7 ; \
455 ba,a,pt %xcc, fault_64bit_##target ; \
458 #define fill_64bit_asi(asival, asival_unaligned, target) \
459 add %sp, SPOFF, %g1 ; \
460 wr %g0, asival, %asi ; \
465 ba,a,pt %xcc, fault_64bit_##target ; \
468 #define spill_64bit_asi(asival, asival_unaligned, target) \
469 wr %g0, asival_unaligned, %asi ; \
470 add %sp, SPOFF, %g1 ; \
471 SPILL_ASI_SET(stxa, 8) ; \
475 ba,a,pt %xcc, fault_64bit_##target ; \
478 #define spill_64clean(asival, asival_unaligned, target) \
479 wr %g0, asival_unaligned, %asi ; \
480 add %sp, SPOFF, %g1 ; \
481 SPILL_ASI_SET(stxa, 8) ; \
483 mov WSTATE_USER64, %g7 ; \
485 ba,a,pt %xcc, fault_64bit_##target ; \
488 #define fill_64bit_asi(asival, asival_unaligned, target) \
489 wr %g0, asival_unaligned, %asi ; \
490 add %sp, SPOFF, %g1 ; \
491 FILL_ASI_SET(ldxa, 8) ; \
495 ba,a,pt %xcc, fault_64bit_##target ; \
499 #define spill_32bit_asi(asi, target) \
501 SPILL_FILL_MAGIC_TRAP_ON; \
502 SPILL_ASI(sta, 0, 4, asi) ; \
504 SPILL_FILL_MAGIC_TRAP_OFF; \
507 ba,a,pt %xcc, fault_32bit_##target ; \
510 #define spill_32clean(asi, target) \
512 SPILL_FILL_MAGIC_TRAP_ON; \
513 SPILL_ASI(sta, 0, 4, asi) ; \
515 mov WSTATE_USER32, %g7 ; \
517 ba,a,pt %xcc, fault_32bit_##target ; \
520 #define fill_32bit_asi(asi, target) \
522 SPILL_FILL_MAGIC_TRAP_ON; \
523 FILL(lda, 0, 4, asi) ; \
527 ba,a,pt %xcc, fault_32bit_##target ; \
531 ENTRY(fill_64bit_slow_fn0)
533 FILL_ASI_SET(ldxa, 8);
537 ba,a,pt %xcc, fault_64bit_fn0 ;
539 END(fill_64bit_slow_fn0)
540 ENTRY(fill_64bit_slow_not)
541 FILL_ASI_SET(ldxa, 8);
545 ba,a,pt %xcc, fault_64bit_not ;
547 END(fill_64bit_slow_not)
550 .macro spill_32bit_primary_sn0
551 spill_32bit_asi(ASI_AIUP, sn0)
554 .macro spill_64bit_primary_sn0
555 spill_64bit_asi(ASI_LDSTBI_AIUP, ASI_AIUP, sn0)
558 .macro spill_32clean_primary_sn0
559 spill_32clean(ASI_AIUP, sn0)
562 .macro spill_64clean_primary_sn0
563 spill_64clean(ASI_LDSTBI_AIUP, ASI_AIUP, sn0)
566 .macro spill_32bit_nucleus_not
567 spill_32bit_asi(ASI_N, not)
570 .macro spill_64bit_nucleus_not
571 spill_64bit_asi(ASI_LDSTBI_N, ASI_N, not)
574 .macro spill_32bit_secondary_so0
575 spill_32bit_asi(ASI_AIUS, so0)
578 .macro spill_64bit_secondary_so0
579 spill_64bit_asi(ASI_LDSTBI_AIUS, ASI_AIUS, so0)
582 .macro fill_32bit_primary_fn0
583 fill_32bit_asi(ASI_AIUP, fn0)
586 .macro fill_64bit_primary_fn0
587 fill_64bit_asi(ASI_LDSTBI_AIUP, ASI_AIUP, fn0)
590 .macro fill_32bit_nucleus_not
591 fill_32bit_asi(ASI_N, not)
594 .macro fill_64bit_nucleus_not
595 fill_64bit_asi(ASI_LDSTBI_N, ASI_N, not)
598 .macro spill_32bit_tt1_primary_sn1
599 ba,a,pt %xcc, fault_32bit_sn1
604 .macro spill_64bit_tt1_primary_sn1
605 ba,a,pt %xcc, fault_64bit_sn1
610 .macro spill_64bit_ktt1_sk
611 ba,a,pt %xcc, fault_64bit_sk
616 .macro spill_mixed_ktt1_sk
618 bz,a,pt %xcc, fault_32bit_sk
620 ba,a,pt %xcc, fault_64bit_sk
625 .macro spill_32bit_tt1_secondary_so1
626 ba,a,pt %xcc, fault_32bit_so1
631 .macro spill_64bit_tt1_secondary_so1
632 ba,a,pt %xcc, fault_64bit_so1
650 ba,a,pt %xcc, tl1_trap
654 .macro tl0_pil_entry level, mask
659 wr %g1, 0, %clear_softint
665 #define INTR(level, traplvl) \
666 tl ## traplvl ## _pil_entry level, 1 << level
668 #define TICK(traplvl) \
669 tl ## traplvl ## _pil_entry PIL_TICK, 1
671 #define INTR_LEVEL(tl) \
685 tick_ ## tl ## _entry: \
704 .macro dtrace_fasttrap
705 sethi %hi(dtrace_fasttrap_probe_ptr), %g4
706 ldx [%g4 + %lo(dtrace_fasttrap_probe_ptr)], %g4
707 set dtrace_fasttrap_probe, %g1
708 brnz,pn %g4, tl0_utrap
714 set dtrace_pid_probe, %g1
715 ba,pt %xcc, tl0_utrap
721 set dtrace_return_probe, %g1
722 ba,pt %xcc, tl0_utrap
727 .macro dtrace_fasttrap
743 ! fetch FP context into local registers
744 .macro tl0_fpemu_context
745 GET_PCB(PCB_REG) ! 3 instructions
746 ldx [PCB_REG + PCB_TSTATE], %l5 ! %tstate
747 ldx [PCB_REG + PCB_TPC], %l6 ! %tpc
748 ldx [PCB_REG + PCB_TNPC], %l7 ! %tnpc
749 ldx [PCB_REG + PCB_TT], %g2 ! %tt
750 ba,a,pt %xcc, tl0_fpemu_context
754 ENTRY(tl0_fpemu_context)
757 ldx [PCB_REG + PCB_SFAR], %o4
760 or %l1, FPRS_FEF, %l2
762 stx %fsr, [PCB_REG + PCB_PAD]
763 ldx [PCB_REG + PCB_PAD], %l4
768 END(tl0_fpemu_context)
770 .macro tl0_fp_restore
771 GET_PCB(PCB_REG) ! 3 instructions
772 ldx [%g6 + PCB_FLAGS], %g1
773 ba,pt %xcc, tl0_fp_restore
774 wr %g0, FPRS_FEF, %fprs
779 GET_PCB(PCB_REG) ! 3 instructions
780 ldx [PCB_REG + PCB_FLAGS], %g1
781 andcc %g1, PCB_FEF, %g0
782 bnz,pt %xcc, tl0_fp_restore
783 wr %g0, FPRS_FEF, %fprs
788 ENTRY(tl0_fp_restore)
789 andn %g1, PCB_FEF, %g1
790 stx %g1, [%g6 + PCB_FLAGS]
792 ldd [PCB_REG + PCB_UFP + (0 * 64)], %f0
793 ldd [PCB_REG + PCB_UFP + (1 * 64)], %f16
794 ldd [PCB_REG + PCB_UFP + (2 * 64)], %f32
795 ldd [PCB_REG + PCB_UFP + (3 * 64)], %f48
799 .macro tl1_insn_excptn
805 .macro tl1_soft count
807 tl1_gen T_SOFT | T_KERNEL
815 tl0_reserved 8 ! 0x0-0x7
820 tl0_reserved 6 ! 0xa-0xf
822 tl0_gen T_ILLEGAL_INSTRUCTION ! 0x10
824 tl0_gen T_PRIVILEGED_OPCODE ! 0x11
825 tl0_reserved 14 ! 0x12-0x1f
829 tl0_gen T_FP_EXCEPTION_IEEE_754 ! 0x21
831 tl0_gen T_FP_EXCEPTION_OTHER ! 0x22
833 tl0_gen T_TAG_OVERFLOW ! 0x23
837 tl0_gen T_DIVISION_BY_ZERO ! 0x28
838 tl0_reserved 7 ! 0x29-0x2f
843 tl0_reserved 2 ! 0x32-0x33
847 tl0_gen T_RESERVED ! 0x35
849 tl0_gen T_RESERVED ! 0x36
851 tl0_gen T_PRIVILEGED_ACTION ! 0x37
852 tl0_reserved 9 ! 0x38-0x40
855 tl0_reserved 18 ! 0x50-0x61
857 tl0_gen T_VA_WATCHPOINT ! 0x62
858 tl0_reserved 9 ! 0x63-0x6b
861 tl0_reserved 9 ! 0x6d-0x75
863 tl0_gen T_BREAKPOINT ! 0x76
864 tl0_reserved 5 ! 0x77-0x7b
869 tl0_resumable_error_7e:
870 resumable_error ! 0x7e
871 tl0_nonresumable_error_7f:
872 nonresumable_error ! 0x7f
873 tl0_spill_n_normal_80:
875 tl0_reserved 4 ! 0x80
877 spill_32bit_primary_sn0 ! 0x84
879 spill_64bit_primary_sn0 ! 0x88
881 spill_32clean_primary_sn0 ! 0x8c
883 spill_64clean_primary_sn0 ! 0x90
885 spill_32bit_nucleus_not ! 0x94
887 spill_64bit_nucleus_not ! 0x98
891 tl0_reserved 4 ! 0xa0
893 spill_32bit_secondary_so0 ! 0xa4
895 spill_64bit_secondary_so0 ! 0xa8
897 spill_32bit_secondary_so0 ! 0xac
899 spill_64bit_secondary_so0 ! 0xb0
901 tl0_reserved 4 ! 0xb4
903 tl0_reserved 4 ! 0xb8
905 tl0_reserved 4 ! 0xbc
907 tl0_reserved 4 ! 0xc0
909 fill_32bit_primary_fn0 ! 0xc4
911 fill_64bit_primary_fn0 ! 0xc8
913 fill_32bit_primary_fn0 ! 0xcc
915 fill_64bit_primary_fn0 ! 0xd0
917 fill_32bit_nucleus_not ! 0xd4
919 fill_64bit_nucleus_not ! 0xd8
923 tl0_reserved 32 ! 0xe0-0xff
925 tl0_gen T_SYSCALL ! 0x100
926 tl0_gen T_BREAKPOINT ! 0x101
927 tl0_gen T_DIVISION_BY_ZERO ! 0x102
928 tl0_reserved 1 ! 0x103
929 tl0_gen T_CLEAN_WINDOW ! 0x104
930 tl0_gen T_RANGE_CHECK ! 0x105
931 tl0_gen T_FIX_ALIGNMENT ! 0x106
932 tl0_gen T_INTEGER_OVERFLOW ! 0x107
933 tl0_gen T_SYSCALL ! 0x108
934 tl0_gen T_SYSCALL ! 0x109
935 tl0_fp_restore ! 0x10a
936 tl0_fpemu_context ! 0x10b
937 tl0_reserved 4 ! 0x10c-0x10f
938 tl0_gen T_TRAP_INSTRUCTION_16 ! 0x110
939 tl0_gen T_TRAP_INSTRUCTION_17 ! 0x111
940 tl0_gen T_TRAP_INSTRUCTION_18 ! 0x112
941 tl0_gen T_TRAP_INSTRUCTION_19 ! 0x113
942 tl0_gen T_TRAP_INSTRUCTION_20 ! 0x114
943 tl0_gen T_TRAP_INSTRUCTION_21 ! 0x115
944 tl0_gen T_TRAP_INSTRUCTION_22 ! 0x116
945 tl0_gen T_TRAP_INSTRUCTION_23 ! 0x117
946 tl0_gen T_TRAP_INSTRUCTION_24 ! 0x118
947 tl0_gen T_TRAP_INSTRUCTION_25 ! 0x119
948 tl0_gen T_TRAP_INSTRUCTION_26 ! 0x11a
949 tl0_gen T_TRAP_INSTRUCTION_27 ! 0x11b
950 tl0_gen T_TRAP_INSTRUCTION_28 ! 0x11c
951 tl0_gen T_TRAP_INSTRUCTION_29 ! 0x11d
952 tl0_gen T_TRAP_INSTRUCTION_30 ! 0x11e
953 tl0_gen T_TRAP_INSTRUCTION_31 ! 0x11f
954 tl0_reserved 24 ! 0x120-0x137
958 dtrace_fasttrap ! 0x139
960 dtrace_return ! 0x13a
961 tl0_reserved 5 ! 0x13b - 0x13f
962 tl0_gen T_SYSCALL ! 0x140 LP64 system call
964 tl0_gen T_SYSCALL ! 0x142
965 tl0_gen T_SYSCALL ! 0x143
966 tl0_reserved 188 ! 0x144-0x1ff
968 tl1_reserved 9 ! 0x200-0x208
971 tl1_reserved 26 ! 0x20a-0x223
972 tl1_clean_window_224:
975 tl1_reserved 8 ! 0x228-0x22f
980 tl1_reserved 2 ! 0x232-0x233
983 tl1_reserved 55 ! 0x235-0x26b
986 tl1_reserved 18 ! 0x26c-0x27e
987 tl1_nonresumable_error:
988 nonresumable_error ! 0x27f
991 tl1_reserved 4 ! 0x280
993 spill_32bit_tt1_primary_sn1 ! 0x284
995 spill_64bit_tt1_primary_sn1 ! 0x288
997 spill_32bit_tt1_primary_sn1 ! 0x28c
999 spill_64bit_tt1_primary_sn1 ! 0x290
1001 tl1_reserved 4 ! 0x294
1003 spill_64bit_ktt1_sk ! 0x298
1005 spill_mixed_ktt1_sk ! 0x29c
1008 tl1_reserved 4 ! 0x2a0
1010 spill_32bit_tt1_secondary_so1 ! 0x2a4
1012 spill_64bit_tt1_secondary_so1 ! 0x2a8
1014 spill_32bit_tt1_secondary_so1 ! 0x2ac
1016 spill_64bit_tt1_secondary_so1 ! 0x2b0
1018 tl1_reserved 4 ! 0x2b4
1020 tl1_reserved 4 ! 0x2b8
1022 tl1_reserved 4 ! 0x2bc
1024 tl1_reserved 32 ! 0x2c0-0x2df
1026 tl1_reserved 32 ! 0x2e0-0x2ff
1033 sethi %hi(nwin_minus_one), %g5
1034 ld [%g5 + %lo(nwin_minus_one)], %g5
1037 movneg %xcc, %g5, %g6 ! if (--%cwp < 0) %g6 = nwin-1
1047 wrpr %g0, %g7, %wstate
1053 #define KWBUF64_TO_STACK(SBP,SPP,TMP) \
1054 ldx [SBP + (0*8)], TMP; \
1055 stx TMP, [SPP + SPOFF + 0]; \
1056 ldx [SBP + (1*8)], TMP; \
1057 stx TMP, [SPP + SPOFF + 8]; \
1058 ldx [SBP + (2*8)], TMP; \
1059 stx TMP, [SPP + SPOFF + 16]; \
1060 ldx [SBP + (3*8)], TMP; \
1061 stx TMP, [SPP + SPOFF + 24]; \
1062 ldx [SBP + (4*8)], TMP; \
1063 stx TMP, [SPP + SPOFF + 32]; \
1064 ldx [SBP + (5*8)], TMP; \
1065 stx TMP, [SPP + SPOFF + 40]; \
1066 ldx [SBP + (6*8)], TMP; \
1067 stx TMP, [SPP + SPOFF + 48]; \
1068 ldx [SBP + (7*8)], TMP; \
1069 stx TMP, [SPP + SPOFF + 56]; \
1070 ldx [SBP + (8*8)], TMP; \
1071 stx TMP, [SPP + SPOFF + 64]; \
1072 ldx [SBP + (9*8)], TMP; \
1073 stx TMP, [SPP + SPOFF + 72]; \
1074 ldx [SBP + (10*8)], TMP; \
1075 stx TMP, [SPP + SPOFF + 80]; \
1076 ldx [SBP + (11*8)], TMP; \
1077 stx TMP, [SPP + SPOFF + 88]; \
1078 ldx [SBP + (12*8)], TMP; \
1079 stx TMP, [SPP + SPOFF + 96]; \
1080 ldx [SBP + (13*8)], TMP; \
1081 stx TMP, [SPP + SPOFF + 104]; \
1082 ldx [SBP + (14*8)], TMP; \
1083 stx TMP, [SPP + SPOFF + 112]; \
1084 ldx [SBP + (15*8)], TMP; \
1085 stx TMP, [SPP + SPOFF + 120];
1088 #define fill_64bit_rtt(asi_num) \
1089 wr %g0, asi_num, %asi ;\
1093 ldxa [%sp + SPOFF + 0]%asi, %l0 ;\
1094 ldxa [%sp + SPOFF + 8]%asi, %l1 ;\
1095 ldxa [%sp + SPOFF + 16]%asi, %l2 ;\
1096 ldxa [%sp + SPOFF + 24]%asi, %l3 ;\
1097 ldxa [%sp + SPOFF + 32]%asi, %l4 ;\
1098 ldxa [%sp + SPOFF + 40]%asi, %l5 ;\
1099 ldxa [%sp + SPOFF + 48]%asi, %l6 ;\
1100 ldxa [%sp + SPOFF + 56]%asi, %l7 ;\
1101 ldxa [%sp + SPOFF + 64]%asi, %i0 ;\
1102 ldxa [%sp + SPOFF + 72]%asi, %i1 ;\
1103 ldxa [%sp + SPOFF + 80]%asi, %i2 ;\
1104 ldxa [%sp + SPOFF + 88]%asi, %i3 ;\
1105 ldxa [%sp + SPOFF + 96]%asi, %i4 ;\
1106 ldxa [%sp + SPOFF + 104]%asi, %i5 ;\
1107 ldxa [%sp + SPOFF + 112]%asi, %i6 ;\
1108 ldxa [%sp + SPOFF + 120]%asi, %i7 ;\
1120 stx %g1, [%l7 + TF_ASI]
1121 GET_PCPU_SCRATCH_SLOW(%g6)
1122 wrpr %g0, PSTATE_KERNEL, %pstate ! enable ints
1124 brnz %o1, common_utrap
1129 jmpl %l3, %o7 ! call interrupt handler
1133 ba,pt %xcc, user_rtt
1136 jmpl %l3, %o7 ! call trap handler / syscall
1139 ldx [PCPU_REG + PC_CURPCB], %g6
1140 ldx [%g6 + PCB_KSTACK], %g6
1141 sub %g6, TF_SIZEOF, %sp
1142 add %sp, REGOFF + SPOFF, %l7
1144 ! pil handling needs to be re-visited
1145 wrpr %g0, PIL_TICK, %pil
1146 ldx [PCPU(CURTHREAD)], %l4
1147 lduw [%l4 + TD_FLAGS], %l1
1148 ldx [%l4 + TD_MD + MD_SAVED_PIL], %l0
1149 set TDF_ASTPENDING | TDF_NEEDRESCHED, %l2
1154 ! handle AST and retry return
1158 ba,pt %xcc, user_rtt
1162 ldx [PCPU_REG + PC_CURPCB], %g6
1163 ldx [%g6 + PCB_NSAVED], %l1
1170 ba,pt %xcc, user_rtt
1174 ld [%l7 + TF_WSTATE], %l3
1176 ! restore user globals and outs
1179 ldx [%l7 + TF_ASI], %g1
1180 wrpr %l1, PSTATE_IE, %pstate
1182 RESTORE_GLOBALS_USER(%l7)
1186 wrpr %g0, 0, %pil ! drop pil to 0
1187 wrpr %g0, 1, %tl ! raise tl -> 1 before setting pcontext
1190 GET_MMU_CONTEXT(%g1, %g2)
1192 sethi %hi(FLUSH_ADDR), %g3
1193 SET_MMU_CONTEXT(%g1, %g2)
1194 flush %g3 ! flush required by immu
1195 ! hangover from US I
1200 ldx [%l7 + TF_TPC], %g1
1201 ldx [%l7 + TF_TNPC], %g2
1202 ldx [%l7 + TF_TSTATE], %l0
1203 ldx [%l7 + TF_FPRS], %l1
1207 andn %l0, TSTATE_CWP_MASK, %g6
1209 wr %g0, FPRS_FEF, %fprs
1210 ldx [%l7 + TF_FSR], %fsr
1215 ! switch "other" windows back to "normal" windows and
1216 ! restore to window we originally trapped in
1219 wrpr %g0, 0, %otherwin
1220 add %l3, WSTATE_CLEAN_OFFSET, %l3 ! convert to "clean" wstate
1221 wrpr %g0, %l3, %wstate
1222 wrpr %g0, %g1, %canrestore
1224 rdpr %canrestore, %g1
1226 nop ! no trap, use restore directly
1228 wrpr %g1, %g6, %tstate ! needed by wbuf recovery code
1229 ! hand craft the restore to avoid getting to TL > 2
1234 .global rtt_fill_start
1237 fill_32bit_rtt(ASI_AIUP)
1242 fill_64bit_rtt(ASI_AIUP)
1243 .global rtt_fill_end
1246 restore ! should not trap
1250 ! set %cleanwin to %canrestore
1251 ! set %tstate to the correct %cwp
1252 ! retry resumes user execution
1254 rdpr %canrestore, %g1
1255 wrpr %g0, %g1, %cleanwin
1257 wrpr %g1, %g6, %tstate
1267 stx %g1, [%l7 + TF_ASI]
1268 GET_PCPU_SCRATCH_SLOW(%g6) ! we really shouldn't need this ...
1269 wrpr %g0, PSTATE_KERNEL, %pstate ! enable interrupts
1271 brnz %o1, common_ktrap
1276 jmpl %l3, %o7 ! call trap handler
1283 jmpl %l3, %o7 ! call trap handler
1289 ! restore globals and outs
1292 ldx [%l7 + TF_ASI], %g1
1293 wrpr %l1, PSTATE_IE, %pstate
1296 RESTORE_GLOBALS_KERNEL(%l7)
1298 ! switch to global set 1
1303 ! set %pil from max(old pil, cur_thread_spl)
1305 ldn [%l0 + T_CPU], %l0
1306 ld [%l0 + CPU_BASE_SPL], %l0
1312 ldx [PCPU(CURTHREAD)], %l0
1313 ldx [%l0 + TD_MD + MD_SAVED_PIL], %l0
1318 ! restore to window we originally trapped in
1322 ldx [%l7 + TF_TSTATE], %l0
1323 ldx [%l7 + TF_TPC], %g1
1324 ldx [%l7 + TF_TNPC], %g2
1325 ldx [%l7 + TF_FPRS], %l1
1327 andn %l0, TSTATE_CWP_MASK, %g6
1332 wr %g0, FPRS_FEF, %fprs
1333 ldx [%l7 + TF_FSR], %fsr
1336 rdpr %canrestore, %g1
1338 nop ! can use restore directly
1340 wrpr %g1, %g6, %tstate ! needed by wbuf recovery code
1342 ! avoid going above TL2
1343 fill_64bit_rtt(ASI_N)
1348 ! set %tstate to the correct %cwp
1351 wrpr %g1, %g6, %tstate
1362 save %sp, -(CCFSZ + TF_SIZEOF), %sp
1366 or %g2, T_KERNEL, %g2
1368 ! if the kwbuf is full we need to save to the stack now
1369 ld [PCPU_REG + PC_KWBUF_FULL], %o0
1372 st %g0, [PCPU_REG + PC_KWBUF_FULL]
1373 ldx [PCPU_REG + PC_KWBUF_SP], %o1
1374 add PCPU_REG, PC_KWBUF, %o0
1375 KWBUF64_TO_STACK(%o0, %o1, %o2)
1377 ba,a,pt %xcc, win_saved
1381 ! register convention:
1382 ! %g2=level %g1=mask
1385 SET(intr_handlers, %g7, %g6)
1386 sllx %g4, IH_SHIFT, %g7
1387 ldx [%g6 + %g7], %g1 ! pointer to interrupt handler
1391 ! %g1 pc of trap handler
1392 ! %g2, %g3 args of trap handler
1393 ! %g2 software trap type
1394 ! %g3 additional argument to trap
1402 ! %l5 memory fault info
1403 ! %l6 %pil for priv traps
1407 /* if we're at tl2 we have some extra work to do */
1410 be,pn %xcc, tl1_trap
1414 btst TSTATE_PRIV, %g5
1415 and %g5, TSTATE_CWP_MASK, %g6
1417 bnz,pn %xcc, tl0_ktrap
1422 and %g2, TRAP_MASK, %g4
1424 bge,a,pt %xcc, tl0_skip_utrap
1427 ldx [PCPU(CURTHREAD)], %g5
1428 ldx [%g5 + TD_PROC], %g5
1429 ldx [%g5 + P_MD + MD_UTRAP], %g5
1430 brz,pn %g5, tl0_skip_utrap
1431 sllx %g4, PTR_SHIFT, %g6
1432 ldx [%g5 + %g6], %g5
1433 brz,pn %g5, tl0_skip_utrap
1439 ! 0) save trap state to memory
1440 ldx [PCPU_REG + PC_CURPCB], %g6
1442 stx %g5, [%g6 + PCB_TSTATE]
1444 stx %g5, [%g6 + PCB_TPC]
1446 stx %g5, [%g6 + PCB_TNPC]
1447 stx %g2, [%g6 + PCB_TT]
1448 stx %g3, [%g6 + PCB_SFAR]
1455 /* we need to determine from the hardware the number of register windows */
1456 sethi %hi(nwin_minus_one), %g5
1457 ld [%g5 + %lo(nwin_minus_one)], %g5
1459 mov nwin_minus_one, %g5
1461 ldx [PCPU_REG + PC_CURPCB], %g6
1462 wrpr %g0, %g5, %cleanwin
1463 ldx [%g6 + PCB_KSTACK], %g6
1466 set PCPU_PAGES*PAGE_SIZE - PC_SIZEOF, %g6
1468 sub %g6, SPOFF + CCFSZ, %g6
1469 mov T_KSTACK_FAULT, %g2
1472 sub %g6, TF_SIZEOF, %g6
1475 rdpr %canrestore, %l0
1477 wrpr %g0, 0, %canrestore
1478 sllx %l1, WSTATE_SHIFT, %l1
1479 wrpr %l1, WSTATE_K64, %wstate
1480 wrpr %g0, %l0, %otherwin
1482 ! set pcontext to run kernel
1486 sethi %hi(FLUSH_ADDR), %l2
1487 SET_MMU_CONTEXT(%l1, %l0)
1488 flush %l2 ! flush/membar required by immu for
1489 ! consistency guarantee
1492 mov %g1, %l3 ! set trap/interrupt for tl0
1497 sll %g1, RW_SHIFT, %g1
1498 add %g1, PC_TSBWBUF, %g1
1499 add PCPU_REG, %g1, %g1
1501 TTRACE_ADD_SAFE(%g1, 0, 0, 0, 0, 0)
1504 mov %g2, %o1 ! trap type
1505 mov %g3, %o2 ! fault info if set
1506 mov %g5, %l6 ! %pil if priv trap
1508 ! save state in trapframe
1510 add %sp, REGOFF + SPOFF, %l7
1514 stx %l0, [%l7 + TF_TPC]
1516 stx %l1, [%l7 + TF_TNPC]
1517 stx %l2, [%l7 + TF_TSTATE]
1518 stx %l0, [%l7 + TF_FPRS]
1521 * According to the sparc64 port fp must me enabled
1522 * before reading %fsr
1524 wr %g0, FPRS_FEF, %fprs
1525 stx %fsr, [%l7 + TF_FSR]
1544 wrpr %g0, %g6, %tnpc
1546 ! save g7 before it can be overwritten by PCPU when returning from an interrupt
1548 stx %g7, [%l7 + TF_G7]
1552 set TSTATE_KERNEL, %l1
1553 wrpr %l1, %l0, %tstate
1561 * workaround for CPP brokenness
1563 #define LOADLOAD #LoadLoad
1564 #define LOADSTORE #LoadStore
1565 #define STORESTORE #StoreStore
1570 #define ENTER LOADLOAD
1571 #define EXIT LOADSTORE|STORESTORE
1577 #define THE_LOCK_ENTER(addr, lock_bit, oldval, newval, label1) \
1579 add addr, 8, addr ; \
1580 sllx lock_bit, 56, lock_bit ; \
1582 ldxa [addr]%asi, oldval; \
1583 or oldval, lock_bit, newval; \
1584 andn oldval, lock_bit, oldval; \
1585 casxa [addr]%asi, oldval, newval; \
1586 cmp newval, oldval ; \
1587 bne,pn %xcc, label1 ## b ; \
1591 #define THE_LOCK_EXIT(addr, lock_bit, tmp)\
1593 ldxa [addr + 8]%asi, tmp ; \
1594 andn tmp, lock_bit, tmp ; \
1595 stxa tmp, [addr + 8]%asi ;
1597 #define HASH_LOOKUP(addr, tag, searchtag, faillabel, matchlabel) \
1598 ldda [addr]%asi, tag ; \
1600 be,pn %xcc, faillabel ; \
1602 cmp tag, searchtag ; \
1603 be,pn %xcc, matchlabel ;\
1606 #define RESTORE_TRAPWIN(pcpu, cansave, label1, label2) \
1607 brz cansave, label1 ## f; \
1610 ba,a,pt %xcc, label2 ## f ; \
1612 rdpr %tl, cansave ; \
1614 sll cansave, RW_SHIFT, cansave ; \
1615 add cansave, PC_TSBWBUF, cansave ; \
1616 add pcpu, cansave, cansave ; \
1617 RESTORE_LOCALS_ASI(cansave) ; \
1620 ENTRY(hash_bucket_lock)
1623 and %o1, PSTATE_INTR_DISABLE, %o2
1625 THE_LOCK_ENTER(%o0, %o3, %o4, %o5, 1)
1629 END(hash_bucket_lock)
1632 ENTRY(hash_bucket_unlock)
1636 THE_LOCK_EXIT(%o0, %g2, %g3)
1640 END(hash_bucket_unlock)
1644 ! %g4==fault type (if data miss)
1647 ! %g1==absolute index
1648 ! %g2==hash base, pointer to hash entry
1649 ! %g3==flag bits, TSB (RA)
1650 ! %g4==fault type,entry tag
1654 ENTRY(tsb_miss_handler)
1655 ldxa [%g1 + %g7]ASI_REAL, %g6 ! load in the context
1657 GET_HASH_SCRATCH_USER(%g2)
1658 GET_TSB_SCRATCH_USER(%g4)
1662 GET_HASH_SCRATCH_KERNEL(%g2)
1663 GET_TSB_SCRATCH_KERNEL(%g4)
1666 rdpr %tl, %g1 ! need to use real addresses?
1672 brz,pt %g1, 3f ! for tl == 1
1674 sethi %uhi(VM_MIN_DIRECT_ADDRESS), %g1
1675 wr %g0, ASI_REAL, %asi
1677 mov ASI_LDTD_REAL, %g3
1684 /* XXX use save operation if %g1 > 0 and tl == 1 */
1688 sll %g1, RW_SHIFT, %g1
1689 add %g1, PC_TSBWBUF, %g1
1690 add PCPU_REG, %g1, %g1
1692 TTRACE_ADD_SAFE(%g1, 0, 0, 0, 0, 0)
1694 SAVE_LOCALS_ASI(%g1)
1695 mov 0, %g1 ! cansave is 0
1697 ! %g2 == hash scratch value
1703 srlx %g5, TTARGET_VA_SHIFT, %l0
1704 sllx %g6, TTARGET_CTX_SHIFT, %l1
1705 or %l0, %l1, %l2 ! %l2 == search tag
1707 tsb_miss_compute_hash_addr:
1708 sethi %hi(PAGE_SIZE), %l0
1709 sub %l0, 1, %l1 ! %l1==PAGE_MASK
1711 and %g2, %l1, %l3 ! size stored in lower 13 bits
1712 andn %g2, %l1, %g2 ! actual VA/RA of hash
1714 ! XXX only handle 8k page miss
1715 ! calculate hash index
1716 srlx %g5, PAGE_SHIFT, %l4 ! absolute hash index
1717 sllx %l3, (PAGE_SHIFT - THE_SHIFT), %l0 ! size of hash in THEs
1718 sub %l0, 1, %l5 ! THE_MASK
1719 and %l4, %l5, %l5 ! masked hash index
1720 sllx %l5, THE_SHIFT, %l5 ! masked hash offset
1721 ! fetch hash entries - exit when we find what were looking for
1724 add %g2, %l5, %g2 ! base + offset == entry base
1727 THE_LOCK_ENTER(%g2, %l0, %l7, %l6, 6)
1739 ! %l4 == absolute index
1742 ! %l5 == saved head of bucket
1748 mov %g2, %l5 ! save head of bucket
1749 rdpr %tt, %g3 ! reload trap type
1752 HASH_LOOKUP(%g2, %l6, %l2, tsb_miss_not_found, tsb_miss_found)
1755 HASH_LOOKUP(%g2, %l6, %l2, tsb_miss_not_found, tsb_miss_found)
1758 HASH_LOOKUP(%g2, %l6, %l2, tsb_miss_not_found, tsb_miss_found)
1759 #if HASH_ENTRY_SHIFT > 2
1762 HASH_LOOKUP(%g2, %l6, %l2, tsb_miss_not_found, tsb_miss_found)
1765 HASH_LOOKUP(%g2, %l6, %l2, tsb_miss_not_found, tsb_miss_found)
1768 HASH_LOOKUP(%g2, %l6, %l2, tsb_miss_not_found, tsb_miss_found)
1771 HASH_LOOKUP(%g2, %l6, %l2, tsb_miss_not_found, tsb_miss_found)
1777 sethi %uhi(VM_MIN_DIRECT_ADDRESS), %g3
1782 andn %l7, %g3, %l7 ! generate real address
1785 sethi %hi(0xcafebabe), %g3
1787 or %g3, %lo(0xcafebabe), %g3
1790 beq,pt %xcc, tsb_miss_lookup_0
1794 ! we need to jump to tl0_trap to drop us back down to tl0
1795 ! and take us to trap(...) to service the fault
1797 THE_LOCK_EXIT(%l5, %l0, %g2)
1799 andn %g5, %l1, %g5 ! fault page PA
1801 RESTORE_TRAPWIN(PCPU_REG, %g1, 14, 15)
1803 ! convert hardware trap type to kernel trap type
1804 set trap_conversion, %g2
1805 sllx %g3, INT_SHIFT, %g3
1808 sethi %hi(trap), %g1
1809 or %g6, %g5, %g3 ! trap data
1810 sub %g0, 1, %g4 ! pil info
1812 or %g1, %lo(trap), %g1
1817 cmp %g3, TT_DATA_MISS ! TSB data miss
1819 or %l7, VTD_REF, %l7 ! set referenced unconditionally
1820 cmp %g3, TT_INSTRUCTION_MISS ! TSB instruction miss
1823 cmp %g3, TT_DATA_PROTECTION ! protection fault
1824 bne,pn %xcc, unsupported_fault_trap ! we don't handle any other fault types currently
1826 andcc %l7, VTD_SW_W, %g0 ! write enabled?
1827 bz,a,pn %xcc, prot_fault_trap ! write to read only page
1829 or %l7, VTD_W, %l7 ! add modifed bit
1832 andn %l7, %l0, %l7 ! remove lock bit
1834 and %g4, %l1, %g3 ! shift of TSB in pages
1835 andn %g4, %l1, %l3 ! TSB real address
1838 add %g3, (PAGE_SHIFT - TTE_SHIFT), %g3 ! add shift value for number of ttes / page
1839 sllx %l2, %g3, %g3 ! nttes
1841 subx %g3, 1, %g3 ! TSB_MASK
1842 and %g3, %l4, %g3 ! masked index
1843 sllx %g3, TTE_SHIFT, %g3 ! masked byte offset
1844 add %g3, %l3, %g3 ! TTE RA
1846 rdpr %tl, %l2 ! don't track misses for kernel context
1849 brnz,pn %l2, 13f ! don't track misses at tl > 1
1851 ldx [PCPU(PMAP)], %l1
1852 ld [%l1 + PM_TSB_MISS_COUNT], %l3
1854 st %l3, [%l1 + PM_TSB_MISS_COUNT]
1856 ! if the data value is zero then this is not a capacity miss
1857 ldda [%g3]ASI_LDTD_N, %l2
1860 ! compare the tag bits - if they're the same we're merely
1861 ! changing the mapping
1868 ld [%l1 + PM_TSB_CAP_MISS_COUNT], %l3
1870 st %l3, [%l1 + PM_TSB_CAP_MISS_COUNT]
1872 13: stxa %g0, [%g3 + 8]%asi ! invalidate data
1875 stxa %l6, [%g3]%asi ! store tag
1876 stxa %l7, [%g3 + 8]%asi ! store data
1877 stxa %l7, [%g2 + 8]%asi ! update TTE with ref bit
1880 THE_LOCK_EXIT(%l5, %l0, %l7)
1881 RESTORE_TRAPWIN(PCPU_REG, %g1, 13, 16)
1884 cmp %g3, TT_DATA_PROTECTION
1885 beq,pn %xcc, demap_begin
1886 sethi %hi(PAGE_SIZE), %g1
1901 END(tsb_miss_handler)
1905 * Write to read-only page
1912 ! %l5 == head of bucket
1914 ENTRY(prot_fault_trap)
1915 THE_LOCK_EXIT(%l5, %l0, %g2)
1916 RESTORE_TRAPWIN(PCPU_REG, %g1, 14, 15)
1917 sethi %hi(trap), %g1
1918 mov T_DATA_PROTECTION, %g2
1922 or %g1, %lo(trap), %g1
1923 END(prot_fault_trap)
1927 ENTRY(unsupported_fault_trap)
1932 END(unsupported_fault_trap)
1936 * Freshly forked processes come here when switched to for the first time.
1937 * The arguments to fork_exit() have been setup in the locals, we must move
1940 ENTRY(fork_trampoline)
1945 ba,pt %xcc, user_rtt
1946 add %sp, CCFSZ + SPOFF, %l7
1947 END(fork_trampoline)
1951 .comm intrnames, IV_NAMLEN
1954 .comm intrcnt, IV_MAX * 8
1957 #define TRAP_ENTRY_SHIFT 5
1958 #define TRAP_ENTRY_MASK 0x1ff
1960 ! assume no tl1 handler
1963 set rtt_fill_start, %g6
1967 set rtt_fill_end, %g6
1971 set fault_rtt_fn1, %g7
1974 #ifdef USING_OPTIMIZED_SPILL_FILL
1975 set fill_slow_start, %g6
1979 set fill_slow_end, %g6
1987 bgeu,a,pn %xcc, ptl1_panic
1988 mov PTL1_BAD_TRAP, %g1
1989 ! tpc is in the trap table
1990 ! convert to trap index
1991 srl %g7, TRAP_ENTRY_SHIFT, %g6
1992 and %g6, TRAP_ENTRY_MASK, %g6
1993 ! check for window trap type
1994 and %g6, WTRAP_TTMASK, %g6
1996 bne,a,pn %xcc, ptl1_panic
1997 mov PTL1_BAD_NOT_WTRAP, %g1
1999 andn %g7, WTRAP_ALIGN, %g7
2000 add %g7, WTRAP_FAULTOFF, %g7
2002 wrpr %g0, %g7, %tnpc
2005 GET_MMFSA_SCRATCH(%g7)
2007 ! convert hardware trap type to kernel trap type
2008 set trap_conversion, %g1
2009 sllx %g2, INT_SHIFT, %g2
2012 wr %g0, ASI_REAL, %asi
2013 ldxa [%g7 + MMFSA_D_ADDR]%asi, %g3
2014 ldxa [%g7 + MMFSA_D_CTX]%asi, %g7
2015 ! XXX we're only handling page faults here
2016 srlx %g3, PAGE_SHIFT, %g3
2017 sllx %g3, PAGE_SHIFT, %g3 ! mask off bottom
2021 sllx %g7, TRAP_CTX_SHIFT, %g7