2 * Copyright (c) 2002 Jake Burkholder.
3 * Copyright (c) 2006 Kip Macy <kmacy@FreeBSD.org>
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 #include <machine/asm.h>
29 __FBSDID("$FreeBSD$");
31 #include "opt_simulator.h"
32 #include "opt_trap_trace.h"
34 #include <machine/asi.h>
35 #include <machine/asmacros.h>
36 #include <machine/frame.h>
37 #include <machine/hypervisorvar.h>
38 #include <machine/intr_machdep.h>
39 #include <machine/ktr.h>
40 #include <machine/pstate.h>
46 mov CONS_PUTCHAR, %o5 ; \
53 1: ldx [PCPU(IRHEAD)], %l0
60 2: wrpr %g0, PSTATE_NORMAL, %pstate
62 ldx [%l0 + IR_NEXT], %l1
64 stx %l1, [PCPU(IRHEAD)]
65 PCPU_ADDR(IRHEAD, %l1)
66 stx %l1, [PCPU(IRTAIL)]
68 3: ldx [%l0 + IR_FUNC], %o0
69 ldx [%l0 + IR_ARG], %o1
70 lduw [%l0 + IR_VEC], %o2
72 /* intrcnt[intr_countp[%o2]]++ */
73 SET(intrcnt, %l7, %l2) /* %l2 = intrcnt */
74 prefetcha [%l2] ASI_N, 1
75 SET(intr_countp, %l7, %l3) /* %l3 = intr_countp */
76 sllx %o2, 1, %l4 /* %l4 = vec << 1 */
77 lduh [%l4 + %l3], %l5 /* %l5 = intr_countp[%o2] */
78 sllx %l5, 3, %l6 /* %l6 = intr_countp[%o2] << 3 */
79 add %l6, %l2, %l7 /* %l7 = intrcnt[intr_countp[%o2]] */
84 ldx [PCPU(IRFREE)], %l1
85 stx %l1, [%l0 + IR_NEXT]
86 stx %l0, [PCPU(IRFREE)]
88 wrpr %g0, PSTATE_KERNEL, %pstate
98 * Running tally of invalid CPU mondo interrupts
101 uint64_t cpu_mondo_invalid;
104 .globl cpu_mondo_invalid
119 * (TT 0x7c, TL>0) CPU Mondo Queue Handler
120 * Globals are the Interrupt Globals.
122 * Interrupts in sun4v are delivered to privileged code in the form
123 * of interrupt reports. Each interrupt report is 64-bytes long,
124 * consisting of 8 64-bit words. Each interrupt report is appended
125 * onto the appropriate interrupt queue of which there are currently two,
126 * one for CPU mondos (formerly referred to as cross trap function
127 * requests) and one for device mondos. Each queue has its own
130 * New reports are appended onto the tail of the queue, and privileged
131 * code reads the report from the queue's head. The head pointer is stored
132 * in a register and is equal to the current offset from the base address
133 * of its associated queue.
135 * The structure of the cpu mondo report follows sun4u conventions:
137 * word 0: address of cross-trap handler
138 * word 1: first argument to handler
139 * word 2: second argument to handler
142 * Privileged code is responsible for incrementing the head pointer
143 * to remove each entry. The pointers are updated using modulo
144 * arithmetic such that the queue is empty when head == tail and is full
145 * when the addition of an entry would make head == tail.
147 * This trap handler is called when the cpu mondo queue becomes non-empty
148 * and will continue to be called while interrupts enabled until the
149 * queue becomes empty.
154 GET_PCPU_PHYS_SCRATCH(%g1)
157 sll %g1, RW_SHIFT, %g1
158 add %g1, PC_TSBWBUF, %g1
159 add PCPU_REG, %g1, %g1
160 wr %g0, ASI_REAL, %asi
161 TTRACE_ADD_SAFE(%g1, 0, 0, 0, 0, 0)
165 ! %g5 PC for fasttrap TL>0 handler
168 ! %g3 queue base RA, ackmask
169 ! %g4 queue size mask
171 mov CPU_MONDO_QUEUE_HEAD, %g1
172 ldxa [%g1]ASI_QUEUE, %g6 ! %g6 = head ptr
173 mov CPU_MONDO_QUEUE_TAIL, %g2
174 ldxa [%g2]ASI_QUEUE, %g4 ! %g4 = tail ptr
176 be,pn %xcc, 0f ! head == tail
180 * Get the address of the current CPU and index into
181 * the pcpu structure for the Q values.
184 ldx [PCPU(CPU_Q_RA)], %g3 ! %g3 = queue base PA
185 ldx [PCPU(CPU_Q_SIZE)], %g4 ! %g4 = queue size
186 sub %g4, 1, %g4 ! %g4 = queue size mask
188 ! Load interrupt receive data registers 1 and 2 to fetch
189 ! the arguments for the fast trap handler.
191 ! Since the data words in the interrupt report are not defined yet
192 ! we assume that the consective words contain valid data and preserve
193 ! sun4u's xcall mondo arguments.
195 ! %g5 PC for fasttrap TL>0 handler
199 ldxa [%g3 + %g6]ASI_REAL, %g5 ! get PC from q base + head
200 add %g6, 0x8, %g6 ! inc head 8 bytes
201 ldxa [%g3 + %g6]ASI_REAL, %g1 ! read data word 1
202 add %g6, 0x8, %g6 ! inc head 8 bytes
203 ldxa [%g3 + %g6]ASI_REAL, %g2 ! read data word 2
204 add %g6, 0x8, %g6 ! inc head 8 bytes
205 ldxa [%g3 + %g6]ASI_REAL, %g7 ! read data word 3
206 add %g6, (INTR_REPORT_SIZE - 24) , %g6 ! inc head to next record
208 and %g6, %g4, %g6 ! and size mask for wrap around
209 mov CPU_MONDO_QUEUE_HEAD, %g3
210 stxa %g6, [%g3]ASI_QUEUE ! store head pointer
213 mov %g7, %g3 ! ackmask
215 * For now catch invalid PC being passed via cpu_mondo queue
219 bl,a,pn %xcc, 1f ! branch if bad %pc
222 jmp %g5 ! jump to traphandler
225 ! invalid trap handler, discard it for now
226 set cpu_mondo_invalid, %g4
245 * (TT 0x7d, TL>0) Dev Mondo Queue Handler
246 * Globals are the Interrupt Globals.
248 * Device interrupt reports take the following form:
249 * Bits: 63 11 10 6 5 0
250 * word 0: | zero | | bus port | | device ID |
256 GET_PCPU_PHYS_SCRATCH(%g1)
259 sll %g1, RW_SHIFT, %g1
260 add %g1, PC_TSBWBUF, %g1
261 add PCPU_REG, %g1, %g1
262 wr %g0, ASI_REAL, %asi
263 TTRACE_ADD_SAFE(%g1, 0, 0, 0, 0, 0)
268 * %g1 ptr to intrerrupt vector entry for inum
270 * %g4 queue size mask
280 mov DEV_MONDO_QUEUE_HEAD, %g1
281 ldxa [%g1]ASI_QUEUE, %g6 ! %g6 = head ptr
282 mov DEV_MONDO_QUEUE_TAIL, %g2
283 ldxa [%g2]ASI_QUEUE, %g4 ! %g4 = tail ptr
285 be,pn %xcc, 0f ! head == tail
289 * Get CPU address and index to Q base
292 ldx [PCPU(DEV_Q_RA)], %g3 ! %g3 = queue base PA
298 ldxa [%g3 + %g6]ASI_REAL, %g5 ! %g5 = inum from q base + head
301 * We verify that inum is valid ( < IV_MAX). If it is greater
302 * than IV_MAX, we let the software interrupt handler take care
308 ldx [PCPU(DEV_Q_SIZE)], %g4 ! queue size (delay slot)
311 * Find the function, argument and desired priority from the
314 set intr_vectors, %g1
315 sll %g5, IV_SHIFT, %g2
316 add %g1, %g2, %g1 ! %g1 = &intr_vectors[inum]
319 * Get an intr_request from the free list. There should always be one
320 * unless we are getting an interrupt storm from stray interrupts, in
321 * which case the we will dereference a NULL pointer and panic.
323 ldx [PCPU(IRFREE)], %g2 ! %g2 = ptr to tail element
326 * Sun4v-fixme: It might be better to have the software interrupt
327 * handler deal with this scenario rather than just panic.
330 !ldx [PCPU(DEV_Q_SIZE)], %g4 ! queue size (delay slot)
332 ldx [%g2 + IR_NEXT], %g3 ! %g3 = next element in list
333 stx %g3, [PCPU(IRFREE)] ! %g3 is now the tail
336 * Store the vector number, function, argument, and priority.
338 stw %g5, [%g2 + IR_VEC]
339 ldx [%g1 + IV_FUNC], %g3 ! %g3 = function ptr
340 stx %g3, [%g2 + IR_FUNC]
341 ldx [%g1 + IV_ARG], %g3 ! %g3 = argument ptr
342 stx %g3, [%g2 + IR_ARG]
343 lduw [%g1 + IV_PRI], %g1 ! %g1 = priority
344 stw %g1, [%g2 + IR_PRI]
347 * Link it onto the end of the active list
349 stx %g0, [%g2 + IR_NEXT]
350 ldx [PCPU(IRTAIL)], %g3 ! %g3 = ptr to tail
352 add %g2, IR_NEXT, %g2
353 stx %g2, [PCPU(IRTAIL)]
356 * Unlike Solaris, FreeBSD throws away the rest of the
357 * 64-byte packet. We may want to revisit this philosophy
358 * at some point since information in the packet is defined by
359 * the device and MAY be meaningful.
361 * For now, we retain FreeBSD's sun4u semantics.
363 ldx [PCPU(DEV_Q_SIZE)], %g4 ! queue size
365 1: sub %g4, 1, %g4 ! %g4 = queue size mask
366 add %g6, INTR_REPORT_SIZE, %g6 ! inc head to next record
367 and %g6, %g4, %g6 ! and mask for wrap around
368 mov DEV_MONDO_QUEUE_HEAD, %g3
369 stxa %g6, [%g3]ASI_QUEUE ! increment head offset
373 * Trigger a softint at the level indicated by the priority
377 wr %g6, 0, %set_softint
387 * Done, retry the instruction
405 mov MAP_ITLB|MAP_DTLB, %o3
406 mov MMU_DEMAP_CTX, %o5
408 brnz,a,pn %o0, interrupt_panic_bad_hcall
409 mov MMU_DEMAP_CTX, %o1
415 ba,pt %xcc, set_ackmask
426 mov MAP_ITLB | MAP_DTLB, %o2
427 mov MMU_DEMAP_ALL, %o5
429 brnz,a,pn %o0, interrupt_panic_bad_hcall
430 mov MMU_DEMAP_ALL, %o1
435 ba,pt %xcc, set_ackmask
443 mov MAP_ITLB|MAP_DTLB, %o2
447 brnz,a,pn %o0, interrupt_panic_bad_hcall
448 mov MMU_UNMAP_ADDR, %o1
452 ba,pt %xcc, set_ackmask
457 sethi %hi(PAGE_SIZE), %g5
466 mov MAP_ITLB|MAP_DTLB, %o2
470 brnz,a,pn %o0, interrupt_panic_bad_hcall
471 mov MMU_UNMAP_ADDR, %o1
479 ba,pt %xcc, set_ackmask
486 /* compare current context with one to be updated */
488 GET_MMU_CONTEXT(%g4, %g4)
491 bne,a,pt %xcc, set_ackmask
493 /* update scratch pointer to tsbscratch */
494 wr %g0, ASI_REAL, %asi
495 ldxa [%g2]%asi, %g4 ! tsbscratch
496 ldxa [%g2 + 8]%asi, %g2 ! tsb_ra
497 mov SCRATCH_REG_TSB_USER, %g5
498 SET_SCRATCH(%g5, %g4)
500 /* reset tsb context with hypervisor */
506 mov MMU_TSB_CTXNON0, %o5
513 ba,pt %xcc, set_ackmask
517 ENTRY(tl_ttehashupdate)
518 /* compare current context with one to be updated */
520 GET_MMU_CONTEXT(%g4, %g4)
522 bne,a,pt %xcc, set_ackmask
524 /* update scratch pointer to hashscratch */
525 mov SCRATCH_REG_HASH_USER, %g4
526 SET_SCRATCH(%g4, %g2)
527 END(tl_ttehashupdate)
530 GET_PCPU_PHYS_SCRATCH(%g6)
531 wr %g0, ASI_REAL, %asi
534 /* pcpu->pad[0] = %pc */
536 stxa %g4, [PCPU(PAD)]%asi
538 /* pcpu->pad[1] = %tick */
540 stxa %g4, [PCPU(PAD) + 8]%asi
543 * Increment a counter which might help us notice if we're
544 * stuck in a loop. pcpu->pad[2] = count
546 ldxa [PCPU(PAD) + 16]%asi, %g4
548 stxa %g4, [PCPU(PAD) + 16]%asi
551 lda [PCPU(CPUMASK)]%asi, %g4
554 casa [%g3]%asi, %g1, %g2
562 * interrupt_panic_bad_hcall is called when a hcall returns
569 interrupt_bad_hcall_error:
570 .asciz "hypervisor call 0x%x returned an unexpected error %d"
573 ENTRY(interrupt_panic_bad_hcall)
580 sethi %hi(interrupt_bad_hcall_error), %o0
581 or %o0, %lo(interrupt_bad_hcall_error), %o0
585 END(interrupt_panic_bad_hcall)