1 /******************************************************************************
4 * Xen event and interrupt services for x86 HVM guests.
6 * Copyright (c) 2002-2005, K A Fraser
7 * Copyright (c) 2005, Intel Corporation <xiaofeng.ling@intel.com>
8 * Copyright (c) 2012, Spectra Logic Corporation
9 * Copyright © 2021-2023, Elliott Mitchell
11 * This file may be distributed separately from the Linux kernel, or
12 * incorporated into other software packages, subject to the following license:
14 * Permission is hereby granted, free of charge, to any person obtaining a copy
15 * of this source file (the "Software"), to deal in the Software without
16 * restriction, including without limitation the rights to use, copy, modify,
17 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
18 * and to permit persons to whom the Software is furnished to do so, subject to
19 * the following conditions:
21 * The above copyright notice and this permission notice shall be included in
22 * all copies or substantial portions of the Software.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
25 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
26 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
27 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
28 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
29 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
33 #include <sys/cdefs.h>
34 __FBSDID("$FreeBSD$");
38 #include <sys/param.h>
39 #include <sys/systm.h>
41 #include <sys/malloc.h>
42 #include <sys/kernel.h>
43 #include <sys/limits.h>
45 #include <sys/mutex.h>
46 #include <sys/interrupt.h>
49 #include <sys/refcount.h>
54 #include <machine/intr_machdep.h>
55 #include <x86/apicvar.h>
56 #include <x86/apicreg.h>
57 #include <machine/smp.h>
58 #include <machine/stdarg.h>
60 #include <xen/xen-os.h>
62 #include <xen/hypervisor.h>
63 #include <xen/xen_intr.h>
64 #include <xen/evtchn/evtchnvar.h>
66 #include <dev/xen/xenpci/xenpcivar.h>
67 #include <dev/pci/pcivar.h>
73 /* The code below is the implementation of 2L event channels. */
74 #define NR_EVENT_CHANNELS EVTCHN_2L_NR_CHANNELS
76 static MALLOC_DEFINE(M_XENINTR, "xen_intr", "Xen Interrupt Services");
78 static u_int first_evtchn_irq;
81 * Per-cpu event channel processing state.
83 struct xen_intr_pcpu_data {
85 * The last event channel bitmap section (level one bit) processed.
86 * This is used to ensure we scan all ports before
87 * servicing an already servied port again.
89 u_int last_processed_l1i;
92 * The last event channel processed within the event channel
93 * bitmap being scanned.
95 u_int last_processed_l2i;
97 /** Pointer to this CPU's interrupt statistic counter. */
98 u_long *evtchn_intrcnt;
101 * A bitmap of ports that can be serviced from this CPU.
102 * A set bit means interrupt handling is enabled.
104 u_long evtchn_enabled[sizeof(u_long) * 8];
108 * Start the scan at port 0 by initializing the last scanned
109 * location as the highest numbered event channel port.
111 DPCPU_DEFINE_STATIC(struct xen_intr_pcpu_data, xen_intr_pcpu) = {
112 .last_processed_l1i = LONG_BIT - 1,
113 .last_processed_l2i = LONG_BIT - 1
116 DPCPU_DECLARE(struct vcpu_info *, vcpu_info);
118 #define INVALID_EVTCHN (~(evtchn_port_t)0) /* Invalid event channel */
119 #define is_valid_evtchn(x) ((uintmax_t)(x) < NR_EVENT_CHANNELS)
122 struct intsrc xi_intsrc;
123 enum evtchn_type xi_type;
124 int xi_cpu; /* VCPU for delivery. */
125 int xi_vector; /* Global isrc vector number. */
126 evtchn_port_t xi_port;
129 u_int xi_close:1; /* close on unbind? */
131 volatile u_int xi_refcount;
134 static void xen_intr_suspend(struct pic *);
135 static void xen_intr_resume(struct pic *, bool suspend_cancelled);
136 static void xen_intr_enable_source(struct intsrc *isrc);
137 static void xen_intr_disable_source(struct intsrc *isrc, int eoi);
138 static void xen_intr_eoi_source(struct intsrc *isrc);
139 static void xen_intr_enable_intr(struct intsrc *isrc);
140 static void xen_intr_disable_intr(struct intsrc *isrc);
141 static int xen_intr_vector(struct intsrc *isrc);
142 static int xen_intr_source_pending(struct intsrc *isrc);
143 static int xen_intr_config_intr(struct intsrc *isrc,
144 enum intr_trigger trig, enum intr_polarity pol);
145 static int xen_intr_assign_cpu(struct intsrc *isrc, u_int apic_id);
148 * PIC interface for all event channel port types except physical IRQs.
150 struct pic xen_intr_pic = {
151 .pic_enable_source = xen_intr_enable_source,
152 .pic_disable_source = xen_intr_disable_source,
153 .pic_eoi_source = xen_intr_eoi_source,
154 .pic_enable_intr = xen_intr_enable_intr,
155 .pic_disable_intr = xen_intr_disable_intr,
156 .pic_vector = xen_intr_vector,
157 .pic_source_pending = xen_intr_source_pending,
158 .pic_suspend = xen_intr_suspend,
159 .pic_resume = xen_intr_resume,
160 .pic_config_intr = xen_intr_config_intr,
161 .pic_assign_cpu = xen_intr_assign_cpu
164 static struct mtx xen_intr_isrc_lock;
165 static u_int xen_intr_auto_vector_count;
166 static struct xenisrc *xen_intr_port_to_isrc[NR_EVENT_CHANNELS];
168 /*------------------------- Private Functions --------------------------------*/
171 * Retrieve a handle for a Xen interrupt source.
173 * \param isrc A valid Xen interrupt source structure.
175 * \returns A handle suitable for use with xen_intr_isrc_from_handle()
176 * to retrieve the original Xen interrupt source structure.
179 static inline xen_intr_handle_t
180 xen_intr_handle_from_isrc(struct xenisrc *isrc)
186 * Lookup a Xen interrupt source object given an interrupt binding handle.
188 * \param handle A handle initialized by a previous call to
189 * xen_intr_bind_isrc().
191 * \returns A pointer to the Xen interrupt source object associated
192 * with the given interrupt handle. NULL if no association
195 static inline struct xenisrc *
196 xen_intr_isrc_from_handle(xen_intr_handle_t handle)
198 return ((struct xenisrc *)handle);
202 * Disable signal delivery for an event channel port on the
205 * \param port The event channel port to mask.
207 * This API is used to manage the port<=>CPU binding of event
210 * \note This operation does not preclude reception of an event
211 * for this event channel on another CPU. To mask the
212 * event channel globally, use evtchn_mask().
215 evtchn_cpu_mask_port(u_int cpu, evtchn_port_t port)
217 struct xen_intr_pcpu_data *pcpu;
219 pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu);
220 xen_clear_bit(port, pcpu->evtchn_enabled);
224 * Enable signal delivery for an event channel port on the
227 * \param port The event channel port to unmask.
229 * This API is used to manage the port<=>CPU binding of event
232 * \note This operation does not guarantee that event delivery
233 * is enabled for this event channel port. The port must
234 * also be globally enabled. See evtchn_unmask().
237 evtchn_cpu_unmask_port(u_int cpu, evtchn_port_t port)
239 struct xen_intr_pcpu_data *pcpu;
241 pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu);
242 xen_set_bit(port, pcpu->evtchn_enabled);
246 * Allocate and register a per-cpu Xen upcall interrupt counter.
248 * \param cpu The cpu for which to register this interrupt count.
251 xen_intr_intrcnt_add(u_int cpu)
253 char buf[MAXCOMLEN + 1];
254 struct xen_intr_pcpu_data *pcpu;
256 pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu);
257 if (pcpu->evtchn_intrcnt != NULL)
260 snprintf(buf, sizeof(buf), "cpu%d:xen", cpu);
261 intrcnt_add(buf, &pcpu->evtchn_intrcnt);
265 * Search for an already allocated but currently unused Xen interrupt
268 * \param type Restrict the search to interrupt sources of the given
271 * \return A pointer to a free Xen interrupt source object or NULL.
273 static struct xenisrc *
274 xen_intr_find_unused_isrc(enum evtchn_type type)
278 KASSERT(mtx_owned(&xen_intr_isrc_lock), ("Evtchn isrc lock not held"));
280 for (isrc_idx = 0; isrc_idx < xen_intr_auto_vector_count; isrc_idx ++) {
281 struct xenisrc *isrc;
284 vector = first_evtchn_irq + isrc_idx;
285 isrc = (struct xenisrc *)intr_lookup_source(vector);
287 * Since intr_register_source() must be called while unlocked,
288 * isrc == NULL *will* occur, though very infrequently.
290 * This also allows a very small gap where a foreign intrusion
291 * into Xen's interrupt range could be examined by this test.
293 if (__predict_true(isrc != NULL) &&
294 __predict_true(isrc->xi_intsrc.is_pic == &xen_intr_pic) &&
295 isrc->xi_type == EVTCHN_TYPE_UNBOUND) {
296 KASSERT(isrc->xi_intsrc.is_handlers == 0,
297 ("Free evtchn still has handlers"));
298 isrc->xi_type = type;
306 * Allocate a Xen interrupt source object.
308 * \param type The type of interrupt source to create.
310 * \return A pointer to a newly allocated Xen interrupt source
313 static struct xenisrc *
314 xen_intr_alloc_isrc(enum evtchn_type type)
317 struct xenisrc *isrc;
321 KASSERT(mtx_owned(&xen_intr_isrc_lock), ("Evtchn alloc lock not held"));
323 if (xen_intr_auto_vector_count >= NR_EVENT_CHANNELS) {
326 printf("%s: Xen interrupts exhausted.\n", __func__);
331 vector = first_evtchn_irq + xen_intr_auto_vector_count;
332 xen_intr_auto_vector_count++;
334 KASSERT((intr_lookup_source(vector) == NULL),
335 ("Trying to use an already allocated vector"));
337 mtx_unlock(&xen_intr_isrc_lock);
338 isrc = malloc(sizeof(*isrc), M_XENINTR, M_WAITOK | M_ZERO);
339 isrc->xi_intsrc.is_pic = &xen_intr_pic;
340 isrc->xi_vector = vector;
341 isrc->xi_type = type;
342 error = intr_register_source(&isrc->xi_intsrc);
344 panic("%s(): failed registering interrupt %u, error=%d\n",
345 __func__, vector, error);
346 mtx_lock(&xen_intr_isrc_lock);
352 * Attempt to free an active Xen interrupt source object.
354 * \param isrc The interrupt source object to release.
356 * \returns EBUSY if the source is still in use, otherwise 0.
359 xen_intr_release_isrc(struct xenisrc *isrc)
362 KASSERT(isrc->xi_intsrc.is_handlers == 0,
363 ("Release called, but xenisrc still in use"));
364 mtx_lock(&xen_intr_isrc_lock);
365 if (is_valid_evtchn(isrc->xi_port)) {
366 evtchn_mask_port(isrc->xi_port);
367 evtchn_clear_port(isrc->xi_port);
369 /* Rebind port to CPU 0. */
370 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port);
371 evtchn_cpu_unmask_port(0, isrc->xi_port);
373 if (isrc->xi_close != 0) {
374 struct evtchn_close close = { .port = isrc->xi_port };
376 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
377 panic("EVTCHNOP_close failed");
380 xen_intr_port_to_isrc[isrc->xi_port] = NULL;
383 isrc->xi_type = EVTCHN_TYPE_UNBOUND;
384 isrc->xi_port = INVALID_EVTCHN;
385 isrc->xi_cookie = NULL;
386 mtx_unlock(&xen_intr_isrc_lock);
391 * Associate an interrupt handler with an already allocated local Xen
392 * event channel port.
394 * \param isrcp The returned Xen interrupt object associated with
395 * the specified local port.
396 * \param local_port The event channel to bind.
397 * \param type The event channel type of local_port.
398 * \param intr_owner The device making this bind request.
399 * \param filter An interrupt filter handler. Specify NULL
400 * to always dispatch to the ithread handler.
401 * \param handler An interrupt ithread handler. Optional (can
402 * specify NULL) if all necessary event actions
403 * are performed by filter.
404 * \param arg Argument to present to both filter and handler.
405 * \param irqflags Interrupt handler flags. See sys/bus.h.
406 * \param handlep Pointer to an opaque handle used to manage this
409 * \returns 0 on success, otherwise an errno.
412 xen_intr_bind_isrc(struct xenisrc **isrcp, evtchn_port_t local_port,
413 enum evtchn_type type, const char *intr_owner, driver_filter_t filter,
414 driver_intr_t handler, void *arg, enum intr_type flags,
415 xen_intr_handle_t *port_handlep)
417 struct xenisrc *isrc;
421 if (port_handlep == NULL) {
422 printf("%s: %s: Bad event handle\n", intr_owner, __func__);
426 mtx_lock(&xen_intr_isrc_lock);
427 isrc = xen_intr_find_unused_isrc(type);
429 isrc = xen_intr_alloc_isrc(type);
431 mtx_unlock(&xen_intr_isrc_lock);
435 isrc->xi_port = local_port;
437 xen_intr_port_to_isrc[local_port] = isrc;
438 refcount_init(&isrc->xi_refcount, 1);
439 mtx_unlock(&xen_intr_isrc_lock);
441 /* Assign the opaque handler */
442 *port_handlep = xen_intr_handle_from_isrc(isrc);
445 if (type == EVTCHN_TYPE_PORT) {
447 * By default all interrupts are assigned to vCPU#0
448 * unless specified otherwise, so shuffle them to balance
449 * the interrupt load.
451 xen_intr_assign_cpu(&isrc->xi_intsrc, intr_next_cpu(0));
455 if (filter == NULL && handler == NULL) {
457 * No filter/handler provided, leave the event channel
458 * masked and without a valid handler, the caller is
459 * in charge of setting that up.
465 error = xen_intr_add_handler(intr_owner, filter, handler, arg, flags,
468 xen_intr_release_isrc(isrc);
476 * Determine the event channel ports at the given section of the
477 * event port bitmap which have pending events for the given cpu.
479 * \param pcpu The Xen interrupt pcpu data for the cpu being queried.
480 * \param sh The Xen shared info area.
481 * \param idx The index of the section of the event channel bitmap to
484 * \returns A u_long with bits set for every event channel with pending
488 xen_intr_active_ports(const struct xen_intr_pcpu_data *const pcpu,
491 volatile const shared_info_t *const sh = HYPERVISOR_shared_info;
493 CTASSERT(sizeof(sh->evtchn_mask[0]) == sizeof(sh->evtchn_pending[0]));
494 CTASSERT(sizeof(sh->evtchn_mask[0]) == sizeof(pcpu->evtchn_enabled[0]));
495 CTASSERT(sizeof(sh->evtchn_mask) == sizeof(sh->evtchn_pending));
496 CTASSERT(sizeof(sh->evtchn_mask) == sizeof(pcpu->evtchn_enabled));
497 return (sh->evtchn_pending[idx]
498 & ~sh->evtchn_mask[idx]
499 & pcpu->evtchn_enabled[idx]);
503 * Interrupt handler for processing all Xen event channel events.
505 * \param trap_frame The trap frame context for the current interrupt.
508 xen_intr_handle_upcall(struct trapframe *trap_frame)
510 u_int l1i, l2i, port, cpu __diagused;
511 u_long masked_l1, masked_l2;
512 struct xenisrc *isrc;
514 struct xen_intr_pcpu_data *pc;
518 * Disable preemption in order to always check and fire events
523 cpu = PCPU_GET(cpuid);
524 pc = DPCPU_PTR(xen_intr_pcpu);
525 v = DPCPU_GET(vcpu_info);
527 if (!xen_has_percpu_evtchn()) {
528 KASSERT((cpu == 0), ("Fired PCI event callback on wrong CPU"));
531 v->evtchn_upcall_pending = 0;
534 #ifndef CONFIG_X86 /* No need for a barrier -- XCHG is a barrier on x86. */
535 /* Clear master flag /before/ clearing selector flag. */
540 l1 = atomic_readandclear_long(&v->evtchn_pending_sel);
542 l1i = pc->last_processed_l1i;
543 l2i = pc->last_processed_l2i;
544 (*pc->evtchn_intrcnt)++;
547 l1i = (l1i + 1) % LONG_BIT;
548 masked_l1 = l1 & ((~0UL) << l1i);
550 if (masked_l1 == 0) {
552 * if we masked out all events, wrap around
559 l1i = ffsl(masked_l1) - 1;
562 l2 = xen_intr_active_ports(pc, l1i);
564 l2i = (l2i + 1) % LONG_BIT;
565 masked_l2 = l2 & ((~0UL) << l2i);
567 if (masked_l2 == 0) {
568 /* if we masked out all events, move on */
572 l2i = ffsl(masked_l2) - 1;
575 port = (l1i * LONG_BIT) + l2i;
576 evtchn_clear_port(port);
578 isrc = xen_intr_port_to_isrc[port];
579 if (__predict_false(isrc == NULL))
582 /* Make sure we are firing on the right vCPU */
583 KASSERT((isrc->xi_cpu == PCPU_GET(cpuid)),
584 ("Received unexpected event on vCPU#%d, event bound to vCPU#%d",
585 PCPU_GET(cpuid), isrc->xi_cpu));
587 intr_execute_handlers(&isrc->xi_intsrc, trap_frame);
590 * If this is the final port processed,
591 * we'll pick up here+1 next time.
593 pc->last_processed_l1i = l1i;
594 pc->last_processed_l2i = l2i;
596 } while (l2i != LONG_BIT - 1);
598 l2 = xen_intr_active_ports(pc, l1i);
601 * We handled all ports, so we can clear the
608 if (xen_evtchn_needs_ack)
615 xen_intr_init(void *dummy __unused)
617 shared_info_t *s = HYPERVISOR_shared_info;
618 struct xen_intr_pcpu_data *pcpu;
624 _Static_assert(is_valid_evtchn(0),
625 "is_valid_evtchn(0) fails (unused by Xen, but valid by interface");
626 _Static_assert(is_valid_evtchn(NR_EVENT_CHANNELS - 1),
627 "is_valid_evtchn(max) fails (is a valid channel)");
628 _Static_assert(!is_valid_evtchn(NR_EVENT_CHANNELS),
629 "is_valid_evtchn(>max) fails (NOT a valid channel)");
630 _Static_assert(!is_valid_evtchn(~(evtchn_port_t)0),
631 "is_valid_evtchn(maxint) fails (overflow?)");
632 _Static_assert(!is_valid_evtchn(INVALID_EVTCHN),
633 "is_valid_evtchn(INVALID_EVTCHN) fails (must be invalid!)");
634 _Static_assert(!is_valid_evtchn(-1),
635 "is_valid_evtchn(-1) fails (negative are invalid)");
637 mtx_init(&xen_intr_isrc_lock, "xen-irq-lock", NULL, MTX_DEF);
640 * Set the per-cpu mask of CPU#0 to enable all, since by default all
641 * event channels are bound to CPU#0.
644 pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu);
645 memset(pcpu->evtchn_enabled, i == 0 ? ~0 : 0,
646 sizeof(pcpu->evtchn_enabled));
649 for (i = 0; i < nitems(s->evtchn_mask); i++)
650 atomic_store_rel_long(&s->evtchn_mask[i], ~0);
652 intr_register_pic(&xen_intr_pic);
655 printf("Xen interrupt system initialized\n");
659 SYSINIT(xen_intr_init, SI_SUB_INTR, SI_ORDER_SECOND, xen_intr_init, NULL);
662 xen_intrcnt_init(void *dummy __unused)
670 * Register interrupt count manually as we aren't guaranteed to see a
671 * call to xen_intr_assign_cpu() before our first interrupt.
674 xen_intr_intrcnt_add(i);
676 SYSINIT(xen_intrcnt_init, SI_SUB_INTR, SI_ORDER_MIDDLE, xen_intrcnt_init, NULL);
679 xen_intr_alloc_irqs(void)
682 if (num_io_irqs > UINT_MAX - NR_EVENT_CHANNELS)
683 panic("IRQ allocation overflow (num_msi_irqs too high?)");
684 first_evtchn_irq = num_io_irqs;
685 num_io_irqs += NR_EVENT_CHANNELS;
688 /*--------------------------- Common PIC Functions ---------------------------*/
690 * Prepare this PIC for system suspension.
693 xen_intr_suspend(struct pic *unused)
698 xen_rebind_ipi(struct xenisrc *isrc)
701 int cpu = isrc->xi_cpu;
702 int vcpu_id = pcpu_find(cpu)->pc_vcpu_id;
704 struct evtchn_bind_ipi bind_ipi = { .vcpu = vcpu_id };
706 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
709 panic("unable to rebind xen IPI: %d", error);
711 isrc->xi_port = bind_ipi.port;
713 xen_intr_port_to_isrc[bind_ipi.port] = isrc;
715 error = xen_intr_assign_cpu(&isrc->xi_intsrc,
718 panic("unable to bind xen IPI to CPU#%d: %d",
721 evtchn_unmask_port(bind_ipi.port);
723 panic("Resume IPI event channel on UP");
728 xen_rebind_virq(struct xenisrc *isrc)
730 int cpu = isrc->xi_cpu;
731 int vcpu_id = pcpu_find(cpu)->pc_vcpu_id;
733 struct evtchn_bind_virq bind_virq = { .virq = isrc->xi_virq,
736 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
739 panic("unable to rebind xen VIRQ#%d: %d", isrc->xi_virq, error);
741 isrc->xi_port = bind_virq.port;
743 xen_intr_port_to_isrc[bind_virq.port] = isrc;
746 error = xen_intr_assign_cpu(&isrc->xi_intsrc,
749 panic("unable to bind xen VIRQ#%d to CPU#%d: %d",
750 isrc->xi_virq, cpu, error);
753 evtchn_unmask_port(bind_virq.port);
757 * Return this PIC to service after being suspended.
760 xen_intr_resume(struct pic *unused, bool suspend_cancelled)
762 shared_info_t *s = HYPERVISOR_shared_info;
763 struct xenisrc *isrc;
767 if (suspend_cancelled)
770 /* Reset the per-CPU masks */
772 struct xen_intr_pcpu_data *pcpu;
774 pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu);
775 memset(pcpu->evtchn_enabled, i == 0 ? ~0 : 0,
776 sizeof(pcpu->evtchn_enabled));
779 /* Mask all event channels. */
780 for (i = 0; i < nitems(s->evtchn_mask); i++)
781 atomic_store_rel_long(&s->evtchn_mask[i], ~0);
783 /* Remove port -> isrc mappings */
784 memset(xen_intr_port_to_isrc, 0, sizeof(xen_intr_port_to_isrc));
786 /* Free unused isrcs and rebind VIRQs and IPIs */
787 for (isrc_idx = 0; isrc_idx < xen_intr_auto_vector_count; isrc_idx++) {
790 vector = first_evtchn_irq + isrc_idx;
791 isrc = (struct xenisrc *)intr_lookup_source(vector);
793 isrc->xi_port = INVALID_EVTCHN;
794 switch (isrc->xi_type) {
795 case EVTCHN_TYPE_IPI:
796 xen_rebind_ipi(isrc);
798 case EVTCHN_TYPE_VIRQ:
799 xen_rebind_virq(isrc);
809 * Disable a Xen interrupt source.
811 * \param isrc The interrupt source to disable.
814 xen_intr_disable_intr(struct intsrc *base_isrc)
816 struct xenisrc *isrc = (struct xenisrc *)base_isrc;
818 evtchn_mask_port(isrc->xi_port);
822 * Determine the global interrupt vector number for
823 * a Xen interrupt source.
825 * \param isrc The interrupt source to query.
827 * \return The vector number corresponding to the given interrupt source.
830 xen_intr_vector(struct intsrc *base_isrc)
832 struct xenisrc *isrc = (struct xenisrc *)base_isrc;
834 return (isrc->xi_vector);
838 * Determine whether or not interrupt events are pending on the
839 * the given interrupt source.
841 * \param isrc The interrupt source to query.
843 * \returns 0 if no events are pending, otherwise non-zero.
846 xen_intr_source_pending(struct intsrc *isrc)
849 * EventChannels are edge triggered and never masked.
850 * There can be no pending events.
856 * Perform configuration of an interrupt source.
858 * \param isrc The interrupt source to configure.
859 * \param trig Edge or level.
860 * \param pol Active high or low.
862 * \returns 0 if no events are pending, otherwise non-zero.
865 xen_intr_config_intr(struct intsrc *isrc, enum intr_trigger trig,
866 enum intr_polarity pol)
868 /* Configuration is only possible via the evtchn apis. */
873 * Configure CPU affinity for interrupt source event delivery.
875 * \param isrc The interrupt source to configure.
876 * \param apic_id The apic id of the CPU for handling future events.
878 * \returns 0 if successful, otherwise an errno.
881 xen_intr_assign_cpu(struct intsrc *base_isrc, u_int apic_id)
884 struct evtchn_bind_vcpu bind_vcpu;
885 struct xenisrc *isrc;
886 u_int to_cpu, vcpu_id;
889 if (!xen_has_percpu_evtchn())
892 to_cpu = apic_cpuid(apic_id);
893 vcpu_id = pcpu_find(to_cpu)->pc_vcpu_id;
895 mtx_lock(&xen_intr_isrc_lock);
896 isrc = (struct xenisrc *)base_isrc;
897 if (!is_valid_evtchn(isrc->xi_port)) {
898 mtx_unlock(&xen_intr_isrc_lock);
903 * Mask the event channel while binding it to prevent interrupt
904 * delivery with an inconsistent state in isrc->xi_cpu.
906 masked = evtchn_test_and_set_mask(isrc->xi_port);
907 if ((isrc->xi_type == EVTCHN_TYPE_VIRQ) ||
908 (isrc->xi_type == EVTCHN_TYPE_IPI)) {
910 * Virtual IRQs are associated with a cpu by
911 * the Hypervisor at evtchn_bind_virq time, so
912 * all we need to do is update the per-CPU masks.
914 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port);
915 isrc->xi_cpu = to_cpu;
916 evtchn_cpu_unmask_port(isrc->xi_cpu, isrc->xi_port);
920 bind_vcpu.port = isrc->xi_port;
921 bind_vcpu.vcpu = vcpu_id;
923 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu);
924 if (isrc->xi_cpu != to_cpu) {
926 /* Commit to new binding by removing the old one. */
927 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port);
928 isrc->xi_cpu = to_cpu;
929 evtchn_cpu_unmask_port(isrc->xi_cpu, isrc->xi_port);
935 evtchn_unmask_port(isrc->xi_port);
936 mtx_unlock(&xen_intr_isrc_lock);
943 /*------------------- Virtual Interrupt Source PIC Functions -----------------*/
945 * Mask a level triggered interrupt source.
947 * \param isrc The interrupt source to mask (if necessary).
948 * \param eoi If non-zero, perform any necessary end-of-interrupt
952 xen_intr_disable_source(struct intsrc *base_isrc, int eoi)
954 struct xenisrc *isrc;
956 isrc = (struct xenisrc *)base_isrc;
959 * NB: checking if the event channel is already masked is
960 * needed because the event channel user-space device
961 * masks event channels on its filter as part of its
962 * normal operation, and those shouldn't be automatically
963 * unmasked by the generic interrupt code. The event channel
964 * device will unmask them when needed.
966 isrc->xi_masked = !!evtchn_test_and_set_mask(isrc->xi_port);
970 * Unmask a level triggered interrupt source.
972 * \param isrc The interrupt source to unmask (if necessary).
975 xen_intr_enable_source(struct intsrc *base_isrc)
977 struct xenisrc *isrc;
979 isrc = (struct xenisrc *)base_isrc;
981 if (isrc->xi_masked == 0)
982 evtchn_unmask_port(isrc->xi_port);
986 * Perform any necessary end-of-interrupt acknowledgements.
988 * \param isrc The interrupt source to EOI.
991 xen_intr_eoi_source(struct intsrc *base_isrc)
996 * Enable and unmask the interrupt source.
998 * \param isrc The interrupt source to enable.
1001 xen_intr_enable_intr(struct intsrc *base_isrc)
1003 struct xenisrc *isrc = (struct xenisrc *)base_isrc;
1005 evtchn_unmask_port(isrc->xi_port);
1008 /*--------------------------- Public Functions -------------------------------*/
1009 /*------- API comments for these methods can be found in xen/xenintr.h -------*/
1011 xen_intr_bind_local_port(device_t dev, evtchn_port_t local_port,
1012 driver_filter_t filter, driver_intr_t handler, void *arg,
1013 enum intr_type flags, xen_intr_handle_t *port_handlep)
1015 struct xenisrc *isrc;
1018 error = xen_intr_bind_isrc(&isrc, local_port, EVTCHN_TYPE_PORT,
1019 device_get_nameunit(dev), filter, handler, arg, flags,
1025 * The Event Channel API didn't open this port, so it is not
1026 * responsible for closing it automatically on unbind.
1033 xen_intr_alloc_and_bind_local_port(device_t dev, u_int remote_domain,
1034 driver_filter_t filter, driver_intr_t handler, void *arg,
1035 enum intr_type flags, xen_intr_handle_t *port_handlep)
1037 struct xenisrc *isrc;
1038 struct evtchn_alloc_unbound alloc_unbound;
1041 alloc_unbound.dom = DOMID_SELF;
1042 alloc_unbound.remote_dom = remote_domain;
1043 error = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound,
1047 * XXX Trap Hypercall error code Linuxisms in
1048 * the HYPERCALL layer.
1053 error = xen_intr_bind_isrc(&isrc, alloc_unbound.port, EVTCHN_TYPE_PORT,
1054 device_get_nameunit(dev), filter, handler, arg, flags,
1057 evtchn_close_t close = { .port = alloc_unbound.port };
1058 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
1059 panic("EVTCHNOP_close failed");
1068 xen_intr_bind_remote_port(device_t dev, u_int remote_domain,
1069 u_int remote_port, driver_filter_t filter, driver_intr_t handler,
1070 void *arg, enum intr_type flags, xen_intr_handle_t *port_handlep)
1072 struct xenisrc *isrc;
1073 struct evtchn_bind_interdomain bind_interdomain;
1076 bind_interdomain.remote_dom = remote_domain;
1077 bind_interdomain.remote_port = remote_port;
1078 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain,
1082 * XXX Trap Hypercall error code Linuxisms in
1083 * the HYPERCALL layer.
1088 error = xen_intr_bind_isrc(&isrc, bind_interdomain.local_port,
1089 EVTCHN_TYPE_PORT, device_get_nameunit(dev), filter, handler, arg,
1090 flags, port_handlep);
1092 evtchn_close_t close = { .port = bind_interdomain.local_port };
1093 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
1094 panic("EVTCHNOP_close failed");
1099 * The Event Channel API opened this port, so it is
1100 * responsible for closing it automatically on unbind.
1107 xen_intr_bind_virq(device_t dev, u_int virq, u_int cpu,
1108 driver_filter_t filter, driver_intr_t handler, void *arg,
1109 enum intr_type flags, xen_intr_handle_t *port_handlep)
1111 int vcpu_id = pcpu_find(cpu)->pc_vcpu_id;
1112 struct xenisrc *isrc;
1113 struct evtchn_bind_virq bind_virq = { .virq = virq, .vcpu = vcpu_id };
1117 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &bind_virq);
1120 * XXX Trap Hypercall error code Linuxisms in
1121 * the HYPERCALL layer.
1126 error = xen_intr_bind_isrc(&isrc, bind_virq.port, EVTCHN_TYPE_VIRQ,
1127 device_get_nameunit(dev), filter, handler, arg, flags,
1132 error = intr_event_bind(isrc->xi_intsrc.is_event, cpu);
1136 evtchn_close_t close = { .port = bind_virq.port };
1138 xen_intr_unbind(*port_handlep);
1139 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
1140 panic("EVTCHNOP_close failed");
1145 if (isrc->xi_cpu != cpu) {
1147 * Too early in the boot process for the generic interrupt
1148 * code to perform the binding. Update our event channel
1149 * masks manually so events can't fire on the wrong cpu
1150 * during AP startup.
1152 xen_intr_assign_cpu(&isrc->xi_intsrc, cpu_apic_ids[cpu]);
1157 * The Event Channel API opened this port, so it is
1158 * responsible for closing it automatically on unbind.
1161 isrc->xi_virq = virq;
1167 xen_intr_alloc_and_bind_ipi(u_int cpu, driver_filter_t filter,
1168 enum intr_type flags, xen_intr_handle_t *port_handlep)
1171 int vcpu_id = pcpu_find(cpu)->pc_vcpu_id;
1172 struct xenisrc *isrc;
1173 struct evtchn_bind_ipi bind_ipi = { .vcpu = vcpu_id };
1174 /* Same size as the one used by intr_handler->ih_name. */
1175 char name[MAXCOMLEN + 1];
1179 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi);
1182 * XXX Trap Hypercall error code Linuxisms in
1183 * the HYPERCALL layer.
1188 snprintf(name, sizeof(name), "cpu%u", cpu);
1190 error = xen_intr_bind_isrc(&isrc, bind_ipi.port, EVTCHN_TYPE_IPI,
1191 name, filter, NULL, NULL, flags, port_handlep);
1193 evtchn_close_t close = { .port = bind_ipi.port };
1195 xen_intr_unbind(*port_handlep);
1196 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
1197 panic("EVTCHNOP_close failed");
1201 if (isrc->xi_cpu != cpu) {
1203 * Too early in the boot process for the generic interrupt
1204 * code to perform the binding. Update our event channel
1205 * masks manually so events can't fire on the wrong cpu
1206 * during AP startup.
1208 xen_intr_assign_cpu(&isrc->xi_intsrc, cpu_apic_ids[cpu]);
1212 * The Event Channel API opened this port, so it is
1213 * responsible for closing it automatically on unbind.
1218 return (EOPNOTSUPP);
1223 xen_intr_describe(xen_intr_handle_t port_handle, const char *fmt, ...)
1225 char descr[MAXCOMLEN + 1];
1226 struct xenisrc *isrc;
1229 isrc = xen_intr_isrc_from_handle(port_handle);
1234 vsnprintf(descr, sizeof(descr), fmt, ap);
1236 return (intr_describe(isrc->xi_vector, isrc->xi_cookie, descr));
1240 xen_intr_unbind(xen_intr_handle_t *port_handlep)
1242 struct xenisrc *isrc;
1244 KASSERT(port_handlep != NULL,
1245 ("NULL xen_intr_handle_t passed to %s", __func__));
1247 isrc = xen_intr_isrc_from_handle(*port_handlep);
1248 *port_handlep = NULL;
1252 mtx_lock(&xen_intr_isrc_lock);
1253 if (refcount_release(&isrc->xi_refcount) == 0) {
1254 mtx_unlock(&xen_intr_isrc_lock);
1257 mtx_unlock(&xen_intr_isrc_lock);
1259 if (isrc->xi_cookie != NULL)
1260 intr_remove_handler(isrc->xi_cookie);
1261 xen_intr_release_isrc(isrc);
1265 xen_intr_signal(xen_intr_handle_t handle)
1267 struct xenisrc *isrc;
1269 isrc = xen_intr_isrc_from_handle(handle);
1271 KASSERT(isrc->xi_type == EVTCHN_TYPE_PORT ||
1272 isrc->xi_type == EVTCHN_TYPE_IPI,
1273 ("evtchn_signal on something other than a local port"));
1274 struct evtchn_send send = { .port = isrc->xi_port };
1275 (void)HYPERVISOR_event_channel_op(EVTCHNOP_send, &send);
1280 xen_intr_port(xen_intr_handle_t handle)
1282 struct xenisrc *isrc;
1284 isrc = xen_intr_isrc_from_handle(handle);
1288 return (isrc->xi_port);
1292 xen_intr_add_handler(const char *name, driver_filter_t filter,
1293 driver_intr_t handler, void *arg, enum intr_type flags,
1294 xen_intr_handle_t handle)
1296 struct xenisrc *isrc;
1299 isrc = xen_intr_isrc_from_handle(handle);
1300 if (isrc == NULL || isrc->xi_cookie != NULL)
1303 error = intr_add_handler(name, isrc->xi_vector,filter, handler, arg,
1304 flags|INTR_EXCL, &isrc->xi_cookie, 0);
1306 printf("%s: %s: add handler failed: %d\n", name, __func__,
1313 xen_intr_get_evtchn_from_port(evtchn_port_t port, xen_intr_handle_t *handlep)
1316 if (!is_valid_evtchn(port))
1319 if (handlep == NULL) {
1323 mtx_lock(&xen_intr_isrc_lock);
1324 if (xen_intr_port_to_isrc[port] == NULL) {
1325 mtx_unlock(&xen_intr_isrc_lock);
1328 refcount_acquire(&xen_intr_port_to_isrc[port]->xi_refcount);
1329 mtx_unlock(&xen_intr_isrc_lock);
1331 /* Assign the opaque handler */
1332 *handlep = xen_intr_handle_from_isrc(xen_intr_port_to_isrc[port]);
1339 xen_intr_print_type(enum evtchn_type type)
1341 static const char *evtchn_type_to_string[EVTCHN_TYPE_COUNT] = {
1342 [EVTCHN_TYPE_UNBOUND] = "UNBOUND",
1343 [EVTCHN_TYPE_VIRQ] = "VIRQ",
1344 [EVTCHN_TYPE_IPI] = "IPI",
1345 [EVTCHN_TYPE_PORT] = "PORT",
1348 if (type >= EVTCHN_TYPE_COUNT)
1351 return (evtchn_type_to_string[type]);
1355 xen_intr_dump_port(struct xenisrc *isrc)
1357 struct xen_intr_pcpu_data *pcpu;
1358 shared_info_t *s = HYPERVISOR_shared_info;
1361 db_printf("Port %d Type: %s\n",
1362 isrc->xi_port, xen_intr_print_type(isrc->xi_type));
1363 if (isrc->xi_type == EVTCHN_TYPE_VIRQ)
1364 db_printf("\tVirq: %d\n", isrc->xi_virq);
1366 db_printf("\tMasked: %d Pending: %d\n",
1367 !!xen_test_bit(isrc->xi_port, &s->evtchn_mask[0]),
1368 !!xen_test_bit(isrc->xi_port, &s->evtchn_pending[0]));
1370 db_printf("\tPer-CPU Masks: ");
1372 pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu);
1373 db_printf("cpu#%d: %d ", i,
1374 !!xen_test_bit(isrc->xi_port, pcpu->evtchn_enabled));
1379 DB_SHOW_COMMAND(xen_evtchn, db_show_xen_evtchn)
1383 if (!xen_domain()) {
1384 db_printf("Only available on Xen guests\n");
1388 for (i = 0; i < NR_EVENT_CHANNELS; i++) {
1389 struct xenisrc *isrc;
1391 isrc = xen_intr_port_to_isrc[i];
1395 xen_intr_dump_port(isrc);