2 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions, and the following
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 /* Driver for the VirtIO PCI interface. */
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
32 #include <sys/param.h>
33 #include <sys/systm.h>
35 #include <sys/kernel.h>
36 #include <sys/module.h>
37 #include <sys/malloc.h>
39 #include <machine/bus.h>
40 #include <machine/resource.h>
44 #include <dev/pci/pcivar.h>
45 #include <dev/pci/pcireg.h>
47 #include <dev/virtio/virtio.h>
48 #include <dev/virtio/virtqueue.h>
49 #include <dev/virtio/pci/virtio_pci.h>
51 #include "virtio_bus_if.h"
52 #include "virtio_if.h"
54 struct vtpci_interrupt {
55 struct resource *vti_irq;
60 struct vtpci_virtqueue {
61 struct virtqueue *vtv_vq;
67 struct resource *vtpci_res;
68 struct resource *vtpci_msix_res;
69 uint64_t vtpci_features;
71 #define VTPCI_FLAG_NO_MSI 0x0001
72 #define VTPCI_FLAG_NO_MSIX 0x0002
73 #define VTPCI_FLAG_LEGACY 0x1000
74 #define VTPCI_FLAG_MSI 0x2000
75 #define VTPCI_FLAG_MSIX 0x4000
76 #define VTPCI_FLAG_SHARED_MSIX 0x8000
77 #define VTPCI_FLAG_ITYPE_MASK 0xF000
79 /* This "bus" will only ever have one child. */
80 device_t vtpci_child_dev;
81 struct virtio_feature_desc *vtpci_child_feat_desc;
84 struct vtpci_virtqueue *vtpci_vqs;
87 * Ideally, each virtqueue that the driver provides a callback for will
88 * receive its own MSIX vector. If there are not sufficient vectors
89 * available, then attempt to have all the VQs share one vector. For
90 * MSIX, the configuration changed notifications must be on their own
93 * If MSIX is not available, we will attempt to have the whole device
94 * share one MSI vector, and then, finally, one legacy interrupt.
96 struct vtpci_interrupt vtpci_device_interrupt;
97 struct vtpci_interrupt *vtpci_msix_vq_interrupts;
98 int vtpci_nmsix_resources;
101 static int vtpci_probe(device_t);
102 static int vtpci_attach(device_t);
103 static int vtpci_detach(device_t);
104 static int vtpci_suspend(device_t);
105 static int vtpci_resume(device_t);
106 static int vtpci_shutdown(device_t);
107 static void vtpci_driver_added(device_t, driver_t *);
108 static void vtpci_child_detached(device_t, device_t);
109 static int vtpci_read_ivar(device_t, device_t, int, uintptr_t *);
110 static int vtpci_write_ivar(device_t, device_t, int, uintptr_t);
112 static uint64_t vtpci_negotiate_features(device_t, uint64_t);
113 static int vtpci_with_feature(device_t, uint64_t);
114 static int vtpci_alloc_virtqueues(device_t, int, int,
115 struct vq_alloc_info *);
116 static int vtpci_setup_intr(device_t, enum intr_type);
117 static void vtpci_stop(device_t);
118 static int vtpci_reinit(device_t, uint64_t);
119 static void vtpci_reinit_complete(device_t);
120 static void vtpci_notify_virtqueue(device_t, uint16_t);
121 static uint8_t vtpci_get_status(device_t);
122 static void vtpci_set_status(device_t, uint8_t);
123 static void vtpci_read_dev_config(device_t, bus_size_t, void *, int);
124 static void vtpci_write_dev_config(device_t, bus_size_t, void *, int);
126 static void vtpci_describe_features(struct vtpci_softc *, const char *,
128 static void vtpci_probe_and_attach_child(struct vtpci_softc *);
130 static int vtpci_alloc_msix(struct vtpci_softc *, int);
131 static int vtpci_alloc_msi(struct vtpci_softc *);
132 static int vtpci_alloc_intr_msix_pervq(struct vtpci_softc *);
133 static int vtpci_alloc_intr_msix_shared(struct vtpci_softc *);
134 static int vtpci_alloc_intr_msi(struct vtpci_softc *);
135 static int vtpci_alloc_intr_legacy(struct vtpci_softc *);
136 static int vtpci_alloc_interrupt(struct vtpci_softc *, int, int,
137 struct vtpci_interrupt *);
138 static int vtpci_alloc_intr_resources(struct vtpci_softc *);
140 static int vtpci_setup_legacy_interrupt(struct vtpci_softc *,
142 static int vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *,
144 static int vtpci_setup_msix_interrupts(struct vtpci_softc *,
146 static int vtpci_setup_interrupts(struct vtpci_softc *, enum intr_type);
148 static int vtpci_register_msix_vector(struct vtpci_softc *, int,
149 struct vtpci_interrupt *);
150 static int vtpci_set_host_msix_vectors(struct vtpci_softc *);
151 static int vtpci_reinit_virtqueue(struct vtpci_softc *, int);
153 static void vtpci_free_interrupt(struct vtpci_softc *,
154 struct vtpci_interrupt *);
155 static void vtpci_free_interrupts(struct vtpci_softc *);
156 static void vtpci_free_virtqueues(struct vtpci_softc *);
157 static void vtpci_release_child_resources(struct vtpci_softc *);
158 static void vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *);
159 static void vtpci_reset(struct vtpci_softc *);
161 static void vtpci_select_virtqueue(struct vtpci_softc *, int);
163 static void vtpci_legacy_intr(void *);
164 static int vtpci_vq_shared_intr_filter(void *);
165 static void vtpci_vq_shared_intr(void *);
166 static int vtpci_vq_intr_filter(void *);
167 static void vtpci_vq_intr(void *);
168 static void vtpci_config_intr(void *);
170 #define vtpci_setup_msi_interrupt vtpci_setup_legacy_interrupt
173 * I/O port read/write wrappers.
175 #define vtpci_read_config_1(sc, o) bus_read_1((sc)->vtpci_res, (o))
176 #define vtpci_read_config_2(sc, o) bus_read_2((sc)->vtpci_res, (o))
177 #define vtpci_read_config_4(sc, o) bus_read_4((sc)->vtpci_res, (o))
178 #define vtpci_write_config_1(sc, o, v) bus_write_1((sc)->vtpci_res, (o), (v))
179 #define vtpci_write_config_2(sc, o, v) bus_write_2((sc)->vtpci_res, (o), (v))
180 #define vtpci_write_config_4(sc, o, v) bus_write_4((sc)->vtpci_res, (o), (v))
183 static int vtpci_disable_msix = 0;
184 TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix);
186 static device_method_t vtpci_methods[] = {
187 /* Device interface. */
188 DEVMETHOD(device_probe, vtpci_probe),
189 DEVMETHOD(device_attach, vtpci_attach),
190 DEVMETHOD(device_detach, vtpci_detach),
191 DEVMETHOD(device_suspend, vtpci_suspend),
192 DEVMETHOD(device_resume, vtpci_resume),
193 DEVMETHOD(device_shutdown, vtpci_shutdown),
196 DEVMETHOD(bus_driver_added, vtpci_driver_added),
197 DEVMETHOD(bus_child_detached, vtpci_child_detached),
198 DEVMETHOD(bus_read_ivar, vtpci_read_ivar),
199 DEVMETHOD(bus_write_ivar, vtpci_write_ivar),
201 /* VirtIO bus interface. */
202 DEVMETHOD(virtio_bus_negotiate_features, vtpci_negotiate_features),
203 DEVMETHOD(virtio_bus_with_feature, vtpci_with_feature),
204 DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_alloc_virtqueues),
205 DEVMETHOD(virtio_bus_setup_intr, vtpci_setup_intr),
206 DEVMETHOD(virtio_bus_stop, vtpci_stop),
207 DEVMETHOD(virtio_bus_reinit, vtpci_reinit),
208 DEVMETHOD(virtio_bus_reinit_complete, vtpci_reinit_complete),
209 DEVMETHOD(virtio_bus_notify_vq, vtpci_notify_virtqueue),
210 DEVMETHOD(virtio_bus_read_device_config, vtpci_read_dev_config),
211 DEVMETHOD(virtio_bus_write_device_config, vtpci_write_dev_config),
216 static driver_t vtpci_driver = {
219 sizeof(struct vtpci_softc)
222 devclass_t vtpci_devclass;
224 DRIVER_MODULE(virtio_pci, pci, vtpci_driver, vtpci_devclass, 0, 0);
225 MODULE_VERSION(virtio_pci, 1);
226 MODULE_DEPEND(virtio_pci, pci, 1, 1, 1);
227 MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1);
230 vtpci_probe(device_t dev)
235 if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID)
238 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN ||
239 pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MAX)
242 if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION)
245 name = virtio_device_name(pci_get_subdevice(dev));
249 snprintf(desc, sizeof(desc), "VirtIO PCI %s adapter", name);
250 device_set_desc_copy(dev, desc);
252 return (BUS_PROBE_DEFAULT);
256 vtpci_attach(device_t dev)
258 struct vtpci_softc *sc;
262 sc = device_get_softc(dev);
265 pci_enable_busmaster(dev);
268 sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid,
270 if (sc->vtpci_res == NULL) {
271 device_printf(dev, "cannot map I/O space\n");
275 if (pci_find_cap(dev, PCIY_MSI, NULL) != 0)
276 sc->vtpci_flags |= VTPCI_FLAG_NO_MSI;
278 if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) {
280 sc->vtpci_msix_res = bus_alloc_resource_any(dev,
281 SYS_RES_MEMORY, &rid, RF_ACTIVE);
284 if (sc->vtpci_msix_res == NULL)
285 sc->vtpci_flags |= VTPCI_FLAG_NO_MSIX;
289 /* Tell the host we've noticed this device. */
290 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
292 if ((child = device_add_child(dev, NULL, -1)) == NULL) {
293 device_printf(dev, "cannot create child device\n");
294 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
299 sc->vtpci_child_dev = child;
300 vtpci_probe_and_attach_child(sc);
306 vtpci_detach(device_t dev)
308 struct vtpci_softc *sc;
312 sc = device_get_softc(dev);
314 if ((child = sc->vtpci_child_dev) != NULL) {
315 error = device_delete_child(dev, child);
318 sc->vtpci_child_dev = NULL;
323 if (sc->vtpci_msix_res != NULL) {
324 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1),
326 sc->vtpci_msix_res = NULL;
329 if (sc->vtpci_res != NULL) {
330 bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0),
332 sc->vtpci_res = NULL;
339 vtpci_suspend(device_t dev)
342 return (bus_generic_suspend(dev));
346 vtpci_resume(device_t dev)
349 return (bus_generic_resume(dev));
353 vtpci_shutdown(device_t dev)
356 (void) bus_generic_shutdown(dev);
357 /* Forcibly stop the host device. */
364 vtpci_driver_added(device_t dev, driver_t *driver)
366 struct vtpci_softc *sc;
368 sc = device_get_softc(dev);
370 vtpci_probe_and_attach_child(sc);
374 vtpci_child_detached(device_t dev, device_t child)
376 struct vtpci_softc *sc;
378 sc = device_get_softc(dev);
381 vtpci_release_child_resources(sc);
385 vtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
387 struct vtpci_softc *sc;
389 sc = device_get_softc(dev);
391 if (sc->vtpci_child_dev != child)
395 case VIRTIO_IVAR_DEVTYPE:
396 case VIRTIO_IVAR_SUBDEVICE:
397 *result = pci_get_subdevice(dev);
399 case VIRTIO_IVAR_VENDOR:
400 *result = pci_get_vendor(dev);
402 case VIRTIO_IVAR_DEVICE:
403 *result = pci_get_device(dev);
405 case VIRTIO_IVAR_SUBVENDOR:
406 *result = pci_get_subdevice(dev);
416 vtpci_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
418 struct vtpci_softc *sc;
420 sc = device_get_softc(dev);
422 if (sc->vtpci_child_dev != child)
426 case VIRTIO_IVAR_FEATURE_DESC:
427 sc->vtpci_child_feat_desc = (void *) value;
437 vtpci_negotiate_features(device_t dev, uint64_t child_features)
439 struct vtpci_softc *sc;
440 uint64_t host_features, features;
442 sc = device_get_softc(dev);
444 host_features = vtpci_read_config_4(sc, VIRTIO_PCI_HOST_FEATURES);
445 vtpci_describe_features(sc, "host", host_features);
448 * Limit negotiated features to what the driver, virtqueue, and
451 features = host_features & child_features;
452 features = virtqueue_filter_features(features);
453 sc->vtpci_features = features;
455 vtpci_describe_features(sc, "negotiated", features);
456 vtpci_write_config_4(sc, VIRTIO_PCI_GUEST_FEATURES, features);
462 vtpci_with_feature(device_t dev, uint64_t feature)
464 struct vtpci_softc *sc;
466 sc = device_get_softc(dev);
468 return ((sc->vtpci_features & feature) != 0);
472 vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs,
473 struct vq_alloc_info *vq_info)
475 struct vtpci_softc *sc;
476 struct virtqueue *vq;
477 struct vtpci_virtqueue *vqx;
478 struct vq_alloc_info *info;
482 sc = device_get_softc(dev);
484 if (sc->vtpci_nvqs != 0)
489 sc->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue),
490 M_DEVBUF, M_NOWAIT | M_ZERO);
491 if (sc->vtpci_vqs == NULL)
494 for (idx = 0; idx < nvqs; idx++) {
495 vqx = &sc->vtpci_vqs[idx];
496 info = &vq_info[idx];
498 vtpci_select_virtqueue(sc, idx);
499 size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM);
501 error = virtqueue_alloc(dev, idx, size, VIRTIO_PCI_VRING_ALIGN,
502 0xFFFFFFFFUL, info, &vq);
505 "cannot allocate virtqueue %d: %d\n", idx, error);
509 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN,
510 virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
512 vqx->vtv_vq = *info->vqai_vq = vq;
513 vqx->vtv_no_intr = info->vqai_intr == NULL;
519 vtpci_free_virtqueues(sc);
525 vtpci_setup_intr(device_t dev, enum intr_type type)
527 struct vtpci_softc *sc;
530 sc = device_get_softc(dev);
532 for (attempt = 0; attempt < 5; attempt++) {
534 * Start with the most desirable interrupt configuration and
535 * fallback towards less desirable ones.
539 error = vtpci_alloc_intr_msix_pervq(sc);
542 error = vtpci_alloc_intr_msix_shared(sc);
545 error = vtpci_alloc_intr_msi(sc);
548 error = vtpci_alloc_intr_legacy(sc);
552 "exhausted all interrupt allocation attempts\n");
556 if (error == 0 && vtpci_setup_interrupts(sc, type) == 0)
559 vtpci_cleanup_setup_intr_attempt(sc);
563 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
564 device_printf(dev, "using legacy interrupt\n");
565 else if (sc->vtpci_flags & VTPCI_FLAG_MSI)
566 device_printf(dev, "using MSI interrupt\n");
567 else if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX)
568 device_printf(dev, "using shared MSIX interrupts\n");
570 device_printf(dev, "using per VQ MSIX interrupts\n");
577 vtpci_stop(device_t dev)
580 vtpci_reset(device_get_softc(dev));
584 vtpci_reinit(device_t dev, uint64_t features)
586 struct vtpci_softc *sc;
589 sc = device_get_softc(dev);
592 * Redrive the device initialization. This is a bit of an abuse of
593 * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to
596 * We do not allow the host device to change from what was originally
597 * negotiated beyond what the guest driver changed. MSIX state should
598 * not change, number of virtqueues and their size remain the same, etc.
599 * This will need to be rethought when we want to support migration.
602 if (vtpci_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
606 * Quickly drive the status through ACK and DRIVER. The device
607 * does not become usable again until vtpci_reinit_complete().
609 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
610 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
612 vtpci_negotiate_features(dev, features);
614 for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
615 error = vtpci_reinit_virtqueue(sc, idx);
620 if (sc->vtpci_flags & VTPCI_FLAG_MSIX) {
621 error = vtpci_set_host_msix_vectors(sc);
630 vtpci_reinit_complete(device_t dev)
633 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
637 vtpci_notify_virtqueue(device_t dev, uint16_t queue)
639 struct vtpci_softc *sc;
641 sc = device_get_softc(dev);
643 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_NOTIFY, queue);
647 vtpci_get_status(device_t dev)
649 struct vtpci_softc *sc;
651 sc = device_get_softc(dev);
653 return (vtpci_read_config_1(sc, VIRTIO_PCI_STATUS));
657 vtpci_set_status(device_t dev, uint8_t status)
659 struct vtpci_softc *sc;
661 sc = device_get_softc(dev);
663 if (status != VIRTIO_CONFIG_STATUS_RESET)
664 status |= vtpci_get_status(dev);
666 vtpci_write_config_1(sc, VIRTIO_PCI_STATUS, status);
670 vtpci_read_dev_config(device_t dev, bus_size_t offset,
671 void *dst, int length)
673 struct vtpci_softc *sc;
678 sc = device_get_softc(dev);
679 off = VIRTIO_PCI_CONFIG(sc) + offset;
681 for (d = dst; length > 0; d += size, off += size, length -= size) {
684 *(uint32_t *)d = vtpci_read_config_4(sc, off);
685 } else if (length >= 2) {
687 *(uint16_t *)d = vtpci_read_config_2(sc, off);
690 *d = vtpci_read_config_1(sc, off);
696 vtpci_write_dev_config(device_t dev, bus_size_t offset,
697 void *src, int length)
699 struct vtpci_softc *sc;
704 sc = device_get_softc(dev);
705 off = VIRTIO_PCI_CONFIG(sc) + offset;
707 for (s = src; length > 0; s += size, off += size, length -= size) {
710 vtpci_write_config_4(sc, off, *(uint32_t *)s);
711 } else if (length >= 2) {
713 vtpci_write_config_2(sc, off, *(uint16_t *)s);
716 vtpci_write_config_1(sc, off, *s);
722 vtpci_describe_features(struct vtpci_softc *sc, const char *msg,
728 child = sc->vtpci_child_dev;
730 if (device_is_attached(child) && bootverbose == 0)
733 virtio_describe(dev, msg, features, sc->vtpci_child_feat_desc);
737 vtpci_probe_and_attach_child(struct vtpci_softc *sc)
742 child = sc->vtpci_child_dev;
747 if (device_get_state(child) != DS_NOTPRESENT)
750 if (device_probe(child) != 0)
753 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
754 if (device_attach(child) != 0) {
755 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
757 vtpci_release_child_resources(sc);
758 /* Reset status for future attempt. */
759 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
761 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
762 VIRTIO_ATTACH_COMPLETED(child);
767 vtpci_alloc_msix(struct vtpci_softc *sc, int nvectors)
770 int nmsix, cnt, required;
774 /* Allocate an additional vector for the config changes. */
775 required = nvectors + 1;
777 nmsix = pci_msix_count(dev);
778 if (nmsix < required)
782 if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) {
783 sc->vtpci_nmsix_resources = required;
787 pci_release_msi(dev);
793 vtpci_alloc_msi(struct vtpci_softc *sc)
796 int nmsi, cnt, required;
801 nmsi = pci_msi_count(dev);
806 if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required)
809 pci_release_msi(dev);
815 vtpci_alloc_intr_msix_pervq(struct vtpci_softc *sc)
817 int i, nvectors, error;
819 if (vtpci_disable_msix != 0 ||
820 sc->vtpci_flags & VTPCI_FLAG_NO_MSIX)
823 for (nvectors = 0, i = 0; i < sc->vtpci_nvqs; i++) {
824 if (sc->vtpci_vqs[i].vtv_no_intr == 0)
828 error = vtpci_alloc_msix(sc, nvectors);
832 sc->vtpci_flags |= VTPCI_FLAG_MSIX;
838 vtpci_alloc_intr_msix_shared(struct vtpci_softc *sc)
842 if (vtpci_disable_msix != 0 ||
843 sc->vtpci_flags & VTPCI_FLAG_NO_MSIX)
846 error = vtpci_alloc_msix(sc, 1);
850 sc->vtpci_flags |= VTPCI_FLAG_MSIX | VTPCI_FLAG_SHARED_MSIX;
856 vtpci_alloc_intr_msi(struct vtpci_softc *sc)
860 /* Only BHyVe supports MSI. */
861 if (sc->vtpci_flags & VTPCI_FLAG_NO_MSI)
864 error = vtpci_alloc_msi(sc);
868 sc->vtpci_flags |= VTPCI_FLAG_MSI;
874 vtpci_alloc_intr_legacy(struct vtpci_softc *sc)
877 sc->vtpci_flags |= VTPCI_FLAG_LEGACY;
883 vtpci_alloc_interrupt(struct vtpci_softc *sc, int rid, int flags,
884 struct vtpci_interrupt *intr)
886 struct resource *irq;
888 irq = bus_alloc_resource_any(sc->vtpci_dev, SYS_RES_IRQ, &rid, flags);
899 vtpci_alloc_intr_resources(struct vtpci_softc *sc)
901 struct vtpci_interrupt *intr;
902 int i, rid, flags, nvq_intrs, error;
907 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
908 flags |= RF_SHAREABLE;
913 * For legacy and MSI interrupts, this single resource handles all
914 * interrupts. For MSIX, this resource is used for the configuration
917 intr = &sc->vtpci_device_interrupt;
918 error = vtpci_alloc_interrupt(sc, rid, flags, intr);
919 if (error || sc->vtpci_flags & (VTPCI_FLAG_LEGACY | VTPCI_FLAG_MSI))
922 /* Subtract one for the configuration changed interrupt. */
923 nvq_intrs = sc->vtpci_nmsix_resources - 1;
925 intr = sc->vtpci_msix_vq_interrupts = malloc(nvq_intrs *
926 sizeof(struct vtpci_interrupt), M_DEVBUF, M_NOWAIT | M_ZERO);
927 if (sc->vtpci_msix_vq_interrupts == NULL)
930 for (i = 0, rid++; i < nvq_intrs; i++, rid++, intr++) {
931 error = vtpci_alloc_interrupt(sc, rid, flags, intr);
940 vtpci_setup_legacy_interrupt(struct vtpci_softc *sc, enum intr_type type)
942 struct vtpci_interrupt *intr;
945 intr = &sc->vtpci_device_interrupt;
946 error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, NULL,
947 vtpci_legacy_intr, sc, &intr->vti_handler);
953 vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *sc, enum intr_type type)
955 struct vtpci_virtqueue *vqx;
956 struct vtpci_interrupt *intr;
959 intr = sc->vtpci_msix_vq_interrupts;
961 for (i = 0; i < sc->vtpci_nvqs; i++) {
962 vqx = &sc->vtpci_vqs[i];
964 if (vqx->vtv_no_intr)
967 error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type,
968 vtpci_vq_intr_filter, vtpci_vq_intr, vqx->vtv_vq,
980 vtpci_setup_msix_interrupts(struct vtpci_softc *sc, enum intr_type type)
983 struct vtpci_interrupt *intr;
987 intr = &sc->vtpci_device_interrupt;
989 error = bus_setup_intr(dev, intr->vti_irq, type, NULL,
990 vtpci_config_intr, sc, &intr->vti_handler);
994 if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) {
995 intr = sc->vtpci_msix_vq_interrupts;
996 error = bus_setup_intr(dev, intr->vti_irq, type,
997 vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, sc,
1000 error = vtpci_setup_pervq_msix_interrupts(sc, type);
1002 return (error ? error : vtpci_set_host_msix_vectors(sc));
1006 vtpci_setup_interrupts(struct vtpci_softc *sc, enum intr_type type)
1010 type |= INTR_MPSAFE;
1011 KASSERT(sc->vtpci_flags & VTPCI_FLAG_ITYPE_MASK,
1012 ("%s: no interrupt type selected %#x", __func__, sc->vtpci_flags));
1014 error = vtpci_alloc_intr_resources(sc);
1018 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
1019 error = vtpci_setup_legacy_interrupt(sc, type);
1020 else if (sc->vtpci_flags & VTPCI_FLAG_MSI)
1021 error = vtpci_setup_msi_interrupt(sc, type);
1023 error = vtpci_setup_msix_interrupts(sc, type);
1029 vtpci_register_msix_vector(struct vtpci_softc *sc, int offset,
1030 struct vtpci_interrupt *intr)
1035 dev = sc->vtpci_dev;
1038 /* Map from guest rid to host vector. */
1039 vector = intr->vti_rid - 1;
1041 vector = VIRTIO_MSI_NO_VECTOR;
1043 vtpci_write_config_2(sc, offset, vector);
1045 /* Read vector to determine if the host had sufficient resources. */
1046 if (vtpci_read_config_2(sc, offset) != vector) {
1048 "insufficient host resources for MSIX interrupts\n");
1056 vtpci_set_host_msix_vectors(struct vtpci_softc *sc)
1058 struct vtpci_interrupt *intr, *tintr;
1059 int idx, offset, error;
1061 intr = &sc->vtpci_device_interrupt;
1062 offset = VIRTIO_MSI_CONFIG_VECTOR;
1064 error = vtpci_register_msix_vector(sc, offset, intr);
1068 intr = sc->vtpci_msix_vq_interrupts;
1069 offset = VIRTIO_MSI_QUEUE_VECTOR;
1071 for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
1072 vtpci_select_virtqueue(sc, idx);
1074 if (sc->vtpci_vqs[idx].vtv_no_intr)
1079 error = vtpci_register_msix_vector(sc, offset, tintr);
1084 * For shared MSIX, all the virtqueues share the first
1087 if ((sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0)
1095 vtpci_reinit_virtqueue(struct vtpci_softc *sc, int idx)
1097 struct vtpci_virtqueue *vqx;
1098 struct virtqueue *vq;
1102 vqx = &sc->vtpci_vqs[idx];
1105 KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));
1107 vtpci_select_virtqueue(sc, idx);
1108 size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM);
1110 error = virtqueue_reinit(vq, size);
1114 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN,
1115 virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
1121 vtpci_free_interrupt(struct vtpci_softc *sc, struct vtpci_interrupt *intr)
1125 dev = sc->vtpci_dev;
1127 if (intr->vti_handler != NULL) {
1128 bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler);
1129 intr->vti_handler = NULL;
1132 if (intr->vti_irq != NULL) {
1133 bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid,
1135 intr->vti_irq = NULL;
1141 vtpci_free_interrupts(struct vtpci_softc *sc)
1143 struct vtpci_interrupt *intr;
1146 vtpci_free_interrupt(sc, &sc->vtpci_device_interrupt);
1148 if (sc->vtpci_nmsix_resources != 0) {
1149 nvq_intrs = sc->vtpci_nmsix_resources - 1;
1150 sc->vtpci_nmsix_resources = 0;
1152 intr = sc->vtpci_msix_vq_interrupts;
1154 for (i = 0; i < nvq_intrs; i++, intr++)
1155 vtpci_free_interrupt(sc, intr);
1157 free(sc->vtpci_msix_vq_interrupts, M_DEVBUF);
1158 sc->vtpci_msix_vq_interrupts = NULL;
1162 if (sc->vtpci_flags & (VTPCI_FLAG_MSI | VTPCI_FLAG_MSIX))
1163 pci_release_msi(sc->vtpci_dev);
1165 sc->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK;
1169 vtpci_free_virtqueues(struct vtpci_softc *sc)
1171 struct vtpci_virtqueue *vqx;
1174 for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
1175 vqx = &sc->vtpci_vqs[idx];
1177 vtpci_select_virtqueue(sc, idx);
1178 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 0);
1180 virtqueue_free(vqx->vtv_vq);
1184 free(sc->vtpci_vqs, M_DEVBUF);
1185 sc->vtpci_vqs = NULL;
1190 vtpci_release_child_resources(struct vtpci_softc *sc)
1193 vtpci_free_interrupts(sc);
1194 vtpci_free_virtqueues(sc);
1198 vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *sc)
1202 if (sc->vtpci_flags & VTPCI_FLAG_MSIX) {
1203 vtpci_write_config_2(sc, VIRTIO_MSI_CONFIG_VECTOR,
1204 VIRTIO_MSI_NO_VECTOR);
1206 for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
1207 vtpci_select_virtqueue(sc, idx);
1208 vtpci_write_config_2(sc, VIRTIO_MSI_QUEUE_VECTOR,
1209 VIRTIO_MSI_NO_VECTOR);
1213 vtpci_free_interrupts(sc);
1217 vtpci_reset(struct vtpci_softc *sc)
1221 * Setting the status to RESET sets the host device to
1222 * the original, uninitialized state.
1224 vtpci_set_status(sc->vtpci_dev, VIRTIO_CONFIG_STATUS_RESET);
1228 vtpci_select_virtqueue(struct vtpci_softc *sc, int idx)
1231 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, idx);
1235 vtpci_legacy_intr(void *xsc)
1237 struct vtpci_softc *sc;
1238 struct vtpci_virtqueue *vqx;
1243 vqx = &sc->vtpci_vqs[0];
1245 /* Reading the ISR also clears it. */
1246 isr = vtpci_read_config_1(sc, VIRTIO_PCI_ISR);
1248 if (isr & VIRTIO_PCI_ISR_CONFIG)
1249 vtpci_config_intr(sc);
1251 if (isr & VIRTIO_PCI_ISR_INTR) {
1252 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
1253 if (vqx->vtv_no_intr == 0)
1254 virtqueue_intr(vqx->vtv_vq);
1260 vtpci_vq_shared_intr_filter(void *xsc)
1262 struct vtpci_softc *sc;
1263 struct vtpci_virtqueue *vqx;
1268 vqx = &sc->vtpci_vqs[0];
1270 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
1271 if (vqx->vtv_no_intr == 0)
1272 rc |= virtqueue_intr_filter(vqx->vtv_vq);
1275 return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY);
1279 vtpci_vq_shared_intr(void *xsc)
1281 struct vtpci_softc *sc;
1282 struct vtpci_virtqueue *vqx;
1286 vqx = &sc->vtpci_vqs[0];
1288 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
1289 if (vqx->vtv_no_intr == 0)
1290 virtqueue_intr(vqx->vtv_vq);
1295 vtpci_vq_intr_filter(void *xvq)
1297 struct virtqueue *vq;
1301 rc = virtqueue_intr_filter(vq);
1303 return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY);
1307 vtpci_vq_intr(void *xvq)
1309 struct virtqueue *vq;
1316 vtpci_config_intr(void *xsc)
1318 struct vtpci_softc *sc;
1322 child = sc->vtpci_child_dev;
1325 VIRTIO_CONFIG_CHANGE(child);