2 * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com>
3 * Copyright (c) 2014 The FreeBSD Foundation
6 * This software was developed by SRI International and the University of
7 * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
8 * ("CTSRD"), as part of the DARPA CRASH research programme.
10 * Portions of this software were developed by Andrew Turner
11 * under sponsorship from the FreeBSD Foundation.
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * VirtIO MMIO interface.
37 * This driver is heavily based on VirtIO PCI interface driver.
43 * compatible = "virtio,mmio";
44 * reg = <0x1000 0x100>;
46 * interrupt-parent = <&GIC>;
50 #include <sys/cdefs.h>
51 __FBSDID("$FreeBSD$");
53 #include <sys/param.h>
54 #include <sys/systm.h>
56 #include <sys/kernel.h>
57 #include <sys/module.h>
58 #include <sys/malloc.h>
61 #include <machine/bus.h>
62 #include <machine/resource.h>
64 #include <dev/fdt/fdt_common.h>
65 #include <dev/ofw/openfirm.h>
66 #include <dev/ofw/ofw_bus.h>
67 #include <dev/ofw/ofw_bus_subr.h>
69 #include <dev/virtio/virtio.h>
70 #include <dev/virtio/virtqueue.h>
71 #include <dev/virtio/mmio/virtio_mmio.h>
73 #include "virtio_mmio_if.h"
74 #include "virtio_bus_if.h"
75 #include "virtio_if.h"
79 struct vtmmio_virtqueue {
80 struct virtqueue *vtv_vq;
87 struct resource *res[2];
89 uint64_t vtmmio_features;
90 uint32_t vtmmio_flags;
92 /* This "bus" will only ever have one child. */
93 device_t vtmmio_child_dev;
94 struct virtio_feature_desc *vtmmio_child_feat_desc;
97 struct vtmmio_virtqueue *vtmmio_vqs;
101 static int vtmmio_probe(device_t);
102 static int vtmmio_attach(device_t);
103 static int vtmmio_detach(device_t);
104 static int vtmmio_suspend(device_t);
105 static int vtmmio_resume(device_t);
106 static int vtmmio_shutdown(device_t);
107 static void vtmmio_driver_added(device_t, driver_t *);
108 static void vtmmio_child_detached(device_t, device_t);
109 static int vtmmio_read_ivar(device_t, device_t, int, uintptr_t *);
110 static int vtmmio_write_ivar(device_t, device_t, int, uintptr_t);
111 static uint64_t vtmmio_negotiate_features(device_t, uint64_t);
112 static int vtmmio_with_feature(device_t, uint64_t);
113 static int vtmmio_alloc_virtqueues(device_t, int, int,
114 struct vq_alloc_info *);
115 static int vtmmio_setup_intr(device_t, enum intr_type);
116 static void vtmmio_stop(device_t);
117 static void vtmmio_poll(device_t);
118 static int vtmmio_reinit(device_t, uint64_t);
119 static void vtmmio_reinit_complete(device_t);
120 static void vtmmio_notify_virtqueue(device_t, uint16_t);
121 static uint8_t vtmmio_get_status(device_t);
122 static void vtmmio_set_status(device_t, uint8_t);
123 static void vtmmio_read_dev_config(device_t, bus_size_t, void *, int);
124 static void vtmmio_write_dev_config(device_t, bus_size_t, void *, int);
125 static void vtmmio_describe_features(struct vtmmio_softc *, const char *,
127 static void vtmmio_probe_and_attach_child(struct vtmmio_softc *);
128 static int vtmmio_reinit_virtqueue(struct vtmmio_softc *, int);
129 static void vtmmio_free_interrupts(struct vtmmio_softc *);
130 static void vtmmio_free_virtqueues(struct vtmmio_softc *);
131 static void vtmmio_release_child_resources(struct vtmmio_softc *);
132 static void vtmmio_reset(struct vtmmio_softc *);
133 static void vtmmio_select_virtqueue(struct vtmmio_softc *, int);
134 static void vtmmio_vq_intr(void *);
137 * I/O port read/write wrappers.
139 #define vtmmio_write_config_1(sc, o, v) \
141 bus_write_1((sc)->res[0], (o), (v)); \
142 if (sc->platform != NULL) \
143 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \
145 #define vtmmio_write_config_2(sc, o, v) \
147 bus_write_2((sc)->res[0], (o), (v)); \
148 if (sc->platform != NULL) \
149 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \
151 #define vtmmio_write_config_4(sc, o, v) \
153 bus_write_4((sc)->res[0], (o), (v)); \
154 if (sc->platform != NULL) \
155 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \
158 #define vtmmio_read_config_1(sc, o) \
159 bus_read_1((sc)->res[0], (o))
160 #define vtmmio_read_config_2(sc, o) \
161 bus_read_2((sc)->res[0], (o))
162 #define vtmmio_read_config_4(sc, o) \
163 bus_read_4((sc)->res[0], (o))
165 static device_method_t vtmmio_methods[] = {
166 /* Device interface. */
167 DEVMETHOD(device_probe, vtmmio_probe),
168 DEVMETHOD(device_attach, vtmmio_attach),
169 DEVMETHOD(device_detach, vtmmio_detach),
170 DEVMETHOD(device_suspend, vtmmio_suspend),
171 DEVMETHOD(device_resume, vtmmio_resume),
172 DEVMETHOD(device_shutdown, vtmmio_shutdown),
175 DEVMETHOD(bus_driver_added, vtmmio_driver_added),
176 DEVMETHOD(bus_child_detached, vtmmio_child_detached),
177 DEVMETHOD(bus_read_ivar, vtmmio_read_ivar),
178 DEVMETHOD(bus_write_ivar, vtmmio_write_ivar),
180 /* VirtIO bus interface. */
181 DEVMETHOD(virtio_bus_negotiate_features, vtmmio_negotiate_features),
182 DEVMETHOD(virtio_bus_with_feature, vtmmio_with_feature),
183 DEVMETHOD(virtio_bus_alloc_virtqueues, vtmmio_alloc_virtqueues),
184 DEVMETHOD(virtio_bus_setup_intr, vtmmio_setup_intr),
185 DEVMETHOD(virtio_bus_stop, vtmmio_stop),
186 DEVMETHOD(virtio_bus_poll, vtmmio_poll),
187 DEVMETHOD(virtio_bus_reinit, vtmmio_reinit),
188 DEVMETHOD(virtio_bus_reinit_complete, vtmmio_reinit_complete),
189 DEVMETHOD(virtio_bus_notify_vq, vtmmio_notify_virtqueue),
190 DEVMETHOD(virtio_bus_read_device_config, vtmmio_read_dev_config),
191 DEVMETHOD(virtio_bus_write_device_config, vtmmio_write_dev_config),
196 static driver_t vtmmio_driver = {
199 sizeof(struct vtmmio_softc)
202 devclass_t vtmmio_devclass;
204 DRIVER_MODULE(virtio_mmio, simplebus, vtmmio_driver, vtmmio_devclass, 0, 0);
205 DRIVER_MODULE(virtio_mmio, ofwbus, vtmmio_driver, vtmmio_devclass, 0, 0);
206 MODULE_VERSION(virtio_mmio, 1);
207 MODULE_DEPEND(virtio_mmio, simplebus, 1, 1, 1);
208 MODULE_DEPEND(virtio_mmio, virtio, 1, 1, 1);
211 vtmmio_setup_intr(device_t dev, enum intr_type type)
213 struct vtmmio_softc *sc;
217 sc = device_get_softc(dev);
219 if (sc->platform != NULL) {
220 err = VIRTIO_MMIO_SETUP_INTR(sc->platform, sc->dev,
223 /* Okay we have backend-specific interrupts */
229 sc->res[1] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
232 device_printf(dev, "Can't allocate interrupt\n");
236 if (bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC | INTR_MPSAFE,
237 NULL, vtmmio_vq_intr, sc, &sc->ih)) {
238 device_printf(dev, "Can't setup the interrupt\n");
246 vtmmio_probe(device_t dev)
249 if (!ofw_bus_status_okay(dev))
252 if (!ofw_bus_is_compatible(dev, "virtio,mmio"))
255 device_set_desc(dev, "VirtIO MMIO adapter");
256 return (BUS_PROBE_DEFAULT);
260 vtmmio_setup_platform(struct vtmmio_softc *sc)
262 phandle_t platform_node;
269 if ((node = ofw_bus_get_node(sc->dev)) == -1)
272 if (OF_searchencprop(node, "platform", &xref,
273 sizeof(xref)) == -1) {
277 platform_node = OF_node_from_xref(xref);
279 SLIST_FOREACH(ic, &fdt_ic_list_head, fdt_ics) {
280 if (ic->iph == platform_node) {
281 sc->platform = ic->dev;
286 if (sc->platform == NULL) {
287 /* No platform-specific device. Ignore it. */
294 vtmmio_attach(device_t dev)
296 struct vtmmio_softc *sc;
300 sc = device_get_softc(dev);
303 vtmmio_setup_platform(sc);
306 sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
309 device_printf(dev, "Cannot allocate memory window.\n");
315 /* Tell the host we've noticed this device. */
316 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
318 if ((child = device_add_child(dev, NULL, -1)) == NULL) {
319 device_printf(dev, "Cannot create child device.\n");
320 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
325 sc->vtmmio_child_dev = child;
326 vtmmio_probe_and_attach_child(sc);
332 vtmmio_detach(device_t dev)
334 struct vtmmio_softc *sc;
338 sc = device_get_softc(dev);
340 if ((child = sc->vtmmio_child_dev) != NULL) {
341 error = device_delete_child(dev, child);
344 sc->vtmmio_child_dev = NULL;
349 if (sc->res[0] != NULL) {
350 bus_release_resource(dev, SYS_RES_MEMORY, 0,
359 vtmmio_suspend(device_t dev)
362 return (bus_generic_suspend(dev));
366 vtmmio_resume(device_t dev)
369 return (bus_generic_resume(dev));
373 vtmmio_shutdown(device_t dev)
376 (void) bus_generic_shutdown(dev);
378 /* Forcibly stop the host device. */
385 vtmmio_driver_added(device_t dev, driver_t *driver)
387 struct vtmmio_softc *sc;
389 sc = device_get_softc(dev);
391 vtmmio_probe_and_attach_child(sc);
395 vtmmio_child_detached(device_t dev, device_t child)
397 struct vtmmio_softc *sc;
399 sc = device_get_softc(dev);
402 vtmmio_release_child_resources(sc);
406 vtmmio_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
408 struct vtmmio_softc *sc;
410 sc = device_get_softc(dev);
412 if (sc->vtmmio_child_dev != child)
416 case VIRTIO_IVAR_DEVTYPE:
417 case VIRTIO_IVAR_SUBDEVICE:
418 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_DEVICE_ID);
420 case VIRTIO_IVAR_VENDOR:
421 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_VENDOR_ID);
431 vtmmio_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
433 struct vtmmio_softc *sc;
435 sc = device_get_softc(dev);
437 if (sc->vtmmio_child_dev != child)
441 case VIRTIO_IVAR_FEATURE_DESC:
442 sc->vtmmio_child_feat_desc = (void *) value;
452 vtmmio_negotiate_features(device_t dev, uint64_t child_features)
454 struct vtmmio_softc *sc;
455 uint64_t host_features, features;
457 sc = device_get_softc(dev);
459 host_features = vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES);
460 vtmmio_describe_features(sc, "host", host_features);
463 * Limit negotiated features to what the driver, virtqueue, and
466 features = host_features & child_features;
467 features = virtqueue_filter_features(features);
468 sc->vtmmio_features = features;
470 vtmmio_describe_features(sc, "negotiated", features);
471 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features);
477 vtmmio_with_feature(device_t dev, uint64_t feature)
479 struct vtmmio_softc *sc;
481 sc = device_get_softc(dev);
483 return ((sc->vtmmio_features & feature) != 0);
487 vtmmio_alloc_virtqueues(device_t dev, int flags, int nvqs,
488 struct vq_alloc_info *vq_info)
490 struct vtmmio_virtqueue *vqx;
491 struct vq_alloc_info *info;
492 struct vtmmio_softc *sc;
493 struct virtqueue *vq;
497 sc = device_get_softc(dev);
499 if (sc->vtmmio_nvqs != 0)
504 sc->vtmmio_vqs = malloc(nvqs * sizeof(struct vtmmio_virtqueue),
505 M_DEVBUF, M_NOWAIT | M_ZERO);
506 if (sc->vtmmio_vqs == NULL)
509 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE, 1 << PAGE_SHIFT);
511 for (idx = 0; idx < nvqs; idx++) {
512 vqx = &sc->vtmmio_vqs[idx];
513 info = &vq_info[idx];
515 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx);
517 vtmmio_select_virtqueue(sc, idx);
518 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX);
520 error = virtqueue_alloc(dev, idx, size,
521 VIRTIO_MMIO_VRING_ALIGN, 0xFFFFFFFFUL, info, &vq);
524 "cannot allocate virtqueue %d: %d\n",
529 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size);
530 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN,
531 VIRTIO_MMIO_VRING_ALIGN);
533 device_printf(dev, "virtqueue paddr 0x%08lx\n",
534 (uint64_t)virtqueue_paddr(vq));
536 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN,
537 virtqueue_paddr(vq) >> PAGE_SHIFT);
539 vqx->vtv_vq = *info->vqai_vq = vq;
540 vqx->vtv_no_intr = info->vqai_intr == NULL;
546 vtmmio_free_virtqueues(sc);
552 vtmmio_stop(device_t dev)
555 vtmmio_reset(device_get_softc(dev));
559 vtmmio_poll(device_t dev)
561 struct vtmmio_softc *sc;
563 sc = device_get_softc(dev);
565 if (sc->platform != NULL)
566 VIRTIO_MMIO_POLL(sc->platform);
570 vtmmio_reinit(device_t dev, uint64_t features)
572 struct vtmmio_softc *sc;
575 sc = device_get_softc(dev);
577 if (vtmmio_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
581 * Quickly drive the status through ACK and DRIVER. The device
582 * does not become usable again until vtmmio_reinit_complete().
584 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
585 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
587 vtmmio_negotiate_features(dev, features);
589 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
590 error = vtmmio_reinit_virtqueue(sc, idx);
599 vtmmio_reinit_complete(device_t dev)
602 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
606 vtmmio_notify_virtqueue(device_t dev, uint16_t queue)
608 struct vtmmio_softc *sc;
610 sc = device_get_softc(dev);
612 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NOTIFY, queue);
616 vtmmio_get_status(device_t dev)
618 struct vtmmio_softc *sc;
620 sc = device_get_softc(dev);
622 return (vtmmio_read_config_4(sc, VIRTIO_MMIO_STATUS));
626 vtmmio_set_status(device_t dev, uint8_t status)
628 struct vtmmio_softc *sc;
630 sc = device_get_softc(dev);
632 if (status != VIRTIO_CONFIG_STATUS_RESET)
633 status |= vtmmio_get_status(dev);
635 vtmmio_write_config_4(sc, VIRTIO_MMIO_STATUS, status);
639 vtmmio_read_dev_config(device_t dev, bus_size_t offset,
640 void *dst, int length)
642 struct vtmmio_softc *sc;
647 sc = device_get_softc(dev);
648 off = VIRTIO_MMIO_CONFIG + offset;
650 for (d = dst; length > 0; d += size, off += size, length -= size) {
651 #ifdef ALLOW_WORD_ALIGNED_ACCESS
654 *(uint32_t *)d = vtmmio_read_config_4(sc, off);
655 } else if (length >= 2) {
657 *(uint16_t *)d = vtmmio_read_config_2(sc, off);
662 *d = vtmmio_read_config_1(sc, off);
668 vtmmio_write_dev_config(device_t dev, bus_size_t offset,
669 void *src, int length)
671 struct vtmmio_softc *sc;
676 sc = device_get_softc(dev);
677 off = VIRTIO_MMIO_CONFIG + offset;
679 for (s = src; length > 0; s += size, off += size, length -= size) {
680 #ifdef ALLOW_WORD_ALIGNED_ACCESS
683 vtmmio_write_config_4(sc, off, *(uint32_t *)s);
684 } else if (length >= 2) {
686 vtmmio_write_config_2(sc, off, *(uint16_t *)s);
691 vtmmio_write_config_1(sc, off, *s);
697 vtmmio_describe_features(struct vtmmio_softc *sc, const char *msg,
703 child = sc->vtmmio_child_dev;
705 if (device_is_attached(child) || bootverbose == 0)
708 virtio_describe(dev, msg, features, sc->vtmmio_child_feat_desc);
712 vtmmio_probe_and_attach_child(struct vtmmio_softc *sc)
717 child = sc->vtmmio_child_dev;
722 if (device_get_state(child) != DS_NOTPRESENT) {
726 if (device_probe(child) != 0) {
730 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
731 if (device_attach(child) != 0) {
732 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
734 vtmmio_release_child_resources(sc);
735 /* Reset status for future attempt. */
736 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
738 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
739 VIRTIO_ATTACH_COMPLETED(child);
744 vtmmio_reinit_virtqueue(struct vtmmio_softc *sc, int idx)
746 struct vtmmio_virtqueue *vqx;
747 struct virtqueue *vq;
751 vqx = &sc->vtmmio_vqs[idx];
754 KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));
756 vtmmio_select_virtqueue(sc, idx);
757 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX);
759 error = virtqueue_reinit(vq, size);
763 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN,
764 virtqueue_paddr(vq) >> PAGE_SHIFT);
770 vtmmio_free_interrupts(struct vtmmio_softc *sc)
774 bus_teardown_intr(sc->dev, sc->res[1], sc->ih);
776 if (sc->res[1] != NULL)
777 bus_release_resource(sc->dev, SYS_RES_IRQ, 0, sc->res[1]);
781 vtmmio_free_virtqueues(struct vtmmio_softc *sc)
783 struct vtmmio_virtqueue *vqx;
786 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
787 vqx = &sc->vtmmio_vqs[idx];
789 vtmmio_select_virtqueue(sc, idx);
790 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 0);
792 virtqueue_free(vqx->vtv_vq);
796 free(sc->vtmmio_vqs, M_DEVBUF);
797 sc->vtmmio_vqs = NULL;
802 vtmmio_release_child_resources(struct vtmmio_softc *sc)
805 vtmmio_free_interrupts(sc);
806 vtmmio_free_virtqueues(sc);
810 vtmmio_reset(struct vtmmio_softc *sc)
814 * Setting the status to RESET sets the host device to
815 * the original, uninitialized state.
817 vtmmio_set_status(sc->dev, VIRTIO_CONFIG_STATUS_RESET);
821 vtmmio_select_virtqueue(struct vtmmio_softc *sc, int idx)
824 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx);
828 vtmmio_vq_intr(void *arg)
830 struct vtmmio_virtqueue *vqx;
831 struct vtmmio_softc *sc;
832 struct virtqueue *vq;
838 status = vtmmio_read_config_4(sc, VIRTIO_MMIO_INTERRUPT_STATUS);
839 vtmmio_write_config_4(sc, VIRTIO_MMIO_INTERRUPT_ACK, status);
841 /* The config changed */
842 if (status & VIRTIO_MMIO_INT_CONFIG)
843 if (sc->vtmmio_child_dev != NULL)
844 VIRTIO_CONFIG_CHANGE(sc->vtmmio_child_dev);
846 /* Notify all virtqueues. */
847 if (status & VIRTIO_MMIO_INT_VRING) {
848 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
849 vqx = &sc->vtmmio_vqs[idx];
850 if (vqx->vtv_no_intr == 0) {