2 * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com>
3 * Copyright (c) 2014 The FreeBSD Foundation
6 * This software was developed by SRI International and the University of
7 * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
8 * ("CTSRD"), as part of the DARPA CRASH research programme.
10 * Portions of this software were developed by Andrew Turner
11 * under sponsorship from the FreeBSD Foundation.
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * VirtIO MMIO interface.
37 * This driver is heavily based on VirtIO PCI interface driver.
40 #include <sys/cdefs.h>
41 __FBSDID("$FreeBSD$");
43 #include <sys/param.h>
44 #include <sys/systm.h>
46 #include <sys/kernel.h>
47 #include <sys/module.h>
48 #include <sys/malloc.h>
51 #include <machine/bus.h>
52 #include <machine/resource.h>
54 #include <dev/virtio/virtio.h>
55 #include <dev/virtio/virtqueue.h>
56 #include <dev/virtio/mmio/virtio_mmio.h>
58 #include "virtio_mmio_if.h"
59 #include "virtio_bus_if.h"
60 #include "virtio_if.h"
64 struct vtmmio_virtqueue {
65 struct virtqueue *vtv_vq;
69 static int vtmmio_detach(device_t);
70 static int vtmmio_suspend(device_t);
71 static int vtmmio_resume(device_t);
72 static int vtmmio_shutdown(device_t);
73 static void vtmmio_driver_added(device_t, driver_t *);
74 static void vtmmio_child_detached(device_t, device_t);
75 static int vtmmio_read_ivar(device_t, device_t, int, uintptr_t *);
76 static int vtmmio_write_ivar(device_t, device_t, int, uintptr_t);
77 static uint64_t vtmmio_negotiate_features(device_t, uint64_t);
78 static int vtmmio_with_feature(device_t, uint64_t);
79 static int vtmmio_alloc_virtqueues(device_t, int, int,
80 struct vq_alloc_info *);
81 static int vtmmio_setup_intr(device_t, enum intr_type);
82 static void vtmmio_stop(device_t);
83 static void vtmmio_poll(device_t);
84 static int vtmmio_reinit(device_t, uint64_t);
85 static void vtmmio_reinit_complete(device_t);
86 static void vtmmio_notify_virtqueue(device_t, uint16_t);
87 static uint8_t vtmmio_get_status(device_t);
88 static void vtmmio_set_status(device_t, uint8_t);
89 static void vtmmio_read_dev_config(device_t, bus_size_t, void *, int);
90 static void vtmmio_write_dev_config(device_t, bus_size_t, void *, int);
91 static void vtmmio_describe_features(struct vtmmio_softc *, const char *,
93 static void vtmmio_probe_and_attach_child(struct vtmmio_softc *);
94 static int vtmmio_reinit_virtqueue(struct vtmmio_softc *, int);
95 static void vtmmio_free_interrupts(struct vtmmio_softc *);
96 static void vtmmio_free_virtqueues(struct vtmmio_softc *);
97 static void vtmmio_release_child_resources(struct vtmmio_softc *);
98 static void vtmmio_reset(struct vtmmio_softc *);
99 static void vtmmio_select_virtqueue(struct vtmmio_softc *, int);
100 static void vtmmio_vq_intr(void *);
103 * I/O port read/write wrappers.
105 #define vtmmio_write_config_1(sc, o, v) \
107 if (sc->platform != NULL) \
108 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \
109 bus_write_1((sc)->res[0], (o), (v)); \
110 if (sc->platform != NULL) \
111 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \
113 #define vtmmio_write_config_2(sc, o, v) \
115 if (sc->platform != NULL) \
116 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \
117 bus_write_2((sc)->res[0], (o), (v)); \
118 if (sc->platform != NULL) \
119 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \
121 #define vtmmio_write_config_4(sc, o, v) \
123 if (sc->platform != NULL) \
124 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \
125 bus_write_4((sc)->res[0], (o), (v)); \
126 if (sc->platform != NULL) \
127 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \
130 #define vtmmio_read_config_1(sc, o) \
131 bus_read_1((sc)->res[0], (o))
132 #define vtmmio_read_config_2(sc, o) \
133 bus_read_2((sc)->res[0], (o))
134 #define vtmmio_read_config_4(sc, o) \
135 bus_read_4((sc)->res[0], (o))
137 static device_method_t vtmmio_methods[] = {
138 /* Device interface. */
139 DEVMETHOD(device_attach, vtmmio_attach),
140 DEVMETHOD(device_detach, vtmmio_detach),
141 DEVMETHOD(device_suspend, vtmmio_suspend),
142 DEVMETHOD(device_resume, vtmmio_resume),
143 DEVMETHOD(device_shutdown, vtmmio_shutdown),
146 DEVMETHOD(bus_driver_added, vtmmio_driver_added),
147 DEVMETHOD(bus_child_detached, vtmmio_child_detached),
148 DEVMETHOD(bus_read_ivar, vtmmio_read_ivar),
149 DEVMETHOD(bus_write_ivar, vtmmio_write_ivar),
151 /* VirtIO bus interface. */
152 DEVMETHOD(virtio_bus_negotiate_features, vtmmio_negotiate_features),
153 DEVMETHOD(virtio_bus_with_feature, vtmmio_with_feature),
154 DEVMETHOD(virtio_bus_alloc_virtqueues, vtmmio_alloc_virtqueues),
155 DEVMETHOD(virtio_bus_setup_intr, vtmmio_setup_intr),
156 DEVMETHOD(virtio_bus_stop, vtmmio_stop),
157 DEVMETHOD(virtio_bus_poll, vtmmio_poll),
158 DEVMETHOD(virtio_bus_reinit, vtmmio_reinit),
159 DEVMETHOD(virtio_bus_reinit_complete, vtmmio_reinit_complete),
160 DEVMETHOD(virtio_bus_notify_vq, vtmmio_notify_virtqueue),
161 DEVMETHOD(virtio_bus_read_device_config, vtmmio_read_dev_config),
162 DEVMETHOD(virtio_bus_write_device_config, vtmmio_write_dev_config),
167 DEFINE_CLASS_0(virtio_mmio, vtmmio_driver, vtmmio_methods,
168 sizeof(struct vtmmio_softc));
170 MODULE_VERSION(virtio_mmio, 1);
173 vtmmio_setup_intr(device_t dev, enum intr_type type)
175 struct vtmmio_softc *sc;
179 sc = device_get_softc(dev);
181 if (sc->platform != NULL) {
182 err = VIRTIO_MMIO_SETUP_INTR(sc->platform, sc->dev,
185 /* Okay we have backend-specific interrupts */
191 sc->res[1] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
194 device_printf(dev, "Can't allocate interrupt\n");
198 if (bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC | INTR_MPSAFE,
199 NULL, vtmmio_vq_intr, sc, &sc->ih)) {
200 device_printf(dev, "Can't setup the interrupt\n");
208 vtmmio_attach(device_t dev)
210 struct vtmmio_softc *sc;
214 sc = device_get_softc(dev);
218 sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
221 device_printf(dev, "Cannot allocate memory window.\n");
227 /* Tell the host we've noticed this device. */
228 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
230 if ((child = device_add_child(dev, NULL, -1)) == NULL) {
231 device_printf(dev, "Cannot create child device.\n");
232 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
237 sc->vtmmio_child_dev = child;
238 vtmmio_probe_and_attach_child(sc);
244 vtmmio_detach(device_t dev)
246 struct vtmmio_softc *sc;
250 sc = device_get_softc(dev);
252 if ((child = sc->vtmmio_child_dev) != NULL) {
253 error = device_delete_child(dev, child);
256 sc->vtmmio_child_dev = NULL;
261 if (sc->res[0] != NULL) {
262 bus_release_resource(dev, SYS_RES_MEMORY, 0,
271 vtmmio_suspend(device_t dev)
274 return (bus_generic_suspend(dev));
278 vtmmio_resume(device_t dev)
281 return (bus_generic_resume(dev));
285 vtmmio_shutdown(device_t dev)
288 (void) bus_generic_shutdown(dev);
290 /* Forcibly stop the host device. */
297 vtmmio_driver_added(device_t dev, driver_t *driver)
299 struct vtmmio_softc *sc;
301 sc = device_get_softc(dev);
303 vtmmio_probe_and_attach_child(sc);
307 vtmmio_child_detached(device_t dev, device_t child)
309 struct vtmmio_softc *sc;
311 sc = device_get_softc(dev);
314 vtmmio_release_child_resources(sc);
318 vtmmio_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
320 struct vtmmio_softc *sc;
322 sc = device_get_softc(dev);
324 if (sc->vtmmio_child_dev != child)
328 case VIRTIO_IVAR_DEVTYPE:
329 case VIRTIO_IVAR_SUBDEVICE:
330 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_DEVICE_ID);
332 case VIRTIO_IVAR_VENDOR:
333 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_VENDOR_ID);
343 vtmmio_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
345 struct vtmmio_softc *sc;
347 sc = device_get_softc(dev);
349 if (sc->vtmmio_child_dev != child)
353 case VIRTIO_IVAR_FEATURE_DESC:
354 sc->vtmmio_child_feat_desc = (void *) value;
364 vtmmio_negotiate_features(device_t dev, uint64_t child_features)
366 struct vtmmio_softc *sc;
367 uint64_t host_features, features;
369 sc = device_get_softc(dev);
371 host_features = vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES);
372 vtmmio_describe_features(sc, "host", host_features);
375 * Limit negotiated features to what the driver, virtqueue, and
378 features = host_features & child_features;
379 features = virtqueue_filter_features(features);
380 sc->vtmmio_features = features;
382 vtmmio_describe_features(sc, "negotiated", features);
383 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features);
389 vtmmio_with_feature(device_t dev, uint64_t feature)
391 struct vtmmio_softc *sc;
393 sc = device_get_softc(dev);
395 return ((sc->vtmmio_features & feature) != 0);
399 vtmmio_alloc_virtqueues(device_t dev, int flags, int nvqs,
400 struct vq_alloc_info *vq_info)
402 struct vtmmio_virtqueue *vqx;
403 struct vq_alloc_info *info;
404 struct vtmmio_softc *sc;
405 struct virtqueue *vq;
409 sc = device_get_softc(dev);
411 if (sc->vtmmio_nvqs != 0)
416 sc->vtmmio_vqs = malloc(nvqs * sizeof(struct vtmmio_virtqueue),
417 M_DEVBUF, M_NOWAIT | M_ZERO);
418 if (sc->vtmmio_vqs == NULL)
421 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE,
424 for (idx = 0; idx < nvqs; idx++) {
425 vqx = &sc->vtmmio_vqs[idx];
426 info = &vq_info[idx];
428 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx);
430 vtmmio_select_virtqueue(sc, idx);
431 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX);
433 error = virtqueue_alloc(dev, idx, size,
434 VIRTIO_MMIO_VRING_ALIGN, 0xFFFFFFFFUL, info, &vq);
437 "cannot allocate virtqueue %d: %d\n",
442 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size);
443 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN,
444 VIRTIO_MMIO_VRING_ALIGN);
446 device_printf(dev, "virtqueue paddr 0x%08lx\n",
447 (uint64_t)virtqueue_paddr(vq));
449 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN,
450 virtqueue_paddr(vq) >> PAGE_SHIFT);
452 vqx->vtv_vq = *info->vqai_vq = vq;
453 vqx->vtv_no_intr = info->vqai_intr == NULL;
459 vtmmio_free_virtqueues(sc);
465 vtmmio_stop(device_t dev)
468 vtmmio_reset(device_get_softc(dev));
472 vtmmio_poll(device_t dev)
474 struct vtmmio_softc *sc;
476 sc = device_get_softc(dev);
478 if (sc->platform != NULL)
479 VIRTIO_MMIO_POLL(sc->platform);
483 vtmmio_reinit(device_t dev, uint64_t features)
485 struct vtmmio_softc *sc;
488 sc = device_get_softc(dev);
490 if (vtmmio_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
494 * Quickly drive the status through ACK and DRIVER. The device
495 * does not become usable again until vtmmio_reinit_complete().
497 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
498 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
500 vtmmio_negotiate_features(dev, features);
502 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE,
505 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
506 error = vtmmio_reinit_virtqueue(sc, idx);
515 vtmmio_reinit_complete(device_t dev)
518 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
522 vtmmio_notify_virtqueue(device_t dev, uint16_t queue)
524 struct vtmmio_softc *sc;
526 sc = device_get_softc(dev);
528 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NOTIFY, queue);
532 vtmmio_get_status(device_t dev)
534 struct vtmmio_softc *sc;
536 sc = device_get_softc(dev);
538 return (vtmmio_read_config_4(sc, VIRTIO_MMIO_STATUS));
542 vtmmio_set_status(device_t dev, uint8_t status)
544 struct vtmmio_softc *sc;
546 sc = device_get_softc(dev);
548 if (status != VIRTIO_CONFIG_STATUS_RESET)
549 status |= vtmmio_get_status(dev);
551 vtmmio_write_config_4(sc, VIRTIO_MMIO_STATUS, status);
555 vtmmio_read_dev_config(device_t dev, bus_size_t offset,
556 void *dst, int length)
558 struct vtmmio_softc *sc;
563 sc = device_get_softc(dev);
564 off = VIRTIO_MMIO_CONFIG + offset;
566 for (d = dst; length > 0; d += size, off += size, length -= size) {
567 #ifdef ALLOW_WORD_ALIGNED_ACCESS
570 *(uint32_t *)d = vtmmio_read_config_4(sc, off);
571 } else if (length >= 2) {
573 *(uint16_t *)d = vtmmio_read_config_2(sc, off);
578 *d = vtmmio_read_config_1(sc, off);
584 vtmmio_write_dev_config(device_t dev, bus_size_t offset,
585 void *src, int length)
587 struct vtmmio_softc *sc;
592 sc = device_get_softc(dev);
593 off = VIRTIO_MMIO_CONFIG + offset;
595 for (s = src; length > 0; s += size, off += size, length -= size) {
596 #ifdef ALLOW_WORD_ALIGNED_ACCESS
599 vtmmio_write_config_4(sc, off, *(uint32_t *)s);
600 } else if (length >= 2) {
602 vtmmio_write_config_2(sc, off, *(uint16_t *)s);
607 vtmmio_write_config_1(sc, off, *s);
613 vtmmio_describe_features(struct vtmmio_softc *sc, const char *msg,
619 child = sc->vtmmio_child_dev;
621 if (device_is_attached(child) || bootverbose == 0)
624 virtio_describe(dev, msg, features, sc->vtmmio_child_feat_desc);
628 vtmmio_probe_and_attach_child(struct vtmmio_softc *sc)
633 child = sc->vtmmio_child_dev;
638 if (device_get_state(child) != DS_NOTPRESENT) {
642 if (device_probe(child) != 0) {
646 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
647 if (device_attach(child) != 0) {
648 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
650 vtmmio_release_child_resources(sc);
651 /* Reset status for future attempt. */
652 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
654 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
655 VIRTIO_ATTACH_COMPLETED(child);
660 vtmmio_reinit_virtqueue(struct vtmmio_softc *sc, int idx)
662 struct vtmmio_virtqueue *vqx;
663 struct virtqueue *vq;
667 vqx = &sc->vtmmio_vqs[idx];
670 KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));
672 vtmmio_select_virtqueue(sc, idx);
673 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX);
675 error = virtqueue_reinit(vq, size);
679 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size);
680 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN,
681 VIRTIO_MMIO_VRING_ALIGN);
683 device_printf(sc->dev, "virtqueue paddr 0x%08lx\n",
684 (uint64_t)virtqueue_paddr(vq));
686 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN,
687 virtqueue_paddr(vq) >> PAGE_SHIFT);
693 vtmmio_free_interrupts(struct vtmmio_softc *sc)
697 bus_teardown_intr(sc->dev, sc->res[1], sc->ih);
699 if (sc->res[1] != NULL)
700 bus_release_resource(sc->dev, SYS_RES_IRQ, 0, sc->res[1]);
704 vtmmio_free_virtqueues(struct vtmmio_softc *sc)
706 struct vtmmio_virtqueue *vqx;
709 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
710 vqx = &sc->vtmmio_vqs[idx];
712 vtmmio_select_virtqueue(sc, idx);
713 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 0);
715 virtqueue_free(vqx->vtv_vq);
719 free(sc->vtmmio_vqs, M_DEVBUF);
720 sc->vtmmio_vqs = NULL;
725 vtmmio_release_child_resources(struct vtmmio_softc *sc)
728 vtmmio_free_interrupts(sc);
729 vtmmio_free_virtqueues(sc);
733 vtmmio_reset(struct vtmmio_softc *sc)
737 * Setting the status to RESET sets the host device to
738 * the original, uninitialized state.
740 vtmmio_set_status(sc->dev, VIRTIO_CONFIG_STATUS_RESET);
744 vtmmio_select_virtqueue(struct vtmmio_softc *sc, int idx)
747 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx);
751 vtmmio_vq_intr(void *arg)
753 struct vtmmio_virtqueue *vqx;
754 struct vtmmio_softc *sc;
755 struct virtqueue *vq;
761 status = vtmmio_read_config_4(sc, VIRTIO_MMIO_INTERRUPT_STATUS);
762 vtmmio_write_config_4(sc, VIRTIO_MMIO_INTERRUPT_ACK, status);
764 /* The config changed */
765 if (status & VIRTIO_MMIO_INT_CONFIG)
766 if (sc->vtmmio_child_dev != NULL)
767 VIRTIO_CONFIG_CHANGE(sc->vtmmio_child_dev);
769 /* Notify all virtqueues. */
770 if (status & VIRTIO_MMIO_INT_VRING) {
771 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
772 vqx = &sc->vtmmio_vqs[idx];
773 if (vqx->vtv_no_intr == 0) {