sys/dev/virtio/mmio/virtio_mmio.c

   1 /*-
   2  * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com>
   3  * Copyright (c) 2014 The FreeBSD Foundation
   4  * All rights reserved.
   5  *
   6  * This software was developed by SRI International and the University of
   7  * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
   8  * ("CTSRD"), as part of the DARPA CRASH research programme.
   9  *
  10  * Portions of this software were developed by Andrew Turner
  11  * under sponsorship from the FreeBSD Foundation.
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  * 1. Redistributions of source code must retain the above copyright
  17  *    notice, this list of conditions and the following disclaimer.
  18  * 2. Redistributions in binary form must reproduce the above copyright
  19  *    notice, this list of conditions and the following disclaimer in the
  20  *    documentation and/or other materials provided with the distribution.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34
  35 /*
  36  * VirtIO MMIO interface.
  37  * This driver is heavily based on VirtIO PCI interface driver.
  38  */
  39
  40 /*
  41  * FDT example:
  42  *              virtio_block@1000 {
  43  *                      compatible = "virtio,mmio";
  44  *                      reg = <0x1000 0x100>;
  45  *                      interrupts = <63>;
  46  *                      interrupt-parent = <&GIC>;
  47  *              };
  48  */
  49
  50 #include <sys/cdefs.h>
  51 __FBSDID("$FreeBSD$");
  52
  53 #include <sys/param.h>
  54 #include <sys/systm.h>
  55 #include <sys/bus.h>
  56 #include <sys/kernel.h>
  57 #include <sys/module.h>
  58 #include <sys/malloc.h>
  59 #include <sys/rman.h>
  60
  61 #include <machine/bus.h>
  62 #include <machine/resource.h>
  63
  64 #include <dev/fdt/fdt_common.h>
  65 #include <dev/ofw/openfirm.h>
  66 #include <dev/ofw/ofw_bus.h>
  67 #include <dev/ofw/ofw_bus_subr.h>
  68
  69 #include <dev/virtio/virtio.h>
  70 #include <dev/virtio/virtqueue.h>
  71 #include <dev/virtio/mmio/virtio_mmio.h>
  72
  73 #include "virtio_mmio_if.h"
  74 #include "virtio_bus_if.h"
  75 #include "virtio_if.h"
  76
  77 #define PAGE_SHIFT      12
  78
  79 struct vtmmio_virtqueue {
  80         struct virtqueue        *vtv_vq;
  81         int                      vtv_no_intr;
  82 };
  83
  84 struct vtmmio_softc {
  85         device_t                        dev;
  86         device_t                        platform;
  87         struct resource                 *res[2];
  88
  89         uint64_t                        vtmmio_features;
  90         uint32_t                        vtmmio_flags;
  91
  92         /* This "bus" will only ever have one child. */
  93         device_t                        vtmmio_child_dev;
  94         struct virtio_feature_desc      *vtmmio_child_feat_desc;
  95
  96         int                             vtmmio_nvqs;
  97         struct vtmmio_virtqueue         *vtmmio_vqs;
  98         void                            *ih;
  99 };
 100
 101 static int      vtmmio_probe(device_t);
 102 static int      vtmmio_attach(device_t);
 103 static int      vtmmio_detach(device_t);
 104 static int      vtmmio_suspend(device_t);
 105 static int      vtmmio_resume(device_t);
 106 static int      vtmmio_shutdown(device_t);
 107 static void     vtmmio_driver_added(device_t, driver_t *);
 108 static void     vtmmio_child_detached(device_t, device_t);
 109 static int      vtmmio_read_ivar(device_t, device_t, int, uintptr_t *);
 110 static int      vtmmio_write_ivar(device_t, device_t, int, uintptr_t);
 111 static uint64_t vtmmio_negotiate_features(device_t, uint64_t);
 112 static int      vtmmio_with_feature(device_t, uint64_t);
 113 static int      vtmmio_alloc_virtqueues(device_t, int, int,
 114                     struct vq_alloc_info *);
 115 static int      vtmmio_setup_intr(device_t, enum intr_type);
 116 static void     vtmmio_stop(device_t);
 117 static void     vtmmio_poll(device_t);
 118 static int      vtmmio_reinit(device_t, uint64_t);
 119 static void     vtmmio_reinit_complete(device_t);
 120 static void     vtmmio_notify_virtqueue(device_t, uint16_t);
 121 static uint8_t  vtmmio_get_status(device_t);
 122 static void     vtmmio_set_status(device_t, uint8_t);
 123 static void     vtmmio_read_dev_config(device_t, bus_size_t, void *, int);
 124 static void     vtmmio_write_dev_config(device_t, bus_size_t, void *, int);
 125 static void     vtmmio_describe_features(struct vtmmio_softc *, const char *,
 126                     uint64_t);
 127 static void     vtmmio_probe_and_attach_child(struct vtmmio_softc *);
 128 static int      vtmmio_reinit_virtqueue(struct vtmmio_softc *, int);
 129 static void     vtmmio_free_interrupts(struct vtmmio_softc *);
 130 static void     vtmmio_free_virtqueues(struct vtmmio_softc *);
 131 static void     vtmmio_release_child_resources(struct vtmmio_softc *);
 132 static void     vtmmio_reset(struct vtmmio_softc *);
 133 static void     vtmmio_select_virtqueue(struct vtmmio_softc *, int);
 134 static void     vtmmio_vq_intr(void *);
 135
 136 /*
 137  * I/O port read/write wrappers.
 138  */
 139 #define vtmmio_write_config_1(sc, o, v)                         \
 140 do {                                                            \
 141         bus_write_1((sc)->res[0], (o), (v));                    \
 142         if (sc->platform != NULL)                               \
 143                 VIRTIO_MMIO_NOTE(sc->platform, (o), (v));       \
 144 } while (0)
 145 #define vtmmio_write_config_2(sc, o, v)                         \
 146 do {                                                            \
 147         bus_write_2((sc)->res[0], (o), (v));                    \
 148         if (sc->platform != NULL)                               \
 149                 VIRTIO_MMIO_NOTE(sc->platform, (o), (v));       \
 150 } while (0)
 151 #define vtmmio_write_config_4(sc, o, v)                         \
 152 do {                                                            \
 153         bus_write_4((sc)->res[0], (o), (v));                    \
 154         if (sc->platform != NULL)                               \
 155                 VIRTIO_MMIO_NOTE(sc->platform, (o), (v));       \
 156 } while (0)
 157
 158 #define vtmmio_read_config_1(sc, o) \
 159         bus_read_1((sc)->res[0], (o))
 160 #define vtmmio_read_config_2(sc, o) \
 161         bus_read_2((sc)->res[0], (o))
 162 #define vtmmio_read_config_4(sc, o) \
 163         bus_read_4((sc)->res[0], (o))
 164
 165 static device_method_t vtmmio_methods[] = {
 166         /* Device interface. */
 167         DEVMETHOD(device_probe,                   vtmmio_probe),
 168         DEVMETHOD(device_attach,                  vtmmio_attach),
 169         DEVMETHOD(device_detach,                  vtmmio_detach),
 170         DEVMETHOD(device_suspend,                 vtmmio_suspend),
 171         DEVMETHOD(device_resume,                  vtmmio_resume),
 172         DEVMETHOD(device_shutdown,                vtmmio_shutdown),
 173
 174         /* Bus interface. */
 175         DEVMETHOD(bus_driver_added,               vtmmio_driver_added),
 176         DEVMETHOD(bus_child_detached,             vtmmio_child_detached),
 177         DEVMETHOD(bus_read_ivar,                  vtmmio_read_ivar),
 178         DEVMETHOD(bus_write_ivar,                 vtmmio_write_ivar),
 179
 180         /* VirtIO bus interface. */
 181         DEVMETHOD(virtio_bus_negotiate_features,  vtmmio_negotiate_features),
 182         DEVMETHOD(virtio_bus_with_feature,        vtmmio_with_feature),
 183         DEVMETHOD(virtio_bus_alloc_virtqueues,    vtmmio_alloc_virtqueues),
 184         DEVMETHOD(virtio_bus_setup_intr,          vtmmio_setup_intr),
 185         DEVMETHOD(virtio_bus_stop,                vtmmio_stop),
 186         DEVMETHOD(virtio_bus_poll,                vtmmio_poll),
 187         DEVMETHOD(virtio_bus_reinit,              vtmmio_reinit),
 188         DEVMETHOD(virtio_bus_reinit_complete,     vtmmio_reinit_complete),
 189         DEVMETHOD(virtio_bus_notify_vq,           vtmmio_notify_virtqueue),
 190         DEVMETHOD(virtio_bus_read_device_config,  vtmmio_read_dev_config),
 191         DEVMETHOD(virtio_bus_write_device_config, vtmmio_write_dev_config),
 192
 193         DEVMETHOD_END
 194 };
 195
 196 static driver_t vtmmio_driver = {
 197         "virtio_mmio",
 198         vtmmio_methods,
 199         sizeof(struct vtmmio_softc)
 200 };
 201
 202 devclass_t vtmmio_devclass;
 203
 204 DRIVER_MODULE(virtio_mmio, simplebus, vtmmio_driver, vtmmio_devclass, 0, 0);
 205 DRIVER_MODULE(virtio_mmio, ofwbus, vtmmio_driver, vtmmio_devclass, 0, 0);
 206 MODULE_VERSION(virtio_mmio, 1);
 207 MODULE_DEPEND(virtio_mmio, simplebus, 1, 1, 1);
 208 MODULE_DEPEND(virtio_mmio, virtio, 1, 1, 1);
 209
 210 static int
 211 vtmmio_setup_intr(device_t dev, enum intr_type type)
 212 {
 213         struct vtmmio_softc *sc;
 214         int rid;
 215         int err;
 216
 217         sc = device_get_softc(dev);
 218
 219         if (sc->platform != NULL) {
 220                 err = VIRTIO_MMIO_SETUP_INTR(sc->platform, sc->dev,
 221                                         vtmmio_vq_intr, sc);
 222                 if (err == 0) {
 223                         /* Okay we have backend-specific interrupts */
 224                         return (0);
 225                 }
 226         }
 227
 228         rid = 0;
 229         sc->res[1] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 230                 RF_ACTIVE);
 231         if (!sc->res[1]) {
 232                 device_printf(dev, "Can't allocate interrupt\n");
 233                 return (ENXIO);
 234         }
 235
 236         if (bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC | INTR_MPSAFE,
 237                 NULL, vtmmio_vq_intr, sc, &sc->ih)) {
 238                 device_printf(dev, "Can't setup the interrupt\n");
 239                 return (ENXIO);
 240         }
 241
 242         return (0);
 243 }
 244
 245 static int
 246 vtmmio_probe(device_t dev)
 247 {
 248
 249         if (!ofw_bus_status_okay(dev))
 250                 return (ENXIO);
 251
 252         if (!ofw_bus_is_compatible(dev, "virtio,mmio"))
 253                 return (ENXIO);
 254
 255         device_set_desc(dev, "VirtIO MMIO adapter");
 256         return (BUS_PROBE_DEFAULT);
 257 }
 258
 259 static int
 260 vtmmio_setup_platform(struct vtmmio_softc *sc)
 261 {
 262         phandle_t platform_node;
 263         struct fdt_ic *ic;
 264         phandle_t xref;
 265         phandle_t node;
 266
 267         sc->platform = NULL;
 268
 269         if ((node = ofw_bus_get_node(sc->dev)) == -1)
 270                 return (ENXIO);
 271
 272         if (OF_searchencprop(node, "platform", &xref,
 273                 sizeof(xref)) == -1) {
 274                 return (ENXIO);
 275         }
 276
 277         platform_node = OF_node_from_xref(xref);
 278
 279         SLIST_FOREACH(ic, &fdt_ic_list_head, fdt_ics) {
 280                 if (ic->iph == platform_node) {
 281                         sc->platform = ic->dev;
 282                         break;
 283                 }
 284         }
 285
 286         if (sc->platform == NULL) {
 287                 /* No platform-specific device. Ignore it. */
 288         }
 289
 290         return (0);
 291 }
 292
 293 static int
 294 vtmmio_attach(device_t dev)
 295 {
 296         struct vtmmio_softc *sc;
 297         device_t child;
 298         int rid;
 299
 300         sc = device_get_softc(dev);
 301         sc->dev = dev;
 302
 303         vtmmio_setup_platform(sc);
 304
 305         rid = 0;
 306         sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 307                         RF_ACTIVE);
 308         if (!sc->res[0]) {
 309                 device_printf(dev, "Cannot allocate memory window.\n");
 310                 return (ENXIO);
 311         }
 312
 313         vtmmio_reset(sc);
 314
 315         /* Tell the host we've noticed this device. */
 316         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
 317
 318         if ((child = device_add_child(dev, NULL, -1)) == NULL) {
 319                 device_printf(dev, "Cannot create child device.\n");
 320                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
 321                 vtmmio_detach(dev);
 322                 return (ENOMEM);
 323         }
 324
 325         sc->vtmmio_child_dev = child;
 326         vtmmio_probe_and_attach_child(sc);
 327
 328         return (0);
 329 }
 330
 331 static int
 332 vtmmio_detach(device_t dev)
 333 {
 334         struct vtmmio_softc *sc;
 335         device_t child;
 336         int error;
 337
 338         sc = device_get_softc(dev);
 339
 340         if ((child = sc->vtmmio_child_dev) != NULL) {
 341                 error = device_delete_child(dev, child);
 342                 if (error)
 343                         return (error);
 344                 sc->vtmmio_child_dev = NULL;
 345         }
 346
 347         vtmmio_reset(sc);
 348
 349         if (sc->res[0] != NULL) {
 350                 bus_release_resource(dev, SYS_RES_MEMORY, 0,
 351                     sc->res[0]);
 352                 sc->res[0] = NULL;
 353         }
 354
 355         return (0);
 356 }
 357
 358 static int
 359 vtmmio_suspend(device_t dev)
 360 {
 361
 362         return (bus_generic_suspend(dev));
 363 }
 364
 365 static int
 366 vtmmio_resume(device_t dev)
 367 {
 368
 369         return (bus_generic_resume(dev));
 370 }
 371
 372 static int
 373 vtmmio_shutdown(device_t dev)
 374 {
 375
 376         (void) bus_generic_shutdown(dev);
 377
 378         /* Forcibly stop the host device. */
 379         vtmmio_stop(dev);
 380
 381         return (0);
 382 }
 383
 384 static void
 385 vtmmio_driver_added(device_t dev, driver_t *driver)
 386 {
 387         struct vtmmio_softc *sc;
 388
 389         sc = device_get_softc(dev);
 390
 391         vtmmio_probe_and_attach_child(sc);
 392 }
 393
 394 static void
 395 vtmmio_child_detached(device_t dev, device_t child)
 396 {
 397         struct vtmmio_softc *sc;
 398
 399         sc = device_get_softc(dev);
 400
 401         vtmmio_reset(sc);
 402         vtmmio_release_child_resources(sc);
 403 }
 404
 405 static int
 406 vtmmio_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
 407 {
 408         struct vtmmio_softc *sc;
 409
 410         sc = device_get_softc(dev);
 411
 412         if (sc->vtmmio_child_dev != child)
 413                 return (ENOENT);
 414
 415         switch (index) {
 416         case VIRTIO_IVAR_DEVTYPE:
 417         case VIRTIO_IVAR_SUBDEVICE:
 418                 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_DEVICE_ID);
 419                 break;
 420         case VIRTIO_IVAR_VENDOR:
 421                 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_VENDOR_ID);
 422                 break;
 423         default:
 424                 return (ENOENT);
 425         }
 426
 427         return (0);
 428 }
 429
 430 static int
 431 vtmmio_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
 432 {
 433         struct vtmmio_softc *sc;
 434
 435         sc = device_get_softc(dev);
 436
 437         if (sc->vtmmio_child_dev != child)
 438                 return (ENOENT);
 439
 440         switch (index) {
 441         case VIRTIO_IVAR_FEATURE_DESC:
 442                 sc->vtmmio_child_feat_desc = (void *) value;
 443                 break;
 444         default:
 445                 return (ENOENT);
 446         }
 447
 448         return (0);
 449 }
 450
 451 static uint64_t
 452 vtmmio_negotiate_features(device_t dev, uint64_t child_features)
 453 {
 454         struct vtmmio_softc *sc;
 455         uint64_t host_features, features;
 456
 457         sc = device_get_softc(dev);
 458
 459         host_features = vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES);
 460         vtmmio_describe_features(sc, "host", host_features);
 461
 462         /*
 463          * Limit negotiated features to what the driver, virtqueue, and
 464          * host all support.
 465          */
 466         features = host_features & child_features;
 467         features = virtqueue_filter_features(features);
 468         sc->vtmmio_features = features;
 469
 470         vtmmio_describe_features(sc, "negotiated", features);
 471         vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features);
 472
 473         return (features);
 474 }
 475
 476 static int
 477 vtmmio_with_feature(device_t dev, uint64_t feature)
 478 {
 479         struct vtmmio_softc *sc;
 480
 481         sc = device_get_softc(dev);
 482
 483         return ((sc->vtmmio_features & feature) != 0);
 484 }
 485
 486 static int
 487 vtmmio_alloc_virtqueues(device_t dev, int flags, int nvqs,
 488     struct vq_alloc_info *vq_info)
 489 {
 490         struct vtmmio_virtqueue *vqx;
 491         struct vq_alloc_info *info;
 492         struct vtmmio_softc *sc;
 493         struct virtqueue *vq;
 494         uint32_t size;
 495         int idx, error;
 496
 497         sc = device_get_softc(dev);
 498
 499         if (sc->vtmmio_nvqs != 0)
 500                 return (EALREADY);
 501         if (nvqs <= 0)
 502                 return (EINVAL);
 503
 504         sc->vtmmio_vqs = malloc(nvqs * sizeof(struct vtmmio_virtqueue),
 505             M_DEVBUF, M_NOWAIT | M_ZERO);
 506         if (sc->vtmmio_vqs == NULL)
 507                 return (ENOMEM);
 508
 509         vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE, 1 << PAGE_SHIFT);
 510
 511         for (idx = 0; idx < nvqs; idx++) {
 512                 vqx = &sc->vtmmio_vqs[idx];
 513                 info = &vq_info[idx];
 514
 515                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx);
 516
 517                 vtmmio_select_virtqueue(sc, idx);
 518                 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX);
 519
 520                 error = virtqueue_alloc(dev, idx, size,
 521                     VIRTIO_MMIO_VRING_ALIGN, 0xFFFFFFFFUL, info, &vq);
 522                 if (error) {
 523                         device_printf(dev,
 524                             "cannot allocate virtqueue %d: %d\n",
 525                             idx, error);
 526                         break;
 527                 }
 528
 529                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size);
 530                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN,
 531                     VIRTIO_MMIO_VRING_ALIGN);
 532 #if 0
 533                 device_printf(dev, "virtqueue paddr 0x%08lx\n",
 534                                 (uint64_t)virtqueue_paddr(vq));
 535 #endif
 536                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN,
 537                         virtqueue_paddr(vq) >> PAGE_SHIFT);
 538
 539                 vqx->vtv_vq = *info->vqai_vq = vq;
 540                 vqx->vtv_no_intr = info->vqai_intr == NULL;
 541
 542                 sc->vtmmio_nvqs++;
 543         }
 544
 545         if (error)
 546                 vtmmio_free_virtqueues(sc);
 547
 548         return (error);
 549 }
 550
 551 static void
 552 vtmmio_stop(device_t dev)
 553 {
 554
 555         vtmmio_reset(device_get_softc(dev));
 556 }
 557
 558 static void
 559 vtmmio_poll(device_t dev)
 560 {
 561         struct vtmmio_softc *sc;
 562
 563         sc = device_get_softc(dev);
 564
 565         if (sc->platform != NULL)
 566                 VIRTIO_MMIO_POLL(sc->platform);
 567 }
 568
 569 static int
 570 vtmmio_reinit(device_t dev, uint64_t features)
 571 {
 572         struct vtmmio_softc *sc;
 573         int idx, error;
 574
 575         sc = device_get_softc(dev);
 576
 577         if (vtmmio_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
 578                 vtmmio_stop(dev);
 579
 580         /*
 581          * Quickly drive the status through ACK and DRIVER. The device
 582          * does not become usable again until vtmmio_reinit_complete().
 583          */
 584         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
 585         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
 586
 587         vtmmio_negotiate_features(dev, features);
 588
 589         for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
 590                 error = vtmmio_reinit_virtqueue(sc, idx);
 591                 if (error)
 592                         return (error);
 593         }
 594
 595         return (0);
 596 }
 597
 598 static void
 599 vtmmio_reinit_complete(device_t dev)
 600 {
 601
 602         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
 603 }
 604
 605 static void
 606 vtmmio_notify_virtqueue(device_t dev, uint16_t queue)
 607 {
 608         struct vtmmio_softc *sc;
 609
 610         sc = device_get_softc(dev);
 611
 612         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NOTIFY, queue);
 613 }
 614
 615 static uint8_t
 616 vtmmio_get_status(device_t dev)
 617 {
 618         struct vtmmio_softc *sc;
 619
 620         sc = device_get_softc(dev);
 621
 622         return (vtmmio_read_config_4(sc, VIRTIO_MMIO_STATUS));
 623 }
 624
 625 static void
 626 vtmmio_set_status(device_t dev, uint8_t status)
 627 {
 628         struct vtmmio_softc *sc;
 629
 630         sc = device_get_softc(dev);
 631
 632         if (status != VIRTIO_CONFIG_STATUS_RESET)
 633                 status |= vtmmio_get_status(dev);
 634
 635         vtmmio_write_config_4(sc, VIRTIO_MMIO_STATUS, status);
 636 }
 637
 638 static void
 639 vtmmio_read_dev_config(device_t dev, bus_size_t offset,
 640     void *dst, int length)
 641 {
 642         struct vtmmio_softc *sc;
 643         bus_size_t off;
 644         uint8_t *d;
 645         int size;
 646
 647         sc = device_get_softc(dev);
 648         off = VIRTIO_MMIO_CONFIG + offset;
 649
 650         for (d = dst; length > 0; d += size, off += size, length -= size) {
 651 #ifdef ALLOW_WORD_ALIGNED_ACCESS
 652                 if (length >= 4) {
 653                         size = 4;
 654                         *(uint32_t *)d = vtmmio_read_config_4(sc, off);
 655                 } else if (length >= 2) {
 656                         size = 2;
 657                         *(uint16_t *)d = vtmmio_read_config_2(sc, off);
 658                 } else
 659 #endif
 660                 {
 661                         size = 1;
 662                         *d = vtmmio_read_config_1(sc, off);
 663                 }
 664         }
 665 }
 666
 667 static void
 668 vtmmio_write_dev_config(device_t dev, bus_size_t offset,
 669     void *src, int length)
 670 {
 671         struct vtmmio_softc *sc;
 672         bus_size_t off;
 673         uint8_t *s;
 674         int size;
 675
 676         sc = device_get_softc(dev);
 677         off = VIRTIO_MMIO_CONFIG + offset;
 678
 679         for (s = src; length > 0; s += size, off += size, length -= size) {
 680 #ifdef ALLOW_WORD_ALIGNED_ACCESS
 681                 if (length >= 4) {
 682                         size = 4;
 683                         vtmmio_write_config_4(sc, off, *(uint32_t *)s);
 684                 } else if (length >= 2) {
 685                         size = 2;
 686                         vtmmio_write_config_2(sc, off, *(uint16_t *)s);
 687                 } else
 688 #endif
 689                 {
 690                         size = 1;
 691                         vtmmio_write_config_1(sc, off, *s);
 692                 }
 693         }
 694 }
 695
 696 static void
 697 vtmmio_describe_features(struct vtmmio_softc *sc, const char *msg,
 698     uint64_t features)
 699 {
 700         device_t dev, child;
 701
 702         dev = sc->dev;
 703         child = sc->vtmmio_child_dev;
 704
 705         if (device_is_attached(child) && bootverbose == 0)
 706                 return;
 707
 708         virtio_describe(dev, msg, features, sc->vtmmio_child_feat_desc);
 709 }
 710
 711 static void
 712 vtmmio_probe_and_attach_child(struct vtmmio_softc *sc)
 713 {
 714         device_t dev, child;
 715
 716         dev = sc->dev;
 717         child = sc->vtmmio_child_dev;
 718
 719         if (child == NULL)
 720                 return;
 721
 722         if (device_get_state(child) != DS_NOTPRESENT) {
 723                 return;
 724         }
 725
 726         if (device_probe(child) != 0) {
 727                 return;
 728         }
 729
 730         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
 731         if (device_attach(child) != 0) {
 732                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
 733                 vtmmio_reset(sc);
 734                 vtmmio_release_child_resources(sc);
 735                 /* Reset status for future attempt. */
 736                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
 737         } else {
 738                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
 739                 VIRTIO_ATTACH_COMPLETED(child);
 740         }
 741 }
 742
 743 static int
 744 vtmmio_reinit_virtqueue(struct vtmmio_softc *sc, int idx)
 745 {
 746         struct vtmmio_virtqueue *vqx;
 747         struct virtqueue *vq;
 748         int error;
 749         uint16_t size;
 750
 751         vqx = &sc->vtmmio_vqs[idx];
 752         vq = vqx->vtv_vq;
 753
 754         KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));
 755
 756         vtmmio_select_virtqueue(sc, idx);
 757         size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX);
 758
 759         error = virtqueue_reinit(vq, size);
 760         if (error)
 761                 return (error);
 762
 763         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN,
 764             virtqueue_paddr(vq) >> PAGE_SHIFT);
 765
 766         return (0);
 767 }
 768
 769 static void
 770 vtmmio_free_interrupts(struct vtmmio_softc *sc)
 771 {
 772
 773         if (sc->ih != NULL)
 774                 bus_teardown_intr(sc->dev, sc->res[1], sc->ih);
 775
 776         if (sc->res[1] != NULL)
 777                 bus_release_resource(sc->dev, SYS_RES_IRQ, 0, sc->res[1]);
 778 }
 779
 780 static void
 781 vtmmio_free_virtqueues(struct vtmmio_softc *sc)
 782 {
 783         struct vtmmio_virtqueue *vqx;
 784         int idx;
 785
 786         for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
 787                 vqx = &sc->vtmmio_vqs[idx];
 788
 789                 vtmmio_select_virtqueue(sc, idx);
 790                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 0);
 791
 792                 virtqueue_free(vqx->vtv_vq);
 793                 vqx->vtv_vq = NULL;
 794         }
 795
 796         free(sc->vtmmio_vqs, M_DEVBUF);
 797         sc->vtmmio_vqs = NULL;
 798         sc->vtmmio_nvqs = 0;
 799 }
 800
 801 static void
 802 vtmmio_release_child_resources(struct vtmmio_softc *sc)
 803 {
 804
 805         vtmmio_free_interrupts(sc);
 806         vtmmio_free_virtqueues(sc);
 807 }
 808
 809 static void
 810 vtmmio_reset(struct vtmmio_softc *sc)
 811 {
 812
 813         /*
 814          * Setting the status to RESET sets the host device to
 815          * the original, uninitialized state.
 816          */
 817         vtmmio_set_status(sc->dev, VIRTIO_CONFIG_STATUS_RESET);
 818 }
 819
 820 static void
 821 vtmmio_select_virtqueue(struct vtmmio_softc *sc, int idx)
 822 {
 823
 824         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx);
 825 }
 826
 827 static void
 828 vtmmio_vq_intr(void *arg)
 829 {
 830         struct vtmmio_virtqueue *vqx;
 831         struct vtmmio_softc *sc;
 832         struct virtqueue *vq;
 833         uint32_t status;
 834         int idx;
 835
 836         sc = arg;
 837
 838         status = vtmmio_read_config_4(sc, VIRTIO_MMIO_INTERRUPT_STATUS);
 839         vtmmio_write_config_4(sc, VIRTIO_MMIO_INTERRUPT_ACK, status);
 840
 841         /* The config changed */
 842         if (status & VIRTIO_MMIO_INT_CONFIG)
 843                 if (sc->vtmmio_child_dev != NULL)
 844                         VIRTIO_CONFIG_CHANGE(sc->vtmmio_child_dev);
 845
 846         /* Notify all virtqueues. */
 847         if (status & VIRTIO_MMIO_INT_VRING) {
 848                 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
 849                         vqx = &sc->vtmmio_vqs[idx];
 850                         if (vqx->vtv_no_intr == 0) {
 851                                 vq = vqx->vtv_vq;
 852                                 virtqueue_intr(vq);
 853                         }
 854                 }
 855         }
 856 }