sys/dev/virtio/mmio/virtio_mmio.c

   1 /*-
   2  * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com>
   3  * All rights reserved.
   4  *
   5  * This software was developed by SRI International and the University of
   6  * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
   7  * ("CTSRD"), as part of the DARPA CRASH research programme.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  */
  30
  31 /*
  32  * VirtIO MMIO interface.
  33  * This driver is heavily based on VirtIO PCI interface driver.
  34  */
  35
  36 /*
  37  * FDT example:
  38  *              virtio_block@1000 {
  39  *                      compatible = "virtio,mmio";
  40  *                      reg = <0x1000 0x100>;
  41  *                      interrupts = <63>;
  42  *                      interrupt-parent = <&GIC>;
  43  *              };
  44  */
  45
  46 #include <sys/cdefs.h>
  47 __FBSDID("$FreeBSD$");
  48
  49 #include <sys/param.h>
  50 #include <sys/systm.h>
  51 #include <sys/bus.h>
  52 #include <sys/kernel.h>
  53 #include <sys/module.h>
  54 #include <sys/malloc.h>
  55 #include <sys/rman.h>
  56
  57 #include <machine/bus.h>
  58 #include <machine/resource.h>
  59
  60 #include <dev/fdt/fdt_common.h>
  61 #include <dev/ofw/openfirm.h>
  62 #include <dev/ofw/ofw_bus.h>
  63 #include <dev/ofw/ofw_bus_subr.h>
  64
  65 #include <dev/virtio/virtio.h>
  66 #include <dev/virtio/virtqueue.h>
  67 #include <dev/virtio/mmio/virtio_mmio.h>
  68
  69 #include "virtio_mmio_if.h"
  70 #include "virtio_bus_if.h"
  71 #include "virtio_if.h"
  72
  73 #define PAGE_SHIFT      12
  74
  75 struct vtmmio_virtqueue {
  76         struct virtqueue        *vtv_vq;
  77         int                      vtv_no_intr;
  78 };
  79
  80 struct vtmmio_softc {
  81         device_t                        dev;
  82         device_t                        platform;
  83         struct resource                 *res[2];
  84
  85         uint64_t                        vtmmio_features;
  86         uint32_t                        vtmmio_flags;
  87
  88         /* This "bus" will only ever have one child. */
  89         device_t                        vtmmio_child_dev;
  90         struct virtio_feature_desc      *vtmmio_child_feat_desc;
  91
  92         int                             vtmmio_nvqs;
  93         struct vtmmio_virtqueue         *vtmmio_vqs;
  94         void                            *ih;
  95 };
  96
  97 static int      vtmmio_probe(device_t);
  98 static int      vtmmio_attach(device_t);
  99 static int      vtmmio_detach(device_t);
 100 static int      vtmmio_suspend(device_t);
 101 static int      vtmmio_resume(device_t);
 102 static int      vtmmio_shutdown(device_t);
 103 static void     vtmmio_driver_added(device_t, driver_t *);
 104 static void     vtmmio_child_detached(device_t, device_t);
 105 static int      vtmmio_read_ivar(device_t, device_t, int, uintptr_t *);
 106 static int      vtmmio_write_ivar(device_t, device_t, int, uintptr_t);
 107 static uint64_t vtmmio_negotiate_features(device_t, uint64_t);
 108 static int      vtmmio_with_feature(device_t, uint64_t);
 109 static int      vtmmio_alloc_virtqueues(device_t, int, int,
 110                     struct vq_alloc_info *);
 111 static int      vtmmio_setup_intr(device_t, enum intr_type);
 112 static void     vtmmio_stop(device_t);
 113 static int      vtmmio_reinit(device_t, uint64_t);
 114 static void     vtmmio_reinit_complete(device_t);
 115 static void     vtmmio_notify_virtqueue(device_t, uint16_t);
 116 static uint8_t  vtmmio_get_status(device_t);
 117 static void     vtmmio_set_status(device_t, uint8_t);
 118 static void     vtmmio_read_dev_config(device_t, bus_size_t, void *, int);
 119 static void     vtmmio_write_dev_config(device_t, bus_size_t, void *, int);
 120 static void     vtmmio_describe_features(struct vtmmio_softc *, const char *,
 121                     uint64_t);
 122 static void     vtmmio_probe_and_attach_child(struct vtmmio_softc *);
 123 static int      vtmmio_reinit_virtqueue(struct vtmmio_softc *, int);
 124 static void     vtmmio_free_interrupts(struct vtmmio_softc *);
 125 static void     vtmmio_free_virtqueues(struct vtmmio_softc *);
 126 static void     vtmmio_release_child_resources(struct vtmmio_softc *);
 127 static void     vtmmio_reset(struct vtmmio_softc *);
 128 static void     vtmmio_select_virtqueue(struct vtmmio_softc *, int);
 129 static void     vtmmio_vq_intr(void *);
 130
 131 /*
 132  * I/O port read/write wrappers.
 133  */
 134 #define vtmmio_write_config_1(sc, o, v) \
 135         bus_write_1((sc)->res[0], (o), (v)); \
 136         VIRTIO_MMIO_NOTE(sc->platform, (o))
 137 #define vtmmio_write_config_2(sc, o, v) \
 138         bus_write_2((sc)->res[0], (o), (v)); \
 139         VIRTIO_MMIO_NOTE(sc->platform, (o))
 140 #define vtmmio_write_config_4(sc, o, v) \
 141         bus_write_4((sc)->res[0], (o), (v)); \
 142         VIRTIO_MMIO_NOTE(sc->platform, (o))
 143
 144 #define vtmmio_read_config_1(sc, o) \
 145         bus_read_1((sc)->res[0], (o))
 146 #define vtmmio_read_config_2(sc, o) \
 147         bus_read_2((sc)->res[0], (o))
 148 #define vtmmio_read_config_4(sc, o) \
 149         bus_read_4((sc)->res[0], (o))
 150
 151 static device_method_t vtmmio_methods[] = {
 152         /* Device interface. */
 153         DEVMETHOD(device_probe,                   vtmmio_probe),
 154         DEVMETHOD(device_attach,                  vtmmio_attach),
 155         DEVMETHOD(device_detach,                  vtmmio_detach),
 156         DEVMETHOD(device_suspend,                 vtmmio_suspend),
 157         DEVMETHOD(device_resume,                  vtmmio_resume),
 158         DEVMETHOD(device_shutdown,                vtmmio_shutdown),
 159
 160         /* Bus interface. */
 161         DEVMETHOD(bus_driver_added,               vtmmio_driver_added),
 162         DEVMETHOD(bus_child_detached,             vtmmio_child_detached),
 163         DEVMETHOD(bus_read_ivar,                  vtmmio_read_ivar),
 164         DEVMETHOD(bus_write_ivar,                 vtmmio_write_ivar),
 165
 166         /* VirtIO bus interface. */
 167         DEVMETHOD(virtio_bus_negotiate_features,  vtmmio_negotiate_features),
 168         DEVMETHOD(virtio_bus_with_feature,        vtmmio_with_feature),
 169         DEVMETHOD(virtio_bus_alloc_virtqueues,    vtmmio_alloc_virtqueues),
 170         DEVMETHOD(virtio_bus_setup_intr,          vtmmio_setup_intr),
 171         DEVMETHOD(virtio_bus_stop,                vtmmio_stop),
 172         DEVMETHOD(virtio_bus_reinit,              vtmmio_reinit),
 173         DEVMETHOD(virtio_bus_reinit_complete,     vtmmio_reinit_complete),
 174         DEVMETHOD(virtio_bus_notify_vq,           vtmmio_notify_virtqueue),
 175         DEVMETHOD(virtio_bus_read_device_config,  vtmmio_read_dev_config),
 176         DEVMETHOD(virtio_bus_write_device_config, vtmmio_write_dev_config),
 177
 178         DEVMETHOD_END
 179 };
 180
 181 static driver_t vtmmio_driver = {
 182         "virtio_mmio",
 183         vtmmio_methods,
 184         sizeof(struct vtmmio_softc)
 185 };
 186
 187 devclass_t vtmmio_devclass;
 188
 189 DRIVER_MODULE(virtio_mmio, simplebus, vtmmio_driver, vtmmio_devclass, 0, 0);
 190 MODULE_VERSION(virtio_mmio, 1);
 191 MODULE_DEPEND(virtio_mmio, simplebus, 1, 1, 1);
 192 MODULE_DEPEND(virtio_mmio, virtio, 1, 1, 1);
 193
 194 static int
 195 vtmmio_setup_intr(device_t dev, enum intr_type type)
 196 {
 197         struct vtmmio_softc *sc;
 198         int rid;
 199         int err;
 200
 201         sc = device_get_softc(dev);
 202
 203         err = VIRTIO_MMIO_SETUP_INTR(sc->platform, sc->dev,
 204                                 vtmmio_vq_intr, sc);
 205         if (err == 0) {
 206                 /* Okay we have backend-specific interrupts */
 207                 return (0);
 208         }
 209
 210         rid = 0;
 211         sc->res[1] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 212                 RF_ACTIVE);
 213         if (!sc->res[1]) {
 214                 device_printf(dev, "Can't allocate interrupt\n");
 215                 return (ENXIO);
 216         }
 217
 218         if (bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC | INTR_MPSAFE,
 219                 NULL, vtmmio_vq_intr, sc, &sc->ih)) {
 220                 device_printf(dev, "Can't setup the interrupt\n");
 221                 return (ENXIO);
 222         }
 223
 224         return (0);
 225 }
 226
 227 static int
 228 vtmmio_probe(device_t dev)
 229 {
 230
 231         if (!ofw_bus_status_okay(dev))
 232                 return (ENXIO);
 233
 234         if (!ofw_bus_is_compatible(dev, "virtio,mmio"))
 235                 return (ENXIO);
 236
 237         device_set_desc(dev, "VirtIO MMIO adapter");
 238         return (BUS_PROBE_DEFAULT);
 239 }
 240
 241 static int
 242 vtmmio_setup_platform(struct vtmmio_softc *sc)
 243 {
 244         phandle_t platform_node;
 245         struct fdt_ic *ic;
 246         phandle_t xref;
 247         phandle_t node;
 248
 249         sc->platform = NULL;
 250
 251         if ((node = ofw_bus_get_node(sc->dev)) == -1)
 252                 return (ENXIO);
 253
 254         if (OF_searchencprop(node, "platform", &xref,
 255                 sizeof(xref)) == -1) {
 256                 return (ENXIO);
 257         }
 258
 259         platform_node = OF_node_from_xref(xref);
 260
 261         SLIST_FOREACH(ic, &fdt_ic_list_head, fdt_ics) {
 262                 if (ic->iph == platform_node) {
 263                         sc->platform = ic->dev;
 264                         break;
 265                 }
 266         }
 267
 268         if (sc->platform == NULL) {
 269                 /* No platform-specific device. Ignore it. */
 270         }
 271
 272         return (0);
 273 }
 274
 275 static int
 276 vtmmio_attach(device_t dev)
 277 {
 278         struct vtmmio_softc *sc;
 279         device_t child;
 280         int rid;
 281
 282         sc = device_get_softc(dev);
 283         sc->dev = dev;
 284
 285         vtmmio_setup_platform(sc);
 286
 287         rid = 0;
 288         sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 289                         RF_ACTIVE);
 290         if (!sc->res[0]) {
 291                 device_printf(dev, "Cannot allocate memory window.\n");
 292                 return (ENXIO);
 293         }
 294
 295         vtmmio_reset(sc);
 296
 297         /* Tell the host we've noticed this device. */
 298         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
 299
 300         if ((child = device_add_child(dev, NULL, -1)) == NULL) {
 301                 device_printf(dev, "Cannot create child device.\n");
 302                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
 303                 vtmmio_detach(dev);
 304                 return (ENOMEM);
 305         }
 306
 307         sc->vtmmio_child_dev = child;
 308         vtmmio_probe_and_attach_child(sc);
 309
 310         return (0);
 311 }
 312
 313 static int
 314 vtmmio_detach(device_t dev)
 315 {
 316         struct vtmmio_softc *sc;
 317         device_t child;
 318         int error;
 319
 320         sc = device_get_softc(dev);
 321
 322         if ((child = sc->vtmmio_child_dev) != NULL) {
 323                 error = device_delete_child(dev, child);
 324                 if (error)
 325                         return (error);
 326                 sc->vtmmio_child_dev = NULL;
 327         }
 328
 329         vtmmio_reset(sc);
 330
 331         if (sc->res[0] != NULL) {
 332                 bus_release_resource(dev, SYS_RES_MEMORY, 0,
 333                     sc->res[0]);
 334                 sc->res[0] = NULL;
 335         }
 336
 337         return (0);
 338 }
 339
 340 static int
 341 vtmmio_suspend(device_t dev)
 342 {
 343
 344         return (bus_generic_suspend(dev));
 345 }
 346
 347 static int
 348 vtmmio_resume(device_t dev)
 349 {
 350
 351         return (bus_generic_resume(dev));
 352 }
 353
 354 static int
 355 vtmmio_shutdown(device_t dev)
 356 {
 357
 358         (void) bus_generic_shutdown(dev);
 359
 360         /* Forcibly stop the host device. */
 361         vtmmio_stop(dev);
 362
 363         return (0);
 364 }
 365
 366 static void
 367 vtmmio_driver_added(device_t dev, driver_t *driver)
 368 {
 369         struct vtmmio_softc *sc;
 370
 371         sc = device_get_softc(dev);
 372
 373         vtmmio_probe_and_attach_child(sc);
 374 }
 375
 376 static void
 377 vtmmio_child_detached(device_t dev, device_t child)
 378 {
 379         struct vtmmio_softc *sc;
 380
 381         sc = device_get_softc(dev);
 382
 383         vtmmio_reset(sc);
 384         vtmmio_release_child_resources(sc);
 385 }
 386
 387 static int
 388 vtmmio_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
 389 {
 390         struct vtmmio_softc *sc;
 391
 392         sc = device_get_softc(dev);
 393
 394         if (sc->vtmmio_child_dev != child)
 395                 return (ENOENT);
 396
 397         switch (index) {
 398         case VIRTIO_IVAR_DEVTYPE:
 399         case VIRTIO_IVAR_SUBDEVICE:
 400                 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_DEVICE_ID);
 401                 break;
 402         case VIRTIO_IVAR_VENDOR:
 403                 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_VENDOR_ID);
 404                 break;
 405         default:
 406                 return (ENOENT);
 407         }
 408
 409         return (0);
 410 }
 411
 412 static int
 413 vtmmio_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
 414 {
 415         struct vtmmio_softc *sc;
 416
 417         sc = device_get_softc(dev);
 418
 419         if (sc->vtmmio_child_dev != child)
 420                 return (ENOENT);
 421
 422         switch (index) {
 423         case VIRTIO_IVAR_FEATURE_DESC:
 424                 sc->vtmmio_child_feat_desc = (void *) value;
 425                 break;
 426         default:
 427                 return (ENOENT);
 428         }
 429
 430         return (0);
 431 }
 432
 433 static uint64_t
 434 vtmmio_negotiate_features(device_t dev, uint64_t child_features)
 435 {
 436         struct vtmmio_softc *sc;
 437         uint64_t host_features, features;
 438
 439         sc = device_get_softc(dev);
 440
 441         host_features = vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES);
 442         vtmmio_describe_features(sc, "host", host_features);
 443
 444         /*
 445          * Limit negotiated features to what the driver, virtqueue, and
 446          * host all support.
 447          */
 448         features = host_features & child_features;
 449         features = virtqueue_filter_features(features);
 450         sc->vtmmio_features = features;
 451
 452         vtmmio_describe_features(sc, "negotiated", features);
 453         vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features);
 454
 455         return (features);
 456 }
 457
 458 static int
 459 vtmmio_with_feature(device_t dev, uint64_t feature)
 460 {
 461         struct vtmmio_softc *sc;
 462
 463         sc = device_get_softc(dev);
 464
 465         return ((sc->vtmmio_features & feature) != 0);
 466 }
 467
 468 static int
 469 vtmmio_alloc_virtqueues(device_t dev, int flags, int nvqs,
 470     struct vq_alloc_info *vq_info)
 471 {
 472         struct vtmmio_virtqueue *vqx;
 473         struct vq_alloc_info *info;
 474         struct vtmmio_softc *sc;
 475         struct virtqueue *vq;
 476         int idx, error;
 477         uint16_t size;
 478
 479         sc = device_get_softc(dev);
 480
 481         if (sc->vtmmio_nvqs != 0)
 482                 return (EALREADY);
 483         if (nvqs <= 0)
 484                 return (EINVAL);
 485
 486         sc->vtmmio_vqs = malloc(nvqs * sizeof(struct vtmmio_virtqueue),
 487             M_DEVBUF, M_NOWAIT | M_ZERO);
 488         if (sc->vtmmio_vqs == NULL)
 489                 return (ENOMEM);
 490
 491         for (idx = 0; idx < nvqs; idx++) {
 492                 vqx = &sc->vtmmio_vqs[idx];
 493                 info = &vq_info[idx];
 494
 495                 vtmmio_select_virtqueue(sc, idx);
 496                 size = vtmmio_read_config_2(sc, VIRTIO_MMIO_QUEUE_NUM);
 497
 498                 error = virtqueue_alloc(dev, idx, size,
 499                     VIRTIO_MMIO_VRING_ALIGN, 0xFFFFFFFFUL, info, &vq);
 500                 if (error) {
 501                         device_printf(dev,
 502                             "cannot allocate virtqueue %d: %d\n",
 503                             idx, error);
 504                         break;
 505                 }
 506 #if 0
 507                 device_printf(dev, "virtqueue paddr 0x%08lx\n",
 508                                 (uint64_t)virtqueue_paddr(vq));
 509 #endif
 510                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN,
 511                         virtqueue_paddr(vq) >> PAGE_SHIFT);
 512
 513                 vqx->vtv_vq = *info->vqai_vq = vq;
 514                 vqx->vtv_no_intr = info->vqai_intr == NULL;
 515
 516                 sc->vtmmio_nvqs++;
 517         }
 518
 519         if (error)
 520                 vtmmio_free_virtqueues(sc);
 521
 522         return (error);
 523 }
 524
 525 static void
 526 vtmmio_stop(device_t dev)
 527 {
 528
 529         vtmmio_reset(device_get_softc(dev));
 530 }
 531
 532 static int
 533 vtmmio_reinit(device_t dev, uint64_t features)
 534 {
 535         struct vtmmio_softc *sc;
 536         int idx, error;
 537
 538         sc = device_get_softc(dev);
 539
 540         if (vtmmio_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
 541                 vtmmio_stop(dev);
 542
 543         /*
 544          * Quickly drive the status through ACK and DRIVER. The device
 545          * does not become usable again until vtmmio_reinit_complete().
 546          */
 547         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
 548         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
 549
 550         vtmmio_negotiate_features(dev, features);
 551
 552         for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
 553                 error = vtmmio_reinit_virtqueue(sc, idx);
 554                 if (error)
 555                         return (error);
 556         }
 557
 558         return (0);
 559 }
 560
 561 static void
 562 vtmmio_reinit_complete(device_t dev)
 563 {
 564
 565         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
 566 }
 567
 568 static void
 569 vtmmio_notify_virtqueue(device_t dev, uint16_t queue)
 570 {
 571         struct vtmmio_softc *sc;
 572
 573         sc = device_get_softc(dev);
 574
 575         vtmmio_write_config_2(sc, VIRTIO_MMIO_QUEUE_NOTIFY, queue);
 576 }
 577
 578 static uint8_t
 579 vtmmio_get_status(device_t dev)
 580 {
 581         struct vtmmio_softc *sc;
 582
 583         sc = device_get_softc(dev);
 584
 585         return (vtmmio_read_config_1(sc, VIRTIO_MMIO_STATUS));
 586 }
 587
 588 static void
 589 vtmmio_set_status(device_t dev, uint8_t status)
 590 {
 591         struct vtmmio_softc *sc;
 592
 593         sc = device_get_softc(dev);
 594
 595         if (status != VIRTIO_CONFIG_STATUS_RESET)
 596                 status |= vtmmio_get_status(dev);
 597
 598         vtmmio_write_config_1(sc, VIRTIO_MMIO_STATUS, status);
 599 }
 600
 601 static void
 602 vtmmio_read_dev_config(device_t dev, bus_size_t offset,
 603     void *dst, int length)
 604 {
 605         struct vtmmio_softc *sc;
 606         bus_size_t off;
 607         uint8_t *d;
 608         int size;
 609
 610         sc = device_get_softc(dev);
 611         off = VIRTIO_MMIO_CONFIG + offset;
 612
 613         for (d = dst; length > 0; d += size, off += size, length -= size) {
 614                 if (length >= 4) {
 615                         size = 4;
 616                         *(uint32_t *)d = vtmmio_read_config_4(sc, off);
 617                 } else if (length >= 2) {
 618                         size = 2;
 619                         *(uint16_t *)d = vtmmio_read_config_2(sc, off);
 620                 } else {
 621                         size = 1;
 622                         *d = vtmmio_read_config_1(sc, off);
 623                 }
 624         }
 625 }
 626
 627 static void
 628 vtmmio_write_dev_config(device_t dev, bus_size_t offset,
 629     void *src, int length)
 630 {
 631         struct vtmmio_softc *sc;
 632         bus_size_t off;
 633         uint8_t *s;
 634         int size;
 635
 636         sc = device_get_softc(dev);
 637         off = VIRTIO_MMIO_CONFIG + offset;
 638
 639         for (s = src; length > 0; s += size, off += size, length -= size) {
 640                 if (length >= 4) {
 641                         size = 4;
 642                         vtmmio_write_config_4(sc, off, *(uint32_t *)s);
 643                 } else if (length >= 2) {
 644                         size = 2;
 645                         vtmmio_write_config_2(sc, off, *(uint16_t *)s);
 646                 } else {
 647                         size = 1;
 648                         vtmmio_write_config_1(sc, off, *s);
 649                 }
 650         }
 651 }
 652
 653 static void
 654 vtmmio_describe_features(struct vtmmio_softc *sc, const char *msg,
 655     uint64_t features)
 656 {
 657         device_t dev, child;
 658
 659         dev = sc->dev;
 660         child = sc->vtmmio_child_dev;
 661
 662         if (device_is_attached(child) && bootverbose == 0)
 663                 return;
 664
 665         virtio_describe(dev, msg, features, sc->vtmmio_child_feat_desc);
 666 }
 667
 668 static void
 669 vtmmio_probe_and_attach_child(struct vtmmio_softc *sc)
 670 {
 671         device_t dev, child;
 672
 673         dev = sc->dev;
 674         child = sc->vtmmio_child_dev;
 675
 676         if (child == NULL)
 677                 return;
 678
 679         if (device_get_state(child) != DS_NOTPRESENT) {
 680                 return;
 681         }
 682
 683         if (device_probe(child) != 0) {
 684                 return;
 685         }
 686
 687         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
 688         if (device_attach(child) != 0) {
 689                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
 690                 vtmmio_reset(sc);
 691                 vtmmio_release_child_resources(sc);
 692                 /* Reset status for future attempt. */
 693                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
 694         } else {
 695                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
 696                 VIRTIO_ATTACH_COMPLETED(child);
 697         }
 698 }
 699
 700 static int
 701 vtmmio_reinit_virtqueue(struct vtmmio_softc *sc, int idx)
 702 {
 703         struct vtmmio_virtqueue *vqx;
 704         struct virtqueue *vq;
 705         int error;
 706         uint16_t size;
 707
 708         vqx = &sc->vtmmio_vqs[idx];
 709         vq = vqx->vtv_vq;
 710
 711         KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));
 712
 713         vtmmio_select_virtqueue(sc, idx);
 714         size = vtmmio_read_config_2(sc, VIRTIO_MMIO_QUEUE_NUM);
 715
 716         error = virtqueue_reinit(vq, size);
 717         if (error)
 718                 return (error);
 719
 720         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN,
 721             virtqueue_paddr(vq) >> PAGE_SHIFT);
 722
 723         return (0);
 724 }
 725
 726 static void
 727 vtmmio_free_interrupts(struct vtmmio_softc *sc)
 728 {
 729
 730         if (sc->ih != NULL)
 731                 bus_teardown_intr(sc->dev, sc->res[1], sc->ih);
 732
 733         if (sc->res[1] != NULL)
 734                 bus_release_resource(sc->dev, SYS_RES_IRQ, 0, sc->res[1]);
 735 }
 736
 737 static void
 738 vtmmio_free_virtqueues(struct vtmmio_softc *sc)
 739 {
 740         struct vtmmio_virtqueue *vqx;
 741         int idx;
 742
 743         for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
 744                 vqx = &sc->vtmmio_vqs[idx];
 745
 746                 vtmmio_select_virtqueue(sc, idx);
 747                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 0);
 748
 749                 virtqueue_free(vqx->vtv_vq);
 750                 vqx->vtv_vq = NULL;
 751         }
 752
 753         free(sc->vtmmio_vqs, M_DEVBUF);
 754         sc->vtmmio_vqs = NULL;
 755         sc->vtmmio_nvqs = 0;
 756 }
 757
 758 static void
 759 vtmmio_release_child_resources(struct vtmmio_softc *sc)
 760 {
 761
 762         vtmmio_free_interrupts(sc);
 763         vtmmio_free_virtqueues(sc);
 764 }
 765
 766 static void
 767 vtmmio_reset(struct vtmmio_softc *sc)
 768 {
 769
 770         /*
 771          * Setting the status to RESET sets the host device to
 772          * the original, uninitialized state.
 773          */
 774         vtmmio_set_status(sc->dev, VIRTIO_CONFIG_STATUS_RESET);
 775 }
 776
 777 static void
 778 vtmmio_select_virtqueue(struct vtmmio_softc *sc, int idx)
 779 {
 780
 781         vtmmio_write_config_2(sc, VIRTIO_MMIO_QUEUE_SEL, idx);
 782 }
 783
 784 static void
 785 vtmmio_vq_intr(void *arg)
 786 {
 787         struct vtmmio_virtqueue *vqx;
 788         struct vtmmio_softc *sc;
 789         struct virtqueue *vq;
 790         int idx;
 791
 792         sc = arg;
 793
 794         /* Notify all virtqueues. */
 795         for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
 796                 vqx = &sc->vtmmio_vqs[idx];
 797                 vq = vqx->vtv_vq;
 798                 virtqueue_intr(vq);
 799         };
 800 }