sys/dev/virtio/mmio/virtio_mmio.c

   1 /*-
   2  * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com>
   3  * Copyright (c) 2014 The FreeBSD Foundation
   4  * All rights reserved.
   5  *
   6  * This software was developed by SRI International and the University of
   7  * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
   8  * ("CTSRD"), as part of the DARPA CRASH research programme.
   9  *
  10  * Portions of this software were developed by Andrew Turner
  11  * under sponsorship from the FreeBSD Foundation.
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  * 1. Redistributions of source code must retain the above copyright
  17  *    notice, this list of conditions and the following disclaimer.
  18  * 2. Redistributions in binary form must reproduce the above copyright
  19  *    notice, this list of conditions and the following disclaimer in the
  20  *    documentation and/or other materials provided with the distribution.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34
  35 /*
  36  * VirtIO MMIO interface.
  37  * This driver is heavily based on VirtIO PCI interface driver.
  38  */
  39
  40 #include <sys/cdefs.h>
  41 __FBSDID("$FreeBSD$");
  42
  43 #include <sys/param.h>
  44 #include <sys/systm.h>
  45 #include <sys/bus.h>
  46 #include <sys/kernel.h>
  47 #include <sys/module.h>
  48 #include <sys/malloc.h>
  49 #include <sys/rman.h>
  50
  51 #include <machine/bus.h>
  52 #include <machine/resource.h>
  53
  54 #include <dev/virtio/virtio.h>
  55 #include <dev/virtio/virtqueue.h>
  56 #include <dev/virtio/mmio/virtio_mmio.h>
  57
  58 #include "virtio_mmio_if.h"
  59 #include "virtio_bus_if.h"
  60 #include "virtio_if.h"
  61
  62 #define PAGE_SHIFT      12
  63
  64 struct vtmmio_virtqueue {
  65         struct virtqueue        *vtv_vq;
  66         int                      vtv_no_intr;
  67 };
  68
  69 static int      vtmmio_detach(device_t);
  70 static int      vtmmio_suspend(device_t);
  71 static int      vtmmio_resume(device_t);
  72 static int      vtmmio_shutdown(device_t);
  73 static void     vtmmio_driver_added(device_t, driver_t *);
  74 static void     vtmmio_child_detached(device_t, device_t);
  75 static int      vtmmio_read_ivar(device_t, device_t, int, uintptr_t *);
  76 static int      vtmmio_write_ivar(device_t, device_t, int, uintptr_t);
  77 static uint64_t vtmmio_negotiate_features(device_t, uint64_t);
  78 static int      vtmmio_with_feature(device_t, uint64_t);
  79 static int      vtmmio_alloc_virtqueues(device_t, int, int,
  80                     struct vq_alloc_info *);
  81 static int      vtmmio_setup_intr(device_t, enum intr_type);
  82 static void     vtmmio_stop(device_t);
  83 static void     vtmmio_poll(device_t);
  84 static int      vtmmio_reinit(device_t, uint64_t);
  85 static void     vtmmio_reinit_complete(device_t);
  86 static void     vtmmio_notify_virtqueue(device_t, uint16_t);
  87 static uint8_t  vtmmio_get_status(device_t);
  88 static void     vtmmio_set_status(device_t, uint8_t);
  89 static void     vtmmio_read_dev_config(device_t, bus_size_t, void *, int);
  90 static void     vtmmio_write_dev_config(device_t, bus_size_t, void *, int);
  91 static void     vtmmio_describe_features(struct vtmmio_softc *, const char *,
  92                     uint64_t);
  93 static void     vtmmio_probe_and_attach_child(struct vtmmio_softc *);
  94 static int      vtmmio_reinit_virtqueue(struct vtmmio_softc *, int);
  95 static void     vtmmio_free_interrupts(struct vtmmio_softc *);
  96 static void     vtmmio_free_virtqueues(struct vtmmio_softc *);
  97 static void     vtmmio_release_child_resources(struct vtmmio_softc *);
  98 static void     vtmmio_reset(struct vtmmio_softc *);
  99 static void     vtmmio_select_virtqueue(struct vtmmio_softc *, int);
 100 static void     vtmmio_vq_intr(void *);
 101
 102 /*
 103  * I/O port read/write wrappers.
 104  */
 105 #define vtmmio_write_config_1(sc, o, v)                         \
 106 do {                                                            \
 107         if (sc->platform != NULL)                               \
 108                 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v));   \
 109         bus_write_1((sc)->res[0], (o), (v));                    \
 110         if (sc->platform != NULL)                               \
 111                 VIRTIO_MMIO_NOTE(sc->platform, (o), (v));       \
 112 } while (0)
 113 #define vtmmio_write_config_2(sc, o, v)                         \
 114 do {                                                            \
 115         if (sc->platform != NULL)                               \
 116                 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v));   \
 117         bus_write_2((sc)->res[0], (o), (v));                    \
 118         if (sc->platform != NULL)                               \
 119                 VIRTIO_MMIO_NOTE(sc->platform, (o), (v));       \
 120 } while (0)
 121 #define vtmmio_write_config_4(sc, o, v)                         \
 122 do {                                                            \
 123         if (sc->platform != NULL)                               \
 124                 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v));   \
 125         bus_write_4((sc)->res[0], (o), (v));                    \
 126         if (sc->platform != NULL)                               \
 127                 VIRTIO_MMIO_NOTE(sc->platform, (o), (v));       \
 128 } while (0)
 129
 130 #define vtmmio_read_config_1(sc, o) \
 131         bus_read_1((sc)->res[0], (o))
 132 #define vtmmio_read_config_2(sc, o) \
 133         bus_read_2((sc)->res[0], (o))
 134 #define vtmmio_read_config_4(sc, o) \
 135         bus_read_4((sc)->res[0], (o))
 136
 137 static device_method_t vtmmio_methods[] = {
 138         /* Device interface. */
 139         DEVMETHOD(device_attach,                  vtmmio_attach),
 140         DEVMETHOD(device_detach,                  vtmmio_detach),
 141         DEVMETHOD(device_suspend,                 vtmmio_suspend),
 142         DEVMETHOD(device_resume,                  vtmmio_resume),
 143         DEVMETHOD(device_shutdown,                vtmmio_shutdown),
 144
 145         /* Bus interface. */
 146         DEVMETHOD(bus_driver_added,               vtmmio_driver_added),
 147         DEVMETHOD(bus_child_detached,             vtmmio_child_detached),
 148         DEVMETHOD(bus_child_pnpinfo_str,          virtio_child_pnpinfo_str),
 149         DEVMETHOD(bus_read_ivar,                  vtmmio_read_ivar),
 150         DEVMETHOD(bus_write_ivar,                 vtmmio_write_ivar),
 151
 152         /* VirtIO bus interface. */
 153         DEVMETHOD(virtio_bus_negotiate_features,  vtmmio_negotiate_features),
 154         DEVMETHOD(virtio_bus_with_feature,        vtmmio_with_feature),
 155         DEVMETHOD(virtio_bus_alloc_virtqueues,    vtmmio_alloc_virtqueues),
 156         DEVMETHOD(virtio_bus_setup_intr,          vtmmio_setup_intr),
 157         DEVMETHOD(virtio_bus_stop,                vtmmio_stop),
 158         DEVMETHOD(virtio_bus_poll,                vtmmio_poll),
 159         DEVMETHOD(virtio_bus_reinit,              vtmmio_reinit),
 160         DEVMETHOD(virtio_bus_reinit_complete,     vtmmio_reinit_complete),
 161         DEVMETHOD(virtio_bus_notify_vq,           vtmmio_notify_virtqueue),
 162         DEVMETHOD(virtio_bus_read_device_config,  vtmmio_read_dev_config),
 163         DEVMETHOD(virtio_bus_write_device_config, vtmmio_write_dev_config),
 164
 165         DEVMETHOD_END
 166 };
 167
 168 DEFINE_CLASS_0(virtio_mmio, vtmmio_driver, vtmmio_methods,
 169     sizeof(struct vtmmio_softc));
 170
 171 MODULE_VERSION(virtio_mmio, 1);
 172
 173 static int
 174 vtmmio_setup_intr(device_t dev, enum intr_type type)
 175 {
 176         struct vtmmio_softc *sc;
 177         int rid;
 178         int err;
 179
 180         sc = device_get_softc(dev);
 181
 182         if (sc->platform != NULL) {
 183                 err = VIRTIO_MMIO_SETUP_INTR(sc->platform, sc->dev,
 184                                         vtmmio_vq_intr, sc);
 185                 if (err == 0) {
 186                         /* Okay we have backend-specific interrupts */
 187                         return (0);
 188                 }
 189         }
 190
 191         rid = 0;
 192         sc->res[1] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 193                 RF_ACTIVE);
 194         if (!sc->res[1]) {
 195                 device_printf(dev, "Can't allocate interrupt\n");
 196                 return (ENXIO);
 197         }
 198
 199         if (bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC | INTR_MPSAFE,
 200                 NULL, vtmmio_vq_intr, sc, &sc->ih)) {
 201                 device_printf(dev, "Can't setup the interrupt\n");
 202                 return (ENXIO);
 203         }
 204
 205         return (0);
 206 }
 207
 208 int
 209 vtmmio_attach(device_t dev)
 210 {
 211         struct vtmmio_softc *sc;
 212         device_t child;
 213         int rid;
 214
 215         sc = device_get_softc(dev);
 216         sc->dev = dev;
 217
 218         rid = 0;
 219         sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 220                         RF_ACTIVE);
 221         if (!sc->res[0]) {
 222                 device_printf(dev, "Cannot allocate memory window.\n");
 223                 return (ENXIO);
 224         }
 225
 226         vtmmio_reset(sc);
 227
 228         /* Tell the host we've noticed this device. */
 229         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
 230
 231         if ((child = device_add_child(dev, NULL, -1)) == NULL) {
 232                 device_printf(dev, "Cannot create child device.\n");
 233                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
 234                 vtmmio_detach(dev);
 235                 return (ENOMEM);
 236         }
 237
 238         sc->vtmmio_child_dev = child;
 239         vtmmio_probe_and_attach_child(sc);
 240
 241         return (0);
 242 }
 243
 244 static int
 245 vtmmio_detach(device_t dev)
 246 {
 247         struct vtmmio_softc *sc;
 248         device_t child;
 249         int error;
 250
 251         sc = device_get_softc(dev);
 252
 253         if ((child = sc->vtmmio_child_dev) != NULL) {
 254                 error = device_delete_child(dev, child);
 255                 if (error)
 256                         return (error);
 257                 sc->vtmmio_child_dev = NULL;
 258         }
 259
 260         vtmmio_reset(sc);
 261
 262         if (sc->res[0] != NULL) {
 263                 bus_release_resource(dev, SYS_RES_MEMORY, 0,
 264                     sc->res[0]);
 265                 sc->res[0] = NULL;
 266         }
 267
 268         return (0);
 269 }
 270
 271 static int
 272 vtmmio_suspend(device_t dev)
 273 {
 274
 275         return (bus_generic_suspend(dev));
 276 }
 277
 278 static int
 279 vtmmio_resume(device_t dev)
 280 {
 281
 282         return (bus_generic_resume(dev));
 283 }
 284
 285 static int
 286 vtmmio_shutdown(device_t dev)
 287 {
 288
 289         (void) bus_generic_shutdown(dev);
 290
 291         /* Forcibly stop the host device. */
 292         vtmmio_stop(dev);
 293
 294         return (0);
 295 }
 296
 297 static void
 298 vtmmio_driver_added(device_t dev, driver_t *driver)
 299 {
 300         struct vtmmio_softc *sc;
 301
 302         sc = device_get_softc(dev);
 303
 304         vtmmio_probe_and_attach_child(sc);
 305 }
 306
 307 static void
 308 vtmmio_child_detached(device_t dev, device_t child)
 309 {
 310         struct vtmmio_softc *sc;
 311
 312         sc = device_get_softc(dev);
 313
 314         vtmmio_reset(sc);
 315         vtmmio_release_child_resources(sc);
 316 }
 317
 318 static int
 319 vtmmio_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
 320 {
 321         struct vtmmio_softc *sc;
 322
 323         sc = device_get_softc(dev);
 324
 325         if (sc->vtmmio_child_dev != child)
 326                 return (ENOENT);
 327
 328         switch (index) {
 329         case VIRTIO_IVAR_DEVTYPE:
 330         case VIRTIO_IVAR_SUBDEVICE:
 331                 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_DEVICE_ID);
 332                 break;
 333         case VIRTIO_IVAR_VENDOR:
 334                 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_VENDOR_ID);
 335                 break;
 336         case VIRTIO_IVAR_SUBVENDOR:
 337         case VIRTIO_IVAR_DEVICE:
 338                 /*
 339                  * Dummy value for fields not present in this bus.  Used by
 340                  * bus-agnostic virtio_child_pnpinfo_str.
 341                  */
 342                 *result = 0;
 343                 break;
 344         default:
 345                 return (ENOENT);
 346         }
 347
 348         return (0);
 349 }
 350
 351 static int
 352 vtmmio_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
 353 {
 354         struct vtmmio_softc *sc;
 355
 356         sc = device_get_softc(dev);
 357
 358         if (sc->vtmmio_child_dev != child)
 359                 return (ENOENT);
 360
 361         switch (index) {
 362         case VIRTIO_IVAR_FEATURE_DESC:
 363                 sc->vtmmio_child_feat_desc = (void *) value;
 364                 break;
 365         default:
 366                 return (ENOENT);
 367         }
 368
 369         return (0);
 370 }
 371
 372 static uint64_t
 373 vtmmio_negotiate_features(device_t dev, uint64_t child_features)
 374 {
 375         struct vtmmio_softc *sc;
 376         uint64_t host_features, features;
 377
 378         sc = device_get_softc(dev);
 379
 380         host_features = vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES);
 381         vtmmio_describe_features(sc, "host", host_features);
 382
 383         /*
 384          * Limit negotiated features to what the driver, virtqueue, and
 385          * host all support.
 386          */
 387         features = host_features & child_features;
 388         features = virtqueue_filter_features(features);
 389         sc->vtmmio_features = features;
 390
 391         vtmmio_describe_features(sc, "negotiated", features);
 392         vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features);
 393
 394         return (features);
 395 }
 396
 397 static int
 398 vtmmio_with_feature(device_t dev, uint64_t feature)
 399 {
 400         struct vtmmio_softc *sc;
 401
 402         sc = device_get_softc(dev);
 403
 404         return ((sc->vtmmio_features & feature) != 0);
 405 }
 406
 407 static int
 408 vtmmio_alloc_virtqueues(device_t dev, int flags, int nvqs,
 409     struct vq_alloc_info *vq_info)
 410 {
 411         struct vtmmio_virtqueue *vqx;
 412         struct vq_alloc_info *info;
 413         struct vtmmio_softc *sc;
 414         struct virtqueue *vq;
 415         uint32_t size;
 416         int idx, error;
 417
 418         sc = device_get_softc(dev);
 419
 420         if (sc->vtmmio_nvqs != 0)
 421                 return (EALREADY);
 422         if (nvqs <= 0)
 423                 return (EINVAL);
 424
 425         sc->vtmmio_vqs = malloc(nvqs * sizeof(struct vtmmio_virtqueue),
 426             M_DEVBUF, M_NOWAIT | M_ZERO);
 427         if (sc->vtmmio_vqs == NULL)
 428                 return (ENOMEM);
 429
 430         vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE,
 431             (1 << PAGE_SHIFT));
 432
 433         for (idx = 0; idx < nvqs; idx++) {
 434                 vqx = &sc->vtmmio_vqs[idx];
 435                 info = &vq_info[idx];
 436
 437                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx);
 438
 439                 vtmmio_select_virtqueue(sc, idx);
 440                 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX);
 441
 442                 error = virtqueue_alloc(dev, idx, size,
 443                     VIRTIO_MMIO_VRING_ALIGN, ~(vm_paddr_t)0, info, &vq);
 444                 if (error) {
 445                         device_printf(dev,
 446                             "cannot allocate virtqueue %d: %d\n",
 447                             idx, error);
 448                         break;
 449                 }
 450
 451                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size);
 452                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN,
 453                     VIRTIO_MMIO_VRING_ALIGN);
 454 #if 0
 455                 device_printf(dev, "virtqueue paddr 0x%08lx\n",
 456                     (uint64_t)virtqueue_paddr(vq));
 457 #endif
 458                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN,
 459                     virtqueue_paddr(vq) >> PAGE_SHIFT);
 460
 461                 vqx->vtv_vq = *info->vqai_vq = vq;
 462                 vqx->vtv_no_intr = info->vqai_intr == NULL;
 463
 464                 sc->vtmmio_nvqs++;
 465         }
 466
 467         if (error)
 468                 vtmmio_free_virtqueues(sc);
 469
 470         return (error);
 471 }
 472
 473 static void
 474 vtmmio_stop(device_t dev)
 475 {
 476
 477         vtmmio_reset(device_get_softc(dev));
 478 }
 479
 480 static void
 481 vtmmio_poll(device_t dev)
 482 {
 483         struct vtmmio_softc *sc;
 484
 485         sc = device_get_softc(dev);
 486
 487         if (sc->platform != NULL)
 488                 VIRTIO_MMIO_POLL(sc->platform);
 489 }
 490
 491 static int
 492 vtmmio_reinit(device_t dev, uint64_t features)
 493 {
 494         struct vtmmio_softc *sc;
 495         int idx, error;
 496
 497         sc = device_get_softc(dev);
 498
 499         if (vtmmio_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
 500                 vtmmio_stop(dev);
 501
 502         /*
 503          * Quickly drive the status through ACK and DRIVER. The device
 504          * does not become usable again until vtmmio_reinit_complete().
 505          */
 506         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
 507         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
 508
 509         vtmmio_negotiate_features(dev, features);
 510
 511         vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE,
 512             (1 << PAGE_SHIFT));
 513
 514         for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
 515                 error = vtmmio_reinit_virtqueue(sc, idx);
 516                 if (error)
 517                         return (error);
 518         }
 519
 520         return (0);
 521 }
 522
 523 static void
 524 vtmmio_reinit_complete(device_t dev)
 525 {
 526
 527         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
 528 }
 529
 530 static void
 531 vtmmio_notify_virtqueue(device_t dev, uint16_t queue)
 532 {
 533         struct vtmmio_softc *sc;
 534
 535         sc = device_get_softc(dev);
 536
 537         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NOTIFY, queue);
 538 }
 539
 540 static uint8_t
 541 vtmmio_get_status(device_t dev)
 542 {
 543         struct vtmmio_softc *sc;
 544
 545         sc = device_get_softc(dev);
 546
 547         return (vtmmio_read_config_4(sc, VIRTIO_MMIO_STATUS));
 548 }
 549
 550 static void
 551 vtmmio_set_status(device_t dev, uint8_t status)
 552 {
 553         struct vtmmio_softc *sc;
 554
 555         sc = device_get_softc(dev);
 556
 557         if (status != VIRTIO_CONFIG_STATUS_RESET)
 558                 status |= vtmmio_get_status(dev);
 559
 560         vtmmio_write_config_4(sc, VIRTIO_MMIO_STATUS, status);
 561 }
 562
 563 static void
 564 vtmmio_read_dev_config(device_t dev, bus_size_t offset,
 565     void *dst, int length)
 566 {
 567         struct vtmmio_softc *sc;
 568         bus_size_t off;
 569         uint8_t *d;
 570         int size;
 571
 572         sc = device_get_softc(dev);
 573         off = VIRTIO_MMIO_CONFIG + offset;
 574
 575         for (d = dst; length > 0; d += size, off += size, length -= size) {
 576 #ifdef ALLOW_WORD_ALIGNED_ACCESS
 577                 if (length >= 4) {
 578                         size = 4;
 579                         *(uint32_t *)d = vtmmio_read_config_4(sc, off);
 580                 } else if (length >= 2) {
 581                         size = 2;
 582                         *(uint16_t *)d = vtmmio_read_config_2(sc, off);
 583                 } else
 584 #endif
 585                 {
 586                         size = 1;
 587                         *d = vtmmio_read_config_1(sc, off);
 588                 }
 589         }
 590 }
 591
 592 static void
 593 vtmmio_write_dev_config(device_t dev, bus_size_t offset,
 594     void *src, int length)
 595 {
 596         struct vtmmio_softc *sc;
 597         bus_size_t off;
 598         uint8_t *s;
 599         int size;
 600
 601         sc = device_get_softc(dev);
 602         off = VIRTIO_MMIO_CONFIG + offset;
 603
 604         for (s = src; length > 0; s += size, off += size, length -= size) {
 605 #ifdef ALLOW_WORD_ALIGNED_ACCESS
 606                 if (length >= 4) {
 607                         size = 4;
 608                         vtmmio_write_config_4(sc, off, *(uint32_t *)s);
 609                 } else if (length >= 2) {
 610                         size = 2;
 611                         vtmmio_write_config_2(sc, off, *(uint16_t *)s);
 612                 } else
 613 #endif
 614                 {
 615                         size = 1;
 616                         vtmmio_write_config_1(sc, off, *s);
 617                 }
 618         }
 619 }
 620
 621 static void
 622 vtmmio_describe_features(struct vtmmio_softc *sc, const char *msg,
 623     uint64_t features)
 624 {
 625         device_t dev, child;
 626
 627         dev = sc->dev;
 628         child = sc->vtmmio_child_dev;
 629
 630         if (device_is_attached(child) || bootverbose == 0)
 631                 return;
 632
 633         virtio_describe(dev, msg, features, sc->vtmmio_child_feat_desc);
 634 }
 635
 636 static void
 637 vtmmio_probe_and_attach_child(struct vtmmio_softc *sc)
 638 {
 639         device_t dev, child;
 640
 641         dev = sc->dev;
 642         child = sc->vtmmio_child_dev;
 643
 644         if (child == NULL)
 645                 return;
 646
 647         if (device_get_state(child) != DS_NOTPRESENT) {
 648                 return;
 649         }
 650
 651         if (device_probe(child) != 0) {
 652                 return;
 653         }
 654
 655         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
 656         if (device_attach(child) != 0) {
 657                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
 658                 vtmmio_reset(sc);
 659                 vtmmio_release_child_resources(sc);
 660                 /* Reset status for future attempt. */
 661                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
 662         } else {
 663                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
 664                 VIRTIO_ATTACH_COMPLETED(child);
 665         }
 666 }
 667
 668 static int
 669 vtmmio_reinit_virtqueue(struct vtmmio_softc *sc, int idx)
 670 {
 671         struct vtmmio_virtqueue *vqx;
 672         struct virtqueue *vq;
 673         int error;
 674         uint16_t size;
 675
 676         vqx = &sc->vtmmio_vqs[idx];
 677         vq = vqx->vtv_vq;
 678
 679         KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));
 680
 681         vtmmio_select_virtqueue(sc, idx);
 682         size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX);
 683
 684         error = virtqueue_reinit(vq, size);
 685         if (error)
 686                 return (error);
 687
 688         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size);
 689         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN,
 690             VIRTIO_MMIO_VRING_ALIGN);
 691 #if 0
 692         device_printf(sc->dev, "virtqueue paddr 0x%08lx\n",
 693             (uint64_t)virtqueue_paddr(vq));
 694 #endif
 695         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN,
 696             virtqueue_paddr(vq) >> PAGE_SHIFT);
 697
 698         return (0);
 699 }
 700
 701 static void
 702 vtmmio_free_interrupts(struct vtmmio_softc *sc)
 703 {
 704
 705         if (sc->ih != NULL)
 706                 bus_teardown_intr(sc->dev, sc->res[1], sc->ih);
 707
 708         if (sc->res[1] != NULL)
 709                 bus_release_resource(sc->dev, SYS_RES_IRQ, 0, sc->res[1]);
 710 }
 711
 712 static void
 713 vtmmio_free_virtqueues(struct vtmmio_softc *sc)
 714 {
 715         struct vtmmio_virtqueue *vqx;
 716         int idx;
 717
 718         for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
 719                 vqx = &sc->vtmmio_vqs[idx];
 720
 721                 vtmmio_select_virtqueue(sc, idx);
 722                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 0);
 723
 724                 virtqueue_free(vqx->vtv_vq);
 725                 vqx->vtv_vq = NULL;
 726         }
 727
 728         free(sc->vtmmio_vqs, M_DEVBUF);
 729         sc->vtmmio_vqs = NULL;
 730         sc->vtmmio_nvqs = 0;
 731 }
 732
 733 static void
 734 vtmmio_release_child_resources(struct vtmmio_softc *sc)
 735 {
 736
 737         vtmmio_free_interrupts(sc);
 738         vtmmio_free_virtqueues(sc);
 739 }
 740
 741 static void
 742 vtmmio_reset(struct vtmmio_softc *sc)
 743 {
 744
 745         /*
 746          * Setting the status to RESET sets the host device to
 747          * the original, uninitialized state.
 748          */
 749         vtmmio_set_status(sc->dev, VIRTIO_CONFIG_STATUS_RESET);
 750 }
 751
 752 static void
 753 vtmmio_select_virtqueue(struct vtmmio_softc *sc, int idx)
 754 {
 755
 756         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx);
 757 }
 758
 759 static void
 760 vtmmio_vq_intr(void *arg)
 761 {
 762         struct vtmmio_virtqueue *vqx;
 763         struct vtmmio_softc *sc;
 764         struct virtqueue *vq;
 765         uint32_t status;
 766         int idx;
 767
 768         sc = arg;
 769
 770         status = vtmmio_read_config_4(sc, VIRTIO_MMIO_INTERRUPT_STATUS);
 771         vtmmio_write_config_4(sc, VIRTIO_MMIO_INTERRUPT_ACK, status);
 772
 773         /* The config changed */
 774         if (status & VIRTIO_MMIO_INT_CONFIG)
 775                 if (sc->vtmmio_child_dev != NULL)
 776                         VIRTIO_CONFIG_CHANGE(sc->vtmmio_child_dev);
 777
 778         /* Notify all virtqueues. */
 779         if (status & VIRTIO_MMIO_INT_VRING) {
 780                 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
 781                         vqx = &sc->vtmmio_vqs[idx];
 782                         if (vqx->vtv_no_intr == 0) {
 783                                 vq = vqx->vtv_vq;
 784                                 virtqueue_intr(vq);
 785                         }
 786                 }
 787         }
 788 }