sys/dev/virtio/mmio/virtio_mmio.c

   1 /*-
   2  * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com>
   3  * Copyright (c) 2014 The FreeBSD Foundation
   4  * All rights reserved.
   5  *
   6  * This software was developed by SRI International and the University of
   7  * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
   8  * ("CTSRD"), as part of the DARPA CRASH research programme.
   9  *
  10  * Portions of this software were developed by Andrew Turner
  11  * under sponsorship from the FreeBSD Foundation.
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  * 1. Redistributions of source code must retain the above copyright
  17  *    notice, this list of conditions and the following disclaimer.
  18  * 2. Redistributions in binary form must reproduce the above copyright
  19  *    notice, this list of conditions and the following disclaimer in the
  20  *    documentation and/or other materials provided with the distribution.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34
  35 /*
  36  * VirtIO MMIO interface.
  37  * This driver is heavily based on VirtIO PCI interface driver.
  38  */
  39
  40 #include <sys/cdefs.h>
  41 __FBSDID("$FreeBSD$");
  42
  43 #include <sys/param.h>
  44 #include <sys/systm.h>
  45 #include <sys/bus.h>
  46 #include <sys/kernel.h>
  47 #include <sys/module.h>
  48 #include <sys/malloc.h>
  49 #include <sys/rman.h>
  50
  51 #include <machine/bus.h>
  52 #include <machine/resource.h>
  53
  54 #include <dev/virtio/virtio.h>
  55 #include <dev/virtio/virtqueue.h>
  56 #include <dev/virtio/mmio/virtio_mmio.h>
  57
  58 #include "virtio_mmio_if.h"
  59 #include "virtio_bus_if.h"
  60 #include "virtio_if.h"
  61
  62 #define PAGE_SHIFT      12
  63
  64 struct vtmmio_virtqueue {
  65         struct virtqueue        *vtv_vq;
  66         int                      vtv_no_intr;
  67 };
  68
  69 static int      vtmmio_detach(device_t);
  70 static int      vtmmio_suspend(device_t);
  71 static int      vtmmio_resume(device_t);
  72 static int      vtmmio_shutdown(device_t);
  73 static void     vtmmio_driver_added(device_t, driver_t *);
  74 static void     vtmmio_child_detached(device_t, device_t);
  75 static int      vtmmio_read_ivar(device_t, device_t, int, uintptr_t *);
  76 static int      vtmmio_write_ivar(device_t, device_t, int, uintptr_t);
  77 static uint64_t vtmmio_negotiate_features(device_t, uint64_t);
  78 static int      vtmmio_with_feature(device_t, uint64_t);
  79 static int      vtmmio_alloc_virtqueues(device_t, int, int,
  80                     struct vq_alloc_info *);
  81 static int      vtmmio_setup_intr(device_t, enum intr_type);
  82 static void     vtmmio_stop(device_t);
  83 static void     vtmmio_poll(device_t);
  84 static int      vtmmio_reinit(device_t, uint64_t);
  85 static void     vtmmio_reinit_complete(device_t);
  86 static void     vtmmio_notify_virtqueue(device_t, uint16_t);
  87 static uint8_t  vtmmio_get_status(device_t);
  88 static void     vtmmio_set_status(device_t, uint8_t);
  89 static void     vtmmio_read_dev_config(device_t, bus_size_t, void *, int);
  90 static void     vtmmio_write_dev_config(device_t, bus_size_t, void *, int);
  91 static void     vtmmio_describe_features(struct vtmmio_softc *, const char *,
  92                     uint64_t);
  93 static void     vtmmio_probe_and_attach_child(struct vtmmio_softc *);
  94 static int      vtmmio_reinit_virtqueue(struct vtmmio_softc *, int);
  95 static void     vtmmio_free_interrupts(struct vtmmio_softc *);
  96 static void     vtmmio_free_virtqueues(struct vtmmio_softc *);
  97 static void     vtmmio_release_child_resources(struct vtmmio_softc *);
  98 static void     vtmmio_reset(struct vtmmio_softc *);
  99 static void     vtmmio_select_virtqueue(struct vtmmio_softc *, int);
 100 static void     vtmmio_vq_intr(void *);
 101
 102 /*
 103  * I/O port read/write wrappers.
 104  */
 105 #define vtmmio_write_config_1(sc, o, v)                         \
 106 do {                                                            \
 107         if (sc->platform != NULL)                               \
 108                 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v));   \
 109         bus_write_1((sc)->res[0], (o), (v));                    \
 110         if (sc->platform != NULL)                               \
 111                 VIRTIO_MMIO_NOTE(sc->platform, (o), (v));       \
 112 } while (0)
 113 #define vtmmio_write_config_2(sc, o, v)                         \
 114 do {                                                            \
 115         if (sc->platform != NULL)                               \
 116                 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v));   \
 117         bus_write_2((sc)->res[0], (o), (v));                    \
 118         if (sc->platform != NULL)                               \
 119                 VIRTIO_MMIO_NOTE(sc->platform, (o), (v));       \
 120 } while (0)
 121 #define vtmmio_write_config_4(sc, o, v)                         \
 122 do {                                                            \
 123         if (sc->platform != NULL)                               \
 124                 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v));   \
 125         bus_write_4((sc)->res[0], (o), (v));                    \
 126         if (sc->platform != NULL)                               \
 127                 VIRTIO_MMIO_NOTE(sc->platform, (o), (v));       \
 128 } while (0)
 129
 130 #define vtmmio_read_config_1(sc, o) \
 131         bus_read_1((sc)->res[0], (o))
 132 #define vtmmio_read_config_2(sc, o) \
 133         bus_read_2((sc)->res[0], (o))
 134 #define vtmmio_read_config_4(sc, o) \
 135         bus_read_4((sc)->res[0], (o))
 136
 137 static device_method_t vtmmio_methods[] = {
 138         /* Device interface. */
 139         DEVMETHOD(device_attach,                  vtmmio_attach),
 140         DEVMETHOD(device_detach,                  vtmmio_detach),
 141         DEVMETHOD(device_suspend,                 vtmmio_suspend),
 142         DEVMETHOD(device_resume,                  vtmmio_resume),
 143         DEVMETHOD(device_shutdown,                vtmmio_shutdown),
 144
 145         /* Bus interface. */
 146         DEVMETHOD(bus_driver_added,               vtmmio_driver_added),
 147         DEVMETHOD(bus_child_detached,             vtmmio_child_detached),
 148         DEVMETHOD(bus_read_ivar,                  vtmmio_read_ivar),
 149         DEVMETHOD(bus_write_ivar,                 vtmmio_write_ivar),
 150
 151         /* VirtIO bus interface. */
 152         DEVMETHOD(virtio_bus_negotiate_features,  vtmmio_negotiate_features),
 153         DEVMETHOD(virtio_bus_with_feature,        vtmmio_with_feature),
 154         DEVMETHOD(virtio_bus_alloc_virtqueues,    vtmmio_alloc_virtqueues),
 155         DEVMETHOD(virtio_bus_setup_intr,          vtmmio_setup_intr),
 156         DEVMETHOD(virtio_bus_stop,                vtmmio_stop),
 157         DEVMETHOD(virtio_bus_poll,                vtmmio_poll),
 158         DEVMETHOD(virtio_bus_reinit,              vtmmio_reinit),
 159         DEVMETHOD(virtio_bus_reinit_complete,     vtmmio_reinit_complete),
 160         DEVMETHOD(virtio_bus_notify_vq,           vtmmio_notify_virtqueue),
 161         DEVMETHOD(virtio_bus_read_device_config,  vtmmio_read_dev_config),
 162         DEVMETHOD(virtio_bus_write_device_config, vtmmio_write_dev_config),
 163
 164         DEVMETHOD_END
 165 };
 166
 167 DEFINE_CLASS_0(virtio_mmio, vtmmio_driver, vtmmio_methods,
 168     sizeof(struct vtmmio_softc));
 169
 170 MODULE_VERSION(virtio_mmio, 1);
 171
 172 static int
 173 vtmmio_setup_intr(device_t dev, enum intr_type type)
 174 {
 175         struct vtmmio_softc *sc;
 176         int rid;
 177         int err;
 178
 179         sc = device_get_softc(dev);
 180
 181         if (sc->platform != NULL) {
 182                 err = VIRTIO_MMIO_SETUP_INTR(sc->platform, sc->dev,
 183                                         vtmmio_vq_intr, sc);
 184                 if (err == 0) {
 185                         /* Okay we have backend-specific interrupts */
 186                         return (0);
 187                 }
 188         }
 189
 190         rid = 0;
 191         sc->res[1] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 192                 RF_ACTIVE);
 193         if (!sc->res[1]) {
 194                 device_printf(dev, "Can't allocate interrupt\n");
 195                 return (ENXIO);
 196         }
 197
 198         if (bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC | INTR_MPSAFE,
 199                 NULL, vtmmio_vq_intr, sc, &sc->ih)) {
 200                 device_printf(dev, "Can't setup the interrupt\n");
 201                 return (ENXIO);
 202         }
 203
 204         return (0);
 205 }
 206
 207 int
 208 vtmmio_attach(device_t dev)
 209 {
 210         struct vtmmio_softc *sc;
 211         device_t child;
 212         int rid;
 213
 214         sc = device_get_softc(dev);
 215         sc->dev = dev;
 216
 217         rid = 0;
 218         sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 219                         RF_ACTIVE);
 220         if (!sc->res[0]) {
 221                 device_printf(dev, "Cannot allocate memory window.\n");
 222                 return (ENXIO);
 223         }
 224
 225         vtmmio_reset(sc);
 226
 227         /* Tell the host we've noticed this device. */
 228         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
 229
 230         if ((child = device_add_child(dev, NULL, -1)) == NULL) {
 231                 device_printf(dev, "Cannot create child device.\n");
 232                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
 233                 vtmmio_detach(dev);
 234                 return (ENOMEM);
 235         }
 236
 237         sc->vtmmio_child_dev = child;
 238         vtmmio_probe_and_attach_child(sc);
 239
 240         return (0);
 241 }
 242
 243 static int
 244 vtmmio_detach(device_t dev)
 245 {
 246         struct vtmmio_softc *sc;
 247         device_t child;
 248         int error;
 249
 250         sc = device_get_softc(dev);
 251
 252         if ((child = sc->vtmmio_child_dev) != NULL) {
 253                 error = device_delete_child(dev, child);
 254                 if (error)
 255                         return (error);
 256                 sc->vtmmio_child_dev = NULL;
 257         }
 258
 259         vtmmio_reset(sc);
 260
 261         if (sc->res[0] != NULL) {
 262                 bus_release_resource(dev, SYS_RES_MEMORY, 0,
 263                     sc->res[0]);
 264                 sc->res[0] = NULL;
 265         }
 266
 267         return (0);
 268 }
 269
 270 static int
 271 vtmmio_suspend(device_t dev)
 272 {
 273
 274         return (bus_generic_suspend(dev));
 275 }
 276
 277 static int
 278 vtmmio_resume(device_t dev)
 279 {
 280
 281         return (bus_generic_resume(dev));
 282 }
 283
 284 static int
 285 vtmmio_shutdown(device_t dev)
 286 {
 287
 288         (void) bus_generic_shutdown(dev);
 289
 290         /* Forcibly stop the host device. */
 291         vtmmio_stop(dev);
 292
 293         return (0);
 294 }
 295
 296 static void
 297 vtmmio_driver_added(device_t dev, driver_t *driver)
 298 {
 299         struct vtmmio_softc *sc;
 300
 301         sc = device_get_softc(dev);
 302
 303         vtmmio_probe_and_attach_child(sc);
 304 }
 305
 306 static void
 307 vtmmio_child_detached(device_t dev, device_t child)
 308 {
 309         struct vtmmio_softc *sc;
 310
 311         sc = device_get_softc(dev);
 312
 313         vtmmio_reset(sc);
 314         vtmmio_release_child_resources(sc);
 315 }
 316
 317 static int
 318 vtmmio_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
 319 {
 320         struct vtmmio_softc *sc;
 321
 322         sc = device_get_softc(dev);
 323
 324         if (sc->vtmmio_child_dev != child)
 325                 return (ENOENT);
 326
 327         switch (index) {
 328         case VIRTIO_IVAR_DEVTYPE:
 329         case VIRTIO_IVAR_SUBDEVICE:
 330                 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_DEVICE_ID);
 331                 break;
 332         case VIRTIO_IVAR_VENDOR:
 333                 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_VENDOR_ID);
 334                 break;
 335         default:
 336                 return (ENOENT);
 337         }
 338
 339         return (0);
 340 }
 341
 342 static int
 343 vtmmio_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
 344 {
 345         struct vtmmio_softc *sc;
 346
 347         sc = device_get_softc(dev);
 348
 349         if (sc->vtmmio_child_dev != child)
 350                 return (ENOENT);
 351
 352         switch (index) {
 353         case VIRTIO_IVAR_FEATURE_DESC:
 354                 sc->vtmmio_child_feat_desc = (void *) value;
 355                 break;
 356         default:
 357                 return (ENOENT);
 358         }
 359
 360         return (0);
 361 }
 362
 363 static uint64_t
 364 vtmmio_negotiate_features(device_t dev, uint64_t child_features)
 365 {
 366         struct vtmmio_softc *sc;
 367         uint64_t host_features, features;
 368
 369         sc = device_get_softc(dev);
 370
 371         host_features = vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES);
 372         vtmmio_describe_features(sc, "host", host_features);
 373
 374         /*
 375          * Limit negotiated features to what the driver, virtqueue, and
 376          * host all support.
 377          */
 378         features = host_features & child_features;
 379         features = virtqueue_filter_features(features);
 380         sc->vtmmio_features = features;
 381
 382         vtmmio_describe_features(sc, "negotiated", features);
 383         vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features);
 384
 385         return (features);
 386 }
 387
 388 static int
 389 vtmmio_with_feature(device_t dev, uint64_t feature)
 390 {
 391         struct vtmmio_softc *sc;
 392
 393         sc = device_get_softc(dev);
 394
 395         return ((sc->vtmmio_features & feature) != 0);
 396 }
 397
 398 static int
 399 vtmmio_alloc_virtqueues(device_t dev, int flags, int nvqs,
 400     struct vq_alloc_info *vq_info)
 401 {
 402         struct vtmmio_virtqueue *vqx;
 403         struct vq_alloc_info *info;
 404         struct vtmmio_softc *sc;
 405         struct virtqueue *vq;
 406         uint32_t size;
 407         int idx, error;
 408
 409         sc = device_get_softc(dev);
 410
 411         if (sc->vtmmio_nvqs != 0)
 412                 return (EALREADY);
 413         if (nvqs <= 0)
 414                 return (EINVAL);
 415
 416         sc->vtmmio_vqs = malloc(nvqs * sizeof(struct vtmmio_virtqueue),
 417             M_DEVBUF, M_NOWAIT | M_ZERO);
 418         if (sc->vtmmio_vqs == NULL)
 419                 return (ENOMEM);
 420
 421         vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE,
 422             (1 << PAGE_SHIFT));
 423
 424         for (idx = 0; idx < nvqs; idx++) {
 425                 vqx = &sc->vtmmio_vqs[idx];
 426                 info = &vq_info[idx];
 427
 428                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx);
 429
 430                 vtmmio_select_virtqueue(sc, idx);
 431                 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX);
 432
 433                 error = virtqueue_alloc(dev, idx, size,
 434                     VIRTIO_MMIO_VRING_ALIGN, 0xFFFFFFFFUL, info, &vq);
 435                 if (error) {
 436                         device_printf(dev,
 437                             "cannot allocate virtqueue %d: %d\n",
 438                             idx, error);
 439                         break;
 440                 }
 441
 442                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size);
 443                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN,
 444                     VIRTIO_MMIO_VRING_ALIGN);
 445 #if 0
 446                 device_printf(dev, "virtqueue paddr 0x%08lx\n",
 447                     (uint64_t)virtqueue_paddr(vq));
 448 #endif
 449                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN,
 450                     virtqueue_paddr(vq) >> PAGE_SHIFT);
 451
 452                 vqx->vtv_vq = *info->vqai_vq = vq;
 453                 vqx->vtv_no_intr = info->vqai_intr == NULL;
 454
 455                 sc->vtmmio_nvqs++;
 456         }
 457
 458         if (error)
 459                 vtmmio_free_virtqueues(sc);
 460
 461         return (error);
 462 }
 463
 464 static void
 465 vtmmio_stop(device_t dev)
 466 {
 467
 468         vtmmio_reset(device_get_softc(dev));
 469 }
 470
 471 static void
 472 vtmmio_poll(device_t dev)
 473 {
 474         struct vtmmio_softc *sc;
 475
 476         sc = device_get_softc(dev);
 477
 478         if (sc->platform != NULL)
 479                 VIRTIO_MMIO_POLL(sc->platform);
 480 }
 481
 482 static int
 483 vtmmio_reinit(device_t dev, uint64_t features)
 484 {
 485         struct vtmmio_softc *sc;
 486         int idx, error;
 487
 488         sc = device_get_softc(dev);
 489
 490         if (vtmmio_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
 491                 vtmmio_stop(dev);
 492
 493         /*
 494          * Quickly drive the status through ACK and DRIVER. The device
 495          * does not become usable again until vtmmio_reinit_complete().
 496          */
 497         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
 498         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
 499
 500         vtmmio_negotiate_features(dev, features);
 501
 502         vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE,
 503             (1 << PAGE_SHIFT));
 504
 505         for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
 506                 error = vtmmio_reinit_virtqueue(sc, idx);
 507                 if (error)
 508                         return (error);
 509         }
 510
 511         return (0);
 512 }
 513
 514 static void
 515 vtmmio_reinit_complete(device_t dev)
 516 {
 517
 518         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
 519 }
 520
 521 static void
 522 vtmmio_notify_virtqueue(device_t dev, uint16_t queue)
 523 {
 524         struct vtmmio_softc *sc;
 525
 526         sc = device_get_softc(dev);
 527
 528         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NOTIFY, queue);
 529 }
 530
 531 static uint8_t
 532 vtmmio_get_status(device_t dev)
 533 {
 534         struct vtmmio_softc *sc;
 535
 536         sc = device_get_softc(dev);
 537
 538         return (vtmmio_read_config_4(sc, VIRTIO_MMIO_STATUS));
 539 }
 540
 541 static void
 542 vtmmio_set_status(device_t dev, uint8_t status)
 543 {
 544         struct vtmmio_softc *sc;
 545
 546         sc = device_get_softc(dev);
 547
 548         if (status != VIRTIO_CONFIG_STATUS_RESET)
 549                 status |= vtmmio_get_status(dev);
 550
 551         vtmmio_write_config_4(sc, VIRTIO_MMIO_STATUS, status);
 552 }
 553
 554 static void
 555 vtmmio_read_dev_config(device_t dev, bus_size_t offset,
 556     void *dst, int length)
 557 {
 558         struct vtmmio_softc *sc;
 559         bus_size_t off;
 560         uint8_t *d;
 561         int size;
 562
 563         sc = device_get_softc(dev);
 564         off = VIRTIO_MMIO_CONFIG + offset;
 565
 566         for (d = dst; length > 0; d += size, off += size, length -= size) {
 567 #ifdef ALLOW_WORD_ALIGNED_ACCESS
 568                 if (length >= 4) {
 569                         size = 4;
 570                         *(uint32_t *)d = vtmmio_read_config_4(sc, off);
 571                 } else if (length >= 2) {
 572                         size = 2;
 573                         *(uint16_t *)d = vtmmio_read_config_2(sc, off);
 574                 } else
 575 #endif
 576                 {
 577                         size = 1;
 578                         *d = vtmmio_read_config_1(sc, off);
 579                 }
 580         }
 581 }
 582
 583 static void
 584 vtmmio_write_dev_config(device_t dev, bus_size_t offset,
 585     void *src, int length)
 586 {
 587         struct vtmmio_softc *sc;
 588         bus_size_t off;
 589         uint8_t *s;
 590         int size;
 591
 592         sc = device_get_softc(dev);
 593         off = VIRTIO_MMIO_CONFIG + offset;
 594
 595         for (s = src; length > 0; s += size, off += size, length -= size) {
 596 #ifdef ALLOW_WORD_ALIGNED_ACCESS
 597                 if (length >= 4) {
 598                         size = 4;
 599                         vtmmio_write_config_4(sc, off, *(uint32_t *)s);
 600                 } else if (length >= 2) {
 601                         size = 2;
 602                         vtmmio_write_config_2(sc, off, *(uint16_t *)s);
 603                 } else
 604 #endif
 605                 {
 606                         size = 1;
 607                         vtmmio_write_config_1(sc, off, *s);
 608                 }
 609         }
 610 }
 611
 612 static void
 613 vtmmio_describe_features(struct vtmmio_softc *sc, const char *msg,
 614     uint64_t features)
 615 {
 616         device_t dev, child;
 617
 618         dev = sc->dev;
 619         child = sc->vtmmio_child_dev;
 620
 621         if (device_is_attached(child) || bootverbose == 0)
 622                 return;
 623
 624         virtio_describe(dev, msg, features, sc->vtmmio_child_feat_desc);
 625 }
 626
 627 static void
 628 vtmmio_probe_and_attach_child(struct vtmmio_softc *sc)
 629 {
 630         device_t dev, child;
 631
 632         dev = sc->dev;
 633         child = sc->vtmmio_child_dev;
 634
 635         if (child == NULL)
 636                 return;
 637
 638         if (device_get_state(child) != DS_NOTPRESENT) {
 639                 return;
 640         }
 641
 642         if (device_probe(child) != 0) {
 643                 return;
 644         }
 645
 646         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
 647         if (device_attach(child) != 0) {
 648                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
 649                 vtmmio_reset(sc);
 650                 vtmmio_release_child_resources(sc);
 651                 /* Reset status for future attempt. */
 652                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
 653         } else {
 654                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
 655                 VIRTIO_ATTACH_COMPLETED(child);
 656         }
 657 }
 658
 659 static int
 660 vtmmio_reinit_virtqueue(struct vtmmio_softc *sc, int idx)
 661 {
 662         struct vtmmio_virtqueue *vqx;
 663         struct virtqueue *vq;
 664         int error;
 665         uint16_t size;
 666
 667         vqx = &sc->vtmmio_vqs[idx];
 668         vq = vqx->vtv_vq;
 669
 670         KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));
 671
 672         vtmmio_select_virtqueue(sc, idx);
 673         size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX);
 674
 675         error = virtqueue_reinit(vq, size);
 676         if (error)
 677                 return (error);
 678
 679         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size);
 680         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN,
 681             VIRTIO_MMIO_VRING_ALIGN);
 682 #if 0
 683         device_printf(sc->dev, "virtqueue paddr 0x%08lx\n",
 684             (uint64_t)virtqueue_paddr(vq));
 685 #endif
 686         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN,
 687             virtqueue_paddr(vq) >> PAGE_SHIFT);
 688
 689         return (0);
 690 }
 691
 692 static void
 693 vtmmio_free_interrupts(struct vtmmio_softc *sc)
 694 {
 695
 696         if (sc->ih != NULL)
 697                 bus_teardown_intr(sc->dev, sc->res[1], sc->ih);
 698
 699         if (sc->res[1] != NULL)
 700                 bus_release_resource(sc->dev, SYS_RES_IRQ, 0, sc->res[1]);
 701 }
 702
 703 static void
 704 vtmmio_free_virtqueues(struct vtmmio_softc *sc)
 705 {
 706         struct vtmmio_virtqueue *vqx;
 707         int idx;
 708
 709         for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
 710                 vqx = &sc->vtmmio_vqs[idx];
 711
 712                 vtmmio_select_virtqueue(sc, idx);
 713                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 0);
 714
 715                 virtqueue_free(vqx->vtv_vq);
 716                 vqx->vtv_vq = NULL;
 717         }
 718
 719         free(sc->vtmmio_vqs, M_DEVBUF);
 720         sc->vtmmio_vqs = NULL;
 721         sc->vtmmio_nvqs = 0;
 722 }
 723
 724 static void
 725 vtmmio_release_child_resources(struct vtmmio_softc *sc)
 726 {
 727
 728         vtmmio_free_interrupts(sc);
 729         vtmmio_free_virtqueues(sc);
 730 }
 731
 732 static void
 733 vtmmio_reset(struct vtmmio_softc *sc)
 734 {
 735
 736         /*
 737          * Setting the status to RESET sets the host device to
 738          * the original, uninitialized state.
 739          */
 740         vtmmio_set_status(sc->dev, VIRTIO_CONFIG_STATUS_RESET);
 741 }
 742
 743 static void
 744 vtmmio_select_virtqueue(struct vtmmio_softc *sc, int idx)
 745 {
 746
 747         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx);
 748 }
 749
 750 static void
 751 vtmmio_vq_intr(void *arg)
 752 {
 753         struct vtmmio_virtqueue *vqx;
 754         struct vtmmio_softc *sc;
 755         struct virtqueue *vq;
 756         uint32_t status;
 757         int idx;
 758
 759         sc = arg;
 760
 761         status = vtmmio_read_config_4(sc, VIRTIO_MMIO_INTERRUPT_STATUS);
 762         vtmmio_write_config_4(sc, VIRTIO_MMIO_INTERRUPT_ACK, status);
 763
 764         /* The config changed */
 765         if (status & VIRTIO_MMIO_INT_CONFIG)
 766                 if (sc->vtmmio_child_dev != NULL)
 767                         VIRTIO_CONFIG_CHANGE(sc->vtmmio_child_dev);
 768
 769         /* Notify all virtqueues. */
 770         if (status & VIRTIO_MMIO_INT_VRING) {
 771                 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
 772                         vqx = &sc->vtmmio_vqs[idx];
 773                         if (vqx->vtv_no_intr == 0) {
 774                                 vq = vqx->vtv_vq;
 775                                 virtqueue_intr(vq);
 776                         }
 777                 }
 778         }
 779 }