sys/dev/virtio/mmio/virtio_mmio.c

   1 /*-
   2  * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com>
   3  * Copyright (c) 2014 The FreeBSD Foundation
   4  * All rights reserved.
   5  *
   6  * This software was developed by SRI International and the University of
   7  * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
   8  * ("CTSRD"), as part of the DARPA CRASH research programme.
   9  *
  10  * Portions of this software were developed by Andrew Turner
  11  * under sponsorship from the FreeBSD Foundation.
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  * 1. Redistributions of source code must retain the above copyright
  17  *    notice, this list of conditions and the following disclaimer.
  18  * 2. Redistributions in binary form must reproduce the above copyright
  19  *    notice, this list of conditions and the following disclaimer in the
  20  *    documentation and/or other materials provided with the distribution.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34
  35 /*
  36  * VirtIO MMIO interface.
  37  * This driver is heavily based on VirtIO PCI interface driver.
  38  */
  39
  40 /*
  41  * FDT example:
  42  *              virtio_block@1000 {
  43  *                      compatible = "virtio,mmio";
  44  *                      reg = <0x1000 0x100>;
  45  *                      interrupts = <63>;
  46  *                      interrupt-parent = <&GIC>;
  47  *              };
  48  */
  49
  50 #include <sys/cdefs.h>
  51 __FBSDID("$FreeBSD$");
  52
  53 #include <sys/param.h>
  54 #include <sys/systm.h>
  55 #include <sys/bus.h>
  56 #include <sys/kernel.h>
  57 #include <sys/module.h>
  58 #include <sys/malloc.h>
  59 #include <sys/rman.h>
  60
  61 #include <machine/bus.h>
  62 #include <machine/resource.h>
  63
  64 #include <dev/fdt/fdt_common.h>
  65 #include <dev/ofw/openfirm.h>
  66 #include <dev/ofw/ofw_bus.h>
  67 #include <dev/ofw/ofw_bus_subr.h>
  68
  69 #include <dev/virtio/virtio.h>
  70 #include <dev/virtio/virtqueue.h>
  71 #include <dev/virtio/mmio/virtio_mmio.h>
  72
  73 #include "virtio_mmio_if.h"
  74 #include "virtio_bus_if.h"
  75 #include "virtio_if.h"
  76
  77 #define PAGE_SHIFT      12
  78
  79 struct vtmmio_virtqueue {
  80         struct virtqueue        *vtv_vq;
  81         int                      vtv_no_intr;
  82 };
  83
  84 struct vtmmio_softc {
  85         device_t                        dev;
  86         device_t                        platform;
  87         struct resource                 *res[2];
  88
  89         uint64_t                        vtmmio_features;
  90         uint32_t                        vtmmio_flags;
  91
  92         /* This "bus" will only ever have one child. */
  93         device_t                        vtmmio_child_dev;
  94         struct virtio_feature_desc      *vtmmio_child_feat_desc;
  95
  96         int                             vtmmio_nvqs;
  97         struct vtmmio_virtqueue         *vtmmio_vqs;
  98         void                            *ih;
  99 };
 100
 101 static int      vtmmio_probe(device_t);
 102 static int      vtmmio_attach(device_t);
 103 static int      vtmmio_detach(device_t);
 104 static int      vtmmio_suspend(device_t);
 105 static int      vtmmio_resume(device_t);
 106 static int      vtmmio_shutdown(device_t);
 107 static void     vtmmio_driver_added(device_t, driver_t *);
 108 static void     vtmmio_child_detached(device_t, device_t);
 109 static int      vtmmio_read_ivar(device_t, device_t, int, uintptr_t *);
 110 static int      vtmmio_write_ivar(device_t, device_t, int, uintptr_t);
 111 static uint64_t vtmmio_negotiate_features(device_t, uint64_t);
 112 static int      vtmmio_with_feature(device_t, uint64_t);
 113 static int      vtmmio_alloc_virtqueues(device_t, int, int,
 114                     struct vq_alloc_info *);
 115 static int      vtmmio_setup_intr(device_t, enum intr_type);
 116 static void     vtmmio_stop(device_t);
 117 static void     vtmmio_poll(device_t);
 118 static int      vtmmio_reinit(device_t, uint64_t);
 119 static void     vtmmio_reinit_complete(device_t);
 120 static void     vtmmio_notify_virtqueue(device_t, uint16_t);
 121 static uint8_t  vtmmio_get_status(device_t);
 122 static void     vtmmio_set_status(device_t, uint8_t);
 123 static void     vtmmio_read_dev_config(device_t, bus_size_t, void *, int);
 124 static void     vtmmio_write_dev_config(device_t, bus_size_t, void *, int);
 125 static void     vtmmio_describe_features(struct vtmmio_softc *, const char *,
 126                     uint64_t);
 127 static void     vtmmio_probe_and_attach_child(struct vtmmio_softc *);
 128 static int      vtmmio_reinit_virtqueue(struct vtmmio_softc *, int);
 129 static void     vtmmio_free_interrupts(struct vtmmio_softc *);
 130 static void     vtmmio_free_virtqueues(struct vtmmio_softc *);
 131 static void     vtmmio_release_child_resources(struct vtmmio_softc *);
 132 static void     vtmmio_reset(struct vtmmio_softc *);
 133 static void     vtmmio_select_virtqueue(struct vtmmio_softc *, int);
 134 static void     vtmmio_vq_intr(void *);
 135
 136 /*
 137  * I/O port read/write wrappers.
 138  */
 139 #define vtmmio_write_config_1(sc, o, v)                         \
 140 do {                                                            \
 141         if (sc->platform != NULL)                               \
 142                 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v));   \
 143         bus_write_1((sc)->res[0], (o), (v));                    \
 144         if (sc->platform != NULL)                               \
 145                 VIRTIO_MMIO_NOTE(sc->platform, (o), (v));       \
 146 } while (0)
 147 #define vtmmio_write_config_2(sc, o, v)                         \
 148 do {                                                            \
 149         if (sc->platform != NULL)                               \
 150                 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v));   \
 151         bus_write_2((sc)->res[0], (o), (v));                    \
 152         if (sc->platform != NULL)                               \
 153                 VIRTIO_MMIO_NOTE(sc->platform, (o), (v));       \
 154 } while (0)
 155 #define vtmmio_write_config_4(sc, o, v)                         \
 156 do {                                                            \
 157         if (sc->platform != NULL)                               \
 158                 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v));   \
 159         bus_write_4((sc)->res[0], (o), (v));                    \
 160         if (sc->platform != NULL)                               \
 161                 VIRTIO_MMIO_NOTE(sc->platform, (o), (v));       \
 162 } while (0)
 163
 164 #define vtmmio_read_config_1(sc, o) \
 165         bus_read_1((sc)->res[0], (o))
 166 #define vtmmio_read_config_2(sc, o) \
 167         bus_read_2((sc)->res[0], (o))
 168 #define vtmmio_read_config_4(sc, o) \
 169         bus_read_4((sc)->res[0], (o))
 170
 171 static device_method_t vtmmio_methods[] = {
 172         /* Device interface. */
 173         DEVMETHOD(device_probe,                   vtmmio_probe),
 174         DEVMETHOD(device_attach,                  vtmmio_attach),
 175         DEVMETHOD(device_detach,                  vtmmio_detach),
 176         DEVMETHOD(device_suspend,                 vtmmio_suspend),
 177         DEVMETHOD(device_resume,                  vtmmio_resume),
 178         DEVMETHOD(device_shutdown,                vtmmio_shutdown),
 179
 180         /* Bus interface. */
 181         DEVMETHOD(bus_driver_added,               vtmmio_driver_added),
 182         DEVMETHOD(bus_child_detached,             vtmmio_child_detached),
 183         DEVMETHOD(bus_read_ivar,                  vtmmio_read_ivar),
 184         DEVMETHOD(bus_write_ivar,                 vtmmio_write_ivar),
 185
 186         /* VirtIO bus interface. */
 187         DEVMETHOD(virtio_bus_negotiate_features,  vtmmio_negotiate_features),
 188         DEVMETHOD(virtio_bus_with_feature,        vtmmio_with_feature),
 189         DEVMETHOD(virtio_bus_alloc_virtqueues,    vtmmio_alloc_virtqueues),
 190         DEVMETHOD(virtio_bus_setup_intr,          vtmmio_setup_intr),
 191         DEVMETHOD(virtio_bus_stop,                vtmmio_stop),
 192         DEVMETHOD(virtio_bus_poll,                vtmmio_poll),
 193         DEVMETHOD(virtio_bus_reinit,              vtmmio_reinit),
 194         DEVMETHOD(virtio_bus_reinit_complete,     vtmmio_reinit_complete),
 195         DEVMETHOD(virtio_bus_notify_vq,           vtmmio_notify_virtqueue),
 196         DEVMETHOD(virtio_bus_read_device_config,  vtmmio_read_dev_config),
 197         DEVMETHOD(virtio_bus_write_device_config, vtmmio_write_dev_config),
 198
 199         DEVMETHOD_END
 200 };
 201
 202 static driver_t vtmmio_driver = {
 203         "virtio_mmio",
 204         vtmmio_methods,
 205         sizeof(struct vtmmio_softc)
 206 };
 207
 208 devclass_t vtmmio_devclass;
 209
 210 DRIVER_MODULE(virtio_mmio, simplebus, vtmmio_driver, vtmmio_devclass, 0, 0);
 211 DRIVER_MODULE(virtio_mmio, ofwbus, vtmmio_driver, vtmmio_devclass, 0, 0);
 212 MODULE_VERSION(virtio_mmio, 1);
 213 MODULE_DEPEND(virtio_mmio, simplebus, 1, 1, 1);
 214 MODULE_DEPEND(virtio_mmio, virtio, 1, 1, 1);
 215
 216 static int
 217 vtmmio_setup_intr(device_t dev, enum intr_type type)
 218 {
 219         struct vtmmio_softc *sc;
 220         int rid;
 221         int err;
 222
 223         sc = device_get_softc(dev);
 224
 225         if (sc->platform != NULL) {
 226                 err = VIRTIO_MMIO_SETUP_INTR(sc->platform, sc->dev,
 227                                         vtmmio_vq_intr, sc);
 228                 if (err == 0) {
 229                         /* Okay we have backend-specific interrupts */
 230                         return (0);
 231                 }
 232         }
 233
 234         rid = 0;
 235         sc->res[1] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 236                 RF_ACTIVE);
 237         if (!sc->res[1]) {
 238                 device_printf(dev, "Can't allocate interrupt\n");
 239                 return (ENXIO);
 240         }
 241
 242         if (bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC | INTR_MPSAFE,
 243                 NULL, vtmmio_vq_intr, sc, &sc->ih)) {
 244                 device_printf(dev, "Can't setup the interrupt\n");
 245                 return (ENXIO);
 246         }
 247
 248         return (0);
 249 }
 250
 251 static int
 252 vtmmio_probe(device_t dev)
 253 {
 254
 255         if (!ofw_bus_status_okay(dev))
 256                 return (ENXIO);
 257
 258         if (!ofw_bus_is_compatible(dev, "virtio,mmio"))
 259                 return (ENXIO);
 260
 261         device_set_desc(dev, "VirtIO MMIO adapter");
 262         return (BUS_PROBE_DEFAULT);
 263 }
 264
 265 static int
 266 vtmmio_setup_platform(struct vtmmio_softc *sc)
 267 {
 268         phandle_t platform_node;
 269         struct fdt_ic *ic;
 270         phandle_t xref;
 271         phandle_t node;
 272
 273         sc->platform = NULL;
 274
 275         if ((node = ofw_bus_get_node(sc->dev)) == -1)
 276                 return (ENXIO);
 277
 278         if (OF_searchencprop(node, "platform", &xref,
 279                 sizeof(xref)) == -1) {
 280                 return (ENXIO);
 281         }
 282
 283         platform_node = OF_node_from_xref(xref);
 284
 285         SLIST_FOREACH(ic, &fdt_ic_list_head, fdt_ics) {
 286                 if (ic->iph == platform_node) {
 287                         sc->platform = ic->dev;
 288                         break;
 289                 }
 290         }
 291
 292         if (sc->platform == NULL) {
 293                 /* No platform-specific device. Ignore it. */
 294         }
 295
 296         return (0);
 297 }
 298
 299 static int
 300 vtmmio_attach(device_t dev)
 301 {
 302         struct vtmmio_softc *sc;
 303         device_t child;
 304         int rid;
 305
 306         sc = device_get_softc(dev);
 307         sc->dev = dev;
 308
 309         vtmmio_setup_platform(sc);
 310
 311         rid = 0;
 312         sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 313                         RF_ACTIVE);
 314         if (!sc->res[0]) {
 315                 device_printf(dev, "Cannot allocate memory window.\n");
 316                 return (ENXIO);
 317         }
 318
 319         vtmmio_reset(sc);
 320
 321         /* Tell the host we've noticed this device. */
 322         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
 323
 324         if ((child = device_add_child(dev, NULL, -1)) == NULL) {
 325                 device_printf(dev, "Cannot create child device.\n");
 326                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
 327                 vtmmio_detach(dev);
 328                 return (ENOMEM);
 329         }
 330
 331         sc->vtmmio_child_dev = child;
 332         vtmmio_probe_and_attach_child(sc);
 333
 334         return (0);
 335 }
 336
 337 static int
 338 vtmmio_detach(device_t dev)
 339 {
 340         struct vtmmio_softc *sc;
 341         device_t child;
 342         int error;
 343
 344         sc = device_get_softc(dev);
 345
 346         if ((child = sc->vtmmio_child_dev) != NULL) {
 347                 error = device_delete_child(dev, child);
 348                 if (error)
 349                         return (error);
 350                 sc->vtmmio_child_dev = NULL;
 351         }
 352
 353         vtmmio_reset(sc);
 354
 355         if (sc->res[0] != NULL) {
 356                 bus_release_resource(dev, SYS_RES_MEMORY, 0,
 357                     sc->res[0]);
 358                 sc->res[0] = NULL;
 359         }
 360
 361         return (0);
 362 }
 363
 364 static int
 365 vtmmio_suspend(device_t dev)
 366 {
 367
 368         return (bus_generic_suspend(dev));
 369 }
 370
 371 static int
 372 vtmmio_resume(device_t dev)
 373 {
 374
 375         return (bus_generic_resume(dev));
 376 }
 377
 378 static int
 379 vtmmio_shutdown(device_t dev)
 380 {
 381
 382         (void) bus_generic_shutdown(dev);
 383
 384         /* Forcibly stop the host device. */
 385         vtmmio_stop(dev);
 386
 387         return (0);
 388 }
 389
 390 static void
 391 vtmmio_driver_added(device_t dev, driver_t *driver)
 392 {
 393         struct vtmmio_softc *sc;
 394
 395         sc = device_get_softc(dev);
 396
 397         vtmmio_probe_and_attach_child(sc);
 398 }
 399
 400 static void
 401 vtmmio_child_detached(device_t dev, device_t child)
 402 {
 403         struct vtmmio_softc *sc;
 404
 405         sc = device_get_softc(dev);
 406
 407         vtmmio_reset(sc);
 408         vtmmio_release_child_resources(sc);
 409 }
 410
 411 static int
 412 vtmmio_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
 413 {
 414         struct vtmmio_softc *sc;
 415
 416         sc = device_get_softc(dev);
 417
 418         if (sc->vtmmio_child_dev != child)
 419                 return (ENOENT);
 420
 421         switch (index) {
 422         case VIRTIO_IVAR_DEVTYPE:
 423         case VIRTIO_IVAR_SUBDEVICE:
 424                 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_DEVICE_ID);
 425                 break;
 426         case VIRTIO_IVAR_VENDOR:
 427                 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_VENDOR_ID);
 428                 break;
 429         default:
 430                 return (ENOENT);
 431         }
 432
 433         return (0);
 434 }
 435
 436 static int
 437 vtmmio_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
 438 {
 439         struct vtmmio_softc *sc;
 440
 441         sc = device_get_softc(dev);
 442
 443         if (sc->vtmmio_child_dev != child)
 444                 return (ENOENT);
 445
 446         switch (index) {
 447         case VIRTIO_IVAR_FEATURE_DESC:
 448                 sc->vtmmio_child_feat_desc = (void *) value;
 449                 break;
 450         default:
 451                 return (ENOENT);
 452         }
 453
 454         return (0);
 455 }
 456
 457 static uint64_t
 458 vtmmio_negotiate_features(device_t dev, uint64_t child_features)
 459 {
 460         struct vtmmio_softc *sc;
 461         uint64_t host_features, features;
 462
 463         sc = device_get_softc(dev);
 464
 465         host_features = vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES);
 466         vtmmio_describe_features(sc, "host", host_features);
 467
 468         /*
 469          * Limit negotiated features to what the driver, virtqueue, and
 470          * host all support.
 471          */
 472         features = host_features & child_features;
 473         features = virtqueue_filter_features(features);
 474         sc->vtmmio_features = features;
 475
 476         vtmmio_describe_features(sc, "negotiated", features);
 477         vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features);
 478
 479         return (features);
 480 }
 481
 482 static int
 483 vtmmio_with_feature(device_t dev, uint64_t feature)
 484 {
 485         struct vtmmio_softc *sc;
 486
 487         sc = device_get_softc(dev);
 488
 489         return ((sc->vtmmio_features & feature) != 0);
 490 }
 491
 492 static int
 493 vtmmio_alloc_virtqueues(device_t dev, int flags, int nvqs,
 494     struct vq_alloc_info *vq_info)
 495 {
 496         struct vtmmio_virtqueue *vqx;
 497         struct vq_alloc_info *info;
 498         struct vtmmio_softc *sc;
 499         struct virtqueue *vq;
 500         uint32_t size;
 501         int idx, error;
 502
 503         sc = device_get_softc(dev);
 504
 505         if (sc->vtmmio_nvqs != 0)
 506                 return (EALREADY);
 507         if (nvqs <= 0)
 508                 return (EINVAL);
 509
 510         sc->vtmmio_vqs = malloc(nvqs * sizeof(struct vtmmio_virtqueue),
 511             M_DEVBUF, M_NOWAIT | M_ZERO);
 512         if (sc->vtmmio_vqs == NULL)
 513                 return (ENOMEM);
 514
 515         vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE, 1 << PAGE_SHIFT);
 516
 517         for (idx = 0; idx < nvqs; idx++) {
 518                 vqx = &sc->vtmmio_vqs[idx];
 519                 info = &vq_info[idx];
 520
 521                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx);
 522
 523                 vtmmio_select_virtqueue(sc, idx);
 524                 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX);
 525
 526                 error = virtqueue_alloc(dev, idx, size,
 527                     VIRTIO_MMIO_VRING_ALIGN, 0xFFFFFFFFUL, info, &vq);
 528                 if (error) {
 529                         device_printf(dev,
 530                             "cannot allocate virtqueue %d: %d\n",
 531                             idx, error);
 532                         break;
 533                 }
 534
 535                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size);
 536                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN,
 537                     VIRTIO_MMIO_VRING_ALIGN);
 538 #if 0
 539                 device_printf(dev, "virtqueue paddr 0x%08lx\n",
 540                                 (uint64_t)virtqueue_paddr(vq));
 541 #endif
 542                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN,
 543                         virtqueue_paddr(vq) >> PAGE_SHIFT);
 544
 545                 vqx->vtv_vq = *info->vqai_vq = vq;
 546                 vqx->vtv_no_intr = info->vqai_intr == NULL;
 547
 548                 sc->vtmmio_nvqs++;
 549         }
 550
 551         if (error)
 552                 vtmmio_free_virtqueues(sc);
 553
 554         return (error);
 555 }
 556
 557 static void
 558 vtmmio_stop(device_t dev)
 559 {
 560
 561         vtmmio_reset(device_get_softc(dev));
 562 }
 563
 564 static void
 565 vtmmio_poll(device_t dev)
 566 {
 567         struct vtmmio_softc *sc;
 568
 569         sc = device_get_softc(dev);
 570
 571         if (sc->platform != NULL)
 572                 VIRTIO_MMIO_POLL(sc->platform);
 573 }
 574
 575 static int
 576 vtmmio_reinit(device_t dev, uint64_t features)
 577 {
 578         struct vtmmio_softc *sc;
 579         int idx, error;
 580
 581         sc = device_get_softc(dev);
 582
 583         if (vtmmio_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
 584                 vtmmio_stop(dev);
 585
 586         /*
 587          * Quickly drive the status through ACK and DRIVER. The device
 588          * does not become usable again until vtmmio_reinit_complete().
 589          */
 590         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
 591         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
 592
 593         vtmmio_negotiate_features(dev, features);
 594
 595         for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
 596                 error = vtmmio_reinit_virtqueue(sc, idx);
 597                 if (error)
 598                         return (error);
 599         }
 600
 601         return (0);
 602 }
 603
 604 static void
 605 vtmmio_reinit_complete(device_t dev)
 606 {
 607
 608         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
 609 }
 610
 611 static void
 612 vtmmio_notify_virtqueue(device_t dev, uint16_t queue)
 613 {
 614         struct vtmmio_softc *sc;
 615
 616         sc = device_get_softc(dev);
 617
 618         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NOTIFY, queue);
 619 }
 620
 621 static uint8_t
 622 vtmmio_get_status(device_t dev)
 623 {
 624         struct vtmmio_softc *sc;
 625
 626         sc = device_get_softc(dev);
 627
 628         return (vtmmio_read_config_4(sc, VIRTIO_MMIO_STATUS));
 629 }
 630
 631 static void
 632 vtmmio_set_status(device_t dev, uint8_t status)
 633 {
 634         struct vtmmio_softc *sc;
 635
 636         sc = device_get_softc(dev);
 637
 638         if (status != VIRTIO_CONFIG_STATUS_RESET)
 639                 status |= vtmmio_get_status(dev);
 640
 641         vtmmio_write_config_4(sc, VIRTIO_MMIO_STATUS, status);
 642 }
 643
 644 static void
 645 vtmmio_read_dev_config(device_t dev, bus_size_t offset,
 646     void *dst, int length)
 647 {
 648         struct vtmmio_softc *sc;
 649         bus_size_t off;
 650         uint8_t *d;
 651         int size;
 652
 653         sc = device_get_softc(dev);
 654         off = VIRTIO_MMIO_CONFIG + offset;
 655
 656         for (d = dst; length > 0; d += size, off += size, length -= size) {
 657 #ifdef ALLOW_WORD_ALIGNED_ACCESS
 658                 if (length >= 4) {
 659                         size = 4;
 660                         *(uint32_t *)d = vtmmio_read_config_4(sc, off);
 661                 } else if (length >= 2) {
 662                         size = 2;
 663                         *(uint16_t *)d = vtmmio_read_config_2(sc, off);
 664                 } else
 665 #endif
 666                 {
 667                         size = 1;
 668                         *d = vtmmio_read_config_1(sc, off);
 669                 }
 670         }
 671 }
 672
 673 static void
 674 vtmmio_write_dev_config(device_t dev, bus_size_t offset,
 675     void *src, int length)
 676 {
 677         struct vtmmio_softc *sc;
 678         bus_size_t off;
 679         uint8_t *s;
 680         int size;
 681
 682         sc = device_get_softc(dev);
 683         off = VIRTIO_MMIO_CONFIG + offset;
 684
 685         for (s = src; length > 0; s += size, off += size, length -= size) {
 686 #ifdef ALLOW_WORD_ALIGNED_ACCESS
 687                 if (length >= 4) {
 688                         size = 4;
 689                         vtmmio_write_config_4(sc, off, *(uint32_t *)s);
 690                 } else if (length >= 2) {
 691                         size = 2;
 692                         vtmmio_write_config_2(sc, off, *(uint16_t *)s);
 693                 } else
 694 #endif
 695                 {
 696                         size = 1;
 697                         vtmmio_write_config_1(sc, off, *s);
 698                 }
 699         }
 700 }
 701
 702 static void
 703 vtmmio_describe_features(struct vtmmio_softc *sc, const char *msg,
 704     uint64_t features)
 705 {
 706         device_t dev, child;
 707
 708         dev = sc->dev;
 709         child = sc->vtmmio_child_dev;
 710
 711         if (device_is_attached(child) || bootverbose == 0)
 712                 return;
 713
 714         virtio_describe(dev, msg, features, sc->vtmmio_child_feat_desc);
 715 }
 716
 717 static void
 718 vtmmio_probe_and_attach_child(struct vtmmio_softc *sc)
 719 {
 720         device_t dev, child;
 721
 722         dev = sc->dev;
 723         child = sc->vtmmio_child_dev;
 724
 725         if (child == NULL)
 726                 return;
 727
 728         if (device_get_state(child) != DS_NOTPRESENT) {
 729                 return;
 730         }
 731
 732         if (device_probe(child) != 0) {
 733                 return;
 734         }
 735
 736         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
 737         if (device_attach(child) != 0) {
 738                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
 739                 vtmmio_reset(sc);
 740                 vtmmio_release_child_resources(sc);
 741                 /* Reset status for future attempt. */
 742                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
 743         } else {
 744                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
 745                 VIRTIO_ATTACH_COMPLETED(child);
 746         }
 747 }
 748
 749 static int
 750 vtmmio_reinit_virtqueue(struct vtmmio_softc *sc, int idx)
 751 {
 752         struct vtmmio_virtqueue *vqx;
 753         struct virtqueue *vq;
 754         int error;
 755         uint16_t size;
 756
 757         vqx = &sc->vtmmio_vqs[idx];
 758         vq = vqx->vtv_vq;
 759
 760         KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));
 761
 762         vtmmio_select_virtqueue(sc, idx);
 763         size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX);
 764
 765         error = virtqueue_reinit(vq, size);
 766         if (error)
 767                 return (error);
 768
 769         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN,
 770             virtqueue_paddr(vq) >> PAGE_SHIFT);
 771
 772         return (0);
 773 }
 774
 775 static void
 776 vtmmio_free_interrupts(struct vtmmio_softc *sc)
 777 {
 778
 779         if (sc->ih != NULL)
 780                 bus_teardown_intr(sc->dev, sc->res[1], sc->ih);
 781
 782         if (sc->res[1] != NULL)
 783                 bus_release_resource(sc->dev, SYS_RES_IRQ, 0, sc->res[1]);
 784 }
 785
 786 static void
 787 vtmmio_free_virtqueues(struct vtmmio_softc *sc)
 788 {
 789         struct vtmmio_virtqueue *vqx;
 790         int idx;
 791
 792         for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
 793                 vqx = &sc->vtmmio_vqs[idx];
 794
 795                 vtmmio_select_virtqueue(sc, idx);
 796                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 0);
 797
 798                 virtqueue_free(vqx->vtv_vq);
 799                 vqx->vtv_vq = NULL;
 800         }
 801
 802         free(sc->vtmmio_vqs, M_DEVBUF);
 803         sc->vtmmio_vqs = NULL;
 804         sc->vtmmio_nvqs = 0;
 805 }
 806
 807 static void
 808 vtmmio_release_child_resources(struct vtmmio_softc *sc)
 809 {
 810
 811         vtmmio_free_interrupts(sc);
 812         vtmmio_free_virtqueues(sc);
 813 }
 814
 815 static void
 816 vtmmio_reset(struct vtmmio_softc *sc)
 817 {
 818
 819         /*
 820          * Setting the status to RESET sets the host device to
 821          * the original, uninitialized state.
 822          */
 823         vtmmio_set_status(sc->dev, VIRTIO_CONFIG_STATUS_RESET);
 824 }
 825
 826 static void
 827 vtmmio_select_virtqueue(struct vtmmio_softc *sc, int idx)
 828 {
 829
 830         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx);
 831 }
 832
 833 static void
 834 vtmmio_vq_intr(void *arg)
 835 {
 836         struct vtmmio_virtqueue *vqx;
 837         struct vtmmio_softc *sc;
 838         struct virtqueue *vq;
 839         uint32_t status;
 840         int idx;
 841
 842         sc = arg;
 843
 844         status = vtmmio_read_config_4(sc, VIRTIO_MMIO_INTERRUPT_STATUS);
 845         vtmmio_write_config_4(sc, VIRTIO_MMIO_INTERRUPT_ACK, status);
 846
 847         /* The config changed */
 848         if (status & VIRTIO_MMIO_INT_CONFIG)
 849                 if (sc->vtmmio_child_dev != NULL)
 850                         VIRTIO_CONFIG_CHANGE(sc->vtmmio_child_dev);
 851
 852         /* Notify all virtqueues. */
 853         if (status & VIRTIO_MMIO_INT_VRING) {
 854                 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
 855                         vqx = &sc->vtmmio_vqs[idx];
 856                         if (vqx->vtv_no_intr == 0) {
 857                                 vq = vqx->vtv_vq;
 858                                 virtqueue_intr(vq);
 859                         }
 860                 }
 861         }
 862 }