sys/dev/virtio/mmio/virtio_mmio.c

   1 /*-
   2  * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com>
   3  * Copyright (c) 2014 The FreeBSD Foundation
   4  * All rights reserved.
   5  *
   6  * This software was developed by SRI International and the University of
   7  * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
   8  * ("CTSRD"), as part of the DARPA CRASH research programme.
   9  *
  10  * Portions of this software were developed by Andrew Turner
  11  * under sponsorship from the FreeBSD Foundation.
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  * 1. Redistributions of source code must retain the above copyright
  17  *    notice, this list of conditions and the following disclaimer.
  18  * 2. Redistributions in binary form must reproduce the above copyright
  19  *    notice, this list of conditions and the following disclaimer in the
  20  *    documentation and/or other materials provided with the distribution.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34
  35 /*
  36  * VirtIO MMIO interface.
  37  * This driver is heavily based on VirtIO PCI interface driver.
  38  */
  39
  40 #include <sys/cdefs.h>
  41 __FBSDID("$FreeBSD$");
  42
  43 #include <sys/param.h>
  44 #include <sys/systm.h>
  45 #include <sys/bus.h>
  46 #include <sys/kernel.h>
  47 #include <sys/module.h>
  48 #include <sys/malloc.h>
  49 #include <sys/rman.h>
  50 #include <sys/endian.h>
  51
  52 #include <machine/bus.h>
  53 #include <machine/resource.h>
  54
  55 #include <dev/virtio/virtio.h>
  56 #include <dev/virtio/virtqueue.h>
  57 #include <dev/virtio/mmio/virtio_mmio.h>
  58
  59 #include "virtio_mmio_if.h"
  60 #include "virtio_bus_if.h"
  61 #include "virtio_if.h"
  62
  63 #define PAGE_SHIFT      12
  64
  65 struct vtmmio_virtqueue {
  66         struct virtqueue        *vtv_vq;
  67         int                      vtv_no_intr;
  68 };
  69
  70 static int      vtmmio_detach(device_t);
  71 static int      vtmmio_suspend(device_t);
  72 static int      vtmmio_resume(device_t);
  73 static int      vtmmio_shutdown(device_t);
  74 static void     vtmmio_driver_added(device_t, driver_t *);
  75 static void     vtmmio_child_detached(device_t, device_t);
  76 static int      vtmmio_read_ivar(device_t, device_t, int, uintptr_t *);
  77 static int      vtmmio_write_ivar(device_t, device_t, int, uintptr_t);
  78 static uint64_t vtmmio_negotiate_features(device_t, uint64_t);
  79 static int      vtmmio_with_feature(device_t, uint64_t);
  80 static void     vtmmio_set_virtqueue(struct vtmmio_softc *sc,
  81                     struct virtqueue *vq, uint32_t size);
  82 static int      vtmmio_alloc_virtqueues(device_t, int, int,
  83                     struct vq_alloc_info *);
  84 static int      vtmmio_setup_intr(device_t, enum intr_type);
  85 static void     vtmmio_stop(device_t);
  86 static void     vtmmio_poll(device_t);
  87 static int      vtmmio_reinit(device_t, uint64_t);
  88 static void     vtmmio_reinit_complete(device_t);
  89 static void     vtmmio_notify_virtqueue(device_t, uint16_t);
  90 static uint8_t  vtmmio_get_status(device_t);
  91 static void     vtmmio_set_status(device_t, uint8_t);
  92 static void     vtmmio_read_dev_config(device_t, bus_size_t, void *, int);
  93 static void     vtmmio_write_dev_config(device_t, bus_size_t, void *, int);
  94 static void     vtmmio_describe_features(struct vtmmio_softc *, const char *,
  95                     uint64_t);
  96 static void     vtmmio_probe_and_attach_child(struct vtmmio_softc *);
  97 static int      vtmmio_reinit_virtqueue(struct vtmmio_softc *, int);
  98 static void     vtmmio_free_interrupts(struct vtmmio_softc *);
  99 static void     vtmmio_free_virtqueues(struct vtmmio_softc *);
 100 static void     vtmmio_release_child_resources(struct vtmmio_softc *);
 101 static void     vtmmio_reset(struct vtmmio_softc *);
 102 static void     vtmmio_select_virtqueue(struct vtmmio_softc *, int);
 103 static void     vtmmio_vq_intr(void *);
 104
 105 /*
 106  * I/O port read/write wrappers.
 107  */
 108 #define vtmmio_write_config_1(sc, o, v)                         \
 109 do {                                                            \
 110         if (sc->platform != NULL)                               \
 111                 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v));   \
 112         bus_write_1((sc)->res[0], (o), (v));                    \
 113         if (sc->platform != NULL)                               \
 114                 VIRTIO_MMIO_NOTE(sc->platform, (o), (v));       \
 115 } while (0)
 116 #define vtmmio_write_config_2(sc, o, v)                         \
 117 do {                                                            \
 118         if (sc->platform != NULL)                               \
 119                 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v));   \
 120         bus_write_2((sc)->res[0], (o), (v));                    \
 121         if (sc->platform != NULL)                               \
 122                 VIRTIO_MMIO_NOTE(sc->platform, (o), (v));       \
 123 } while (0)
 124 #define vtmmio_write_config_4(sc, o, v)                         \
 125 do {                                                            \
 126         if (sc->platform != NULL)                               \
 127                 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v));   \
 128         bus_write_4((sc)->res[0], (o), (v));                    \
 129         if (sc->platform != NULL)                               \
 130                 VIRTIO_MMIO_NOTE(sc->platform, (o), (v));       \
 131 } while (0)
 132
 133 #define vtmmio_read_config_1(sc, o) \
 134         bus_read_1((sc)->res[0], (o))
 135 #define vtmmio_read_config_2(sc, o) \
 136         bus_read_2((sc)->res[0], (o))
 137 #define vtmmio_read_config_4(sc, o) \
 138         bus_read_4((sc)->res[0], (o))
 139
 140 static device_method_t vtmmio_methods[] = {
 141         /* Device interface. */
 142         DEVMETHOD(device_attach,                  vtmmio_attach),
 143         DEVMETHOD(device_detach,                  vtmmio_detach),
 144         DEVMETHOD(device_suspend,                 vtmmio_suspend),
 145         DEVMETHOD(device_resume,                  vtmmio_resume),
 146         DEVMETHOD(device_shutdown,                vtmmio_shutdown),
 147
 148         /* Bus interface. */
 149         DEVMETHOD(bus_driver_added,               vtmmio_driver_added),
 150         DEVMETHOD(bus_child_detached,             vtmmio_child_detached),
 151         DEVMETHOD(bus_child_pnpinfo_str,          virtio_child_pnpinfo_str),
 152         DEVMETHOD(bus_read_ivar,                  vtmmio_read_ivar),
 153         DEVMETHOD(bus_write_ivar,                 vtmmio_write_ivar),
 154
 155         /* VirtIO bus interface. */
 156         DEVMETHOD(virtio_bus_negotiate_features,  vtmmio_negotiate_features),
 157         DEVMETHOD(virtio_bus_with_feature,        vtmmio_with_feature),
 158         DEVMETHOD(virtio_bus_alloc_virtqueues,    vtmmio_alloc_virtqueues),
 159         DEVMETHOD(virtio_bus_setup_intr,          vtmmio_setup_intr),
 160         DEVMETHOD(virtio_bus_stop,                vtmmio_stop),
 161         DEVMETHOD(virtio_bus_poll,                vtmmio_poll),
 162         DEVMETHOD(virtio_bus_reinit,              vtmmio_reinit),
 163         DEVMETHOD(virtio_bus_reinit_complete,     vtmmio_reinit_complete),
 164         DEVMETHOD(virtio_bus_notify_vq,           vtmmio_notify_virtqueue),
 165         DEVMETHOD(virtio_bus_read_device_config,  vtmmio_read_dev_config),
 166         DEVMETHOD(virtio_bus_write_device_config, vtmmio_write_dev_config),
 167
 168         DEVMETHOD_END
 169 };
 170
 171 DEFINE_CLASS_0(virtio_mmio, vtmmio_driver, vtmmio_methods,
 172     sizeof(struct vtmmio_softc));
 173
 174 MODULE_VERSION(virtio_mmio, 1);
 175
 176 static int
 177 vtmmio_setup_intr(device_t dev, enum intr_type type)
 178 {
 179         struct vtmmio_softc *sc;
 180         int rid;
 181         int err;
 182
 183         sc = device_get_softc(dev);
 184
 185         if (sc->platform != NULL) {
 186                 err = VIRTIO_MMIO_SETUP_INTR(sc->platform, sc->dev,
 187                                         vtmmio_vq_intr, sc);
 188                 if (err == 0) {
 189                         /* Okay we have backend-specific interrupts */
 190                         return (0);
 191                 }
 192         }
 193
 194         rid = 0;
 195         sc->res[1] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 196                 RF_ACTIVE);
 197         if (!sc->res[1]) {
 198                 device_printf(dev, "Can't allocate interrupt\n");
 199                 return (ENXIO);
 200         }
 201
 202         if (bus_setup_intr(dev, sc->res[1], type | INTR_MPSAFE,
 203                 NULL, vtmmio_vq_intr, sc, &sc->ih)) {
 204                 device_printf(dev, "Can't setup the interrupt\n");
 205                 return (ENXIO);
 206         }
 207
 208         return (0);
 209 }
 210
 211 int
 212 vtmmio_attach(device_t dev)
 213 {
 214         struct vtmmio_softc *sc;
 215         device_t child;
 216         int rid;
 217
 218         sc = device_get_softc(dev);
 219         sc->dev = dev;
 220
 221         rid = 0;
 222         sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 223                         RF_ACTIVE);
 224         if (!sc->res[0]) {
 225                 device_printf(dev, "Cannot allocate memory window.\n");
 226                 return (ENXIO);
 227         }
 228
 229         sc->vtmmio_version = vtmmio_read_config_4(sc, VIRTIO_MMIO_VERSION);
 230         if (sc->vtmmio_version < 1 || sc->vtmmio_version > 2) {
 231                 device_printf(dev, "Unsupported version: %x\n",
 232                     sc->vtmmio_version);
 233                 bus_release_resource(dev, SYS_RES_MEMORY, 0,
 234                     sc->res[0]);
 235                 sc->res[0] = NULL;
 236                 return (ENXIO);
 237         }
 238
 239         vtmmio_reset(sc);
 240
 241         /* Tell the host we've noticed this device. */
 242         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
 243
 244         if ((child = device_add_child(dev, NULL, -1)) == NULL) {
 245                 device_printf(dev, "Cannot create child device.\n");
 246                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
 247                 vtmmio_detach(dev);
 248                 return (ENOMEM);
 249         }
 250
 251         sc->vtmmio_child_dev = child;
 252         vtmmio_probe_and_attach_child(sc);
 253
 254         return (0);
 255 }
 256
 257 static int
 258 vtmmio_detach(device_t dev)
 259 {
 260         struct vtmmio_softc *sc;
 261         device_t child;
 262         int error;
 263
 264         sc = device_get_softc(dev);
 265
 266         if ((child = sc->vtmmio_child_dev) != NULL) {
 267                 error = device_delete_child(dev, child);
 268                 if (error)
 269                         return (error);
 270                 sc->vtmmio_child_dev = NULL;
 271         }
 272
 273         vtmmio_reset(sc);
 274
 275         if (sc->res[0] != NULL) {
 276                 bus_release_resource(dev, SYS_RES_MEMORY, 0,
 277                     sc->res[0]);
 278                 sc->res[0] = NULL;
 279         }
 280
 281         return (0);
 282 }
 283
 284 static int
 285 vtmmio_suspend(device_t dev)
 286 {
 287
 288         return (bus_generic_suspend(dev));
 289 }
 290
 291 static int
 292 vtmmio_resume(device_t dev)
 293 {
 294
 295         return (bus_generic_resume(dev));
 296 }
 297
 298 static int
 299 vtmmio_shutdown(device_t dev)
 300 {
 301
 302         (void) bus_generic_shutdown(dev);
 303
 304         /* Forcibly stop the host device. */
 305         vtmmio_stop(dev);
 306
 307         return (0);
 308 }
 309
 310 static void
 311 vtmmio_driver_added(device_t dev, driver_t *driver)
 312 {
 313         struct vtmmio_softc *sc;
 314
 315         sc = device_get_softc(dev);
 316
 317         vtmmio_probe_and_attach_child(sc);
 318 }
 319
 320 static void
 321 vtmmio_child_detached(device_t dev, device_t child)
 322 {
 323         struct vtmmio_softc *sc;
 324
 325         sc = device_get_softc(dev);
 326
 327         vtmmio_reset(sc);
 328         vtmmio_release_child_resources(sc);
 329 }
 330
 331 static int
 332 vtmmio_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
 333 {
 334         struct vtmmio_softc *sc;
 335
 336         sc = device_get_softc(dev);
 337
 338         if (sc->vtmmio_child_dev != child)
 339                 return (ENOENT);
 340
 341         switch (index) {
 342         case VIRTIO_IVAR_DEVTYPE:
 343         case VIRTIO_IVAR_SUBDEVICE:
 344                 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_DEVICE_ID);
 345                 break;
 346         case VIRTIO_IVAR_VENDOR:
 347                 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_VENDOR_ID);
 348                 break;
 349         case VIRTIO_IVAR_SUBVENDOR:
 350         case VIRTIO_IVAR_DEVICE:
 351                 /*
 352                  * Dummy value for fields not present in this bus.  Used by
 353                  * bus-agnostic virtio_child_pnpinfo_str.
 354                  */
 355                 *result = 0;
 356                 break;
 357         default:
 358                 return (ENOENT);
 359         }
 360
 361         return (0);
 362 }
 363
 364 static int
 365 vtmmio_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
 366 {
 367         struct vtmmio_softc *sc;
 368
 369         sc = device_get_softc(dev);
 370
 371         if (sc->vtmmio_child_dev != child)
 372                 return (ENOENT);
 373
 374         switch (index) {
 375         case VIRTIO_IVAR_FEATURE_DESC:
 376                 sc->vtmmio_child_feat_desc = (void *) value;
 377                 break;
 378         default:
 379                 return (ENOENT);
 380         }
 381
 382         return (0);
 383 }
 384
 385 static uint64_t
 386 vtmmio_negotiate_features(device_t dev, uint64_t child_features)
 387 {
 388         struct vtmmio_softc *sc;
 389         uint64_t host_features, features;
 390
 391         sc = device_get_softc(dev);
 392
 393         vtmmio_write_config_4(sc, VIRTIO_MMIO_HOST_FEATURES_SEL, 1);
 394         host_features = vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES);
 395         host_features <<= 32;
 396
 397         vtmmio_write_config_4(sc, VIRTIO_MMIO_HOST_FEATURES_SEL, 0);
 398         host_features |= vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES);
 399
 400         vtmmio_describe_features(sc, "host", host_features);
 401
 402         /*
 403          * Limit negotiated features to what the driver, virtqueue, and
 404          * host all support.
 405          */
 406         features = host_features & child_features;
 407         features = virtqueue_filter_features(features);
 408         sc->vtmmio_features = features;
 409
 410         vtmmio_describe_features(sc, "negotiated", features);
 411
 412         vtmmio_write_config_4(sc, VIRTIO_MMIO_HOST_FEATURES_SEL, 1);
 413         vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features >> 32);
 414
 415         vtmmio_write_config_4(sc, VIRTIO_MMIO_HOST_FEATURES_SEL, 0);
 416         vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features);
 417
 418         return (features);
 419 }
 420
 421 static int
 422 vtmmio_with_feature(device_t dev, uint64_t feature)
 423 {
 424         struct vtmmio_softc *sc;
 425
 426         sc = device_get_softc(dev);
 427
 428         return ((sc->vtmmio_features & feature) != 0);
 429 }
 430
 431 static void
 432 vtmmio_set_virtqueue(struct vtmmio_softc *sc, struct virtqueue *vq,
 433     uint32_t size)
 434 {
 435         vm_paddr_t paddr;
 436
 437         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size);
 438 #if 0
 439         device_printf(dev, "virtqueue paddr 0x%08lx\n",
 440             (uint64_t)paddr);
 441 #endif
 442         if (sc->vtmmio_version == 1) {
 443                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN,
 444                     VIRTIO_MMIO_VRING_ALIGN);
 445                 paddr = virtqueue_paddr(vq);
 446                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN,
 447                     paddr >> PAGE_SHIFT);
 448         } else {
 449                 paddr = virtqueue_desc_paddr(vq);
 450                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_DESC_LOW,
 451                     paddr);
 452                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_DESC_HIGH,
 453                     ((uint64_t)paddr) >> 32);
 454
 455                 paddr = virtqueue_avail_paddr(vq);
 456                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_AVAIL_LOW,
 457                     paddr);
 458                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_AVAIL_HIGH,
 459                     ((uint64_t)paddr) >> 32);
 460
 461                 paddr = virtqueue_used_paddr(vq);
 462                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_USED_LOW,
 463                     paddr);
 464                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_USED_HIGH,
 465                     ((uint64_t)paddr) >> 32);
 466
 467                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_READY, 1);
 468         }
 469 }
 470
 471 static int
 472 vtmmio_alloc_virtqueues(device_t dev, int flags, int nvqs,
 473     struct vq_alloc_info *vq_info)
 474 {
 475         struct vtmmio_virtqueue *vqx;
 476         struct vq_alloc_info *info;
 477         struct vtmmio_softc *sc;
 478         struct virtqueue *vq;
 479         uint32_t size;
 480         int idx, error;
 481
 482         sc = device_get_softc(dev);
 483
 484         if (sc->vtmmio_nvqs != 0)
 485                 return (EALREADY);
 486         if (nvqs <= 0)
 487                 return (EINVAL);
 488
 489         sc->vtmmio_vqs = malloc(nvqs * sizeof(struct vtmmio_virtqueue),
 490             M_DEVBUF, M_NOWAIT | M_ZERO);
 491         if (sc->vtmmio_vqs == NULL)
 492                 return (ENOMEM);
 493
 494         vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE,
 495             (1 << PAGE_SHIFT));
 496
 497         for (idx = 0; idx < nvqs; idx++) {
 498                 vqx = &sc->vtmmio_vqs[idx];
 499                 info = &vq_info[idx];
 500
 501                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx);
 502
 503                 vtmmio_select_virtqueue(sc, idx);
 504                 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX);
 505
 506                 error = virtqueue_alloc(dev, idx, size,
 507                     VIRTIO_MMIO_VRING_ALIGN, ~(vm_paddr_t)0, info, &vq);
 508                 if (error) {
 509                         device_printf(dev,
 510                             "cannot allocate virtqueue %d: %d\n",
 511                             idx, error);
 512                         break;
 513                 }
 514
 515                 vtmmio_set_virtqueue(sc, vq, size);
 516
 517                 vqx->vtv_vq = *info->vqai_vq = vq;
 518                 vqx->vtv_no_intr = info->vqai_intr == NULL;
 519
 520                 sc->vtmmio_nvqs++;
 521         }
 522
 523         if (error)
 524                 vtmmio_free_virtqueues(sc);
 525
 526         return (error);
 527 }
 528
 529 static void
 530 vtmmio_stop(device_t dev)
 531 {
 532
 533         vtmmio_reset(device_get_softc(dev));
 534 }
 535
 536 static void
 537 vtmmio_poll(device_t dev)
 538 {
 539         struct vtmmio_softc *sc;
 540
 541         sc = device_get_softc(dev);
 542
 543         if (sc->platform != NULL)
 544                 VIRTIO_MMIO_POLL(sc->platform);
 545 }
 546
 547 static int
 548 vtmmio_reinit(device_t dev, uint64_t features)
 549 {
 550         struct vtmmio_softc *sc;
 551         int idx, error;
 552
 553         sc = device_get_softc(dev);
 554
 555         if (vtmmio_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
 556                 vtmmio_stop(dev);
 557
 558         /*
 559          * Quickly drive the status through ACK and DRIVER. The device
 560          * does not become usable again until vtmmio_reinit_complete().
 561          */
 562         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
 563         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
 564
 565         vtmmio_negotiate_features(dev, features);
 566
 567         vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE,
 568             (1 << PAGE_SHIFT));
 569
 570         for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
 571                 error = vtmmio_reinit_virtqueue(sc, idx);
 572                 if (error)
 573                         return (error);
 574         }
 575
 576         return (0);
 577 }
 578
 579 static void
 580 vtmmio_reinit_complete(device_t dev)
 581 {
 582
 583         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
 584 }
 585
 586 static void
 587 vtmmio_notify_virtqueue(device_t dev, uint16_t queue)
 588 {
 589         struct vtmmio_softc *sc;
 590
 591         sc = device_get_softc(dev);
 592
 593         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NOTIFY, queue);
 594 }
 595
 596 static uint8_t
 597 vtmmio_get_status(device_t dev)
 598 {
 599         struct vtmmio_softc *sc;
 600
 601         sc = device_get_softc(dev);
 602
 603         return (vtmmio_read_config_4(sc, VIRTIO_MMIO_STATUS));
 604 }
 605
 606 static void
 607 vtmmio_set_status(device_t dev, uint8_t status)
 608 {
 609         struct vtmmio_softc *sc;
 610
 611         sc = device_get_softc(dev);
 612
 613         if (status != VIRTIO_CONFIG_STATUS_RESET)
 614                 status |= vtmmio_get_status(dev);
 615
 616         vtmmio_write_config_4(sc, VIRTIO_MMIO_STATUS, status);
 617 }
 618
 619 static void
 620 vtmmio_read_dev_config(device_t dev, bus_size_t offset,
 621     void *dst, int length)
 622 {
 623         struct vtmmio_softc *sc;
 624         bus_size_t off;
 625         uint8_t *d;
 626         int size;
 627         uint64_t low32, high32;
 628
 629         sc = device_get_softc(dev);
 630         off = VIRTIO_MMIO_CONFIG + offset;
 631
 632         /*
 633          * The non-legacy MMIO specification adds the following restriction:
 634          *
 635          *   4.2.2.2: For the device-specific configuration space, the driver
 636          *   MUST use 8 bit wide accesses for 8 bit wide fields, 16 bit wide
 637          *   and aligned accesses for 16 bit wide fields and 32 bit wide and
 638          *   aligned accesses for 32 and 64 bit wide fields.
 639          *
 640          * The endianness also varies between non-legacy and legacy:
 641          *
 642          *   2.4: Note: The device configuration space uses the little-endian
 643          *   format for multi-byte fields.
 644          *
 645          *   2.4.3: Note that for legacy interfaces, device configuration space
 646          *   is generally the guest’s native endian, rather than PCI’s
 647          *   little-endian. The correct endian-ness is documented for each
 648          *   device.
 649          */
 650         if (sc->vtmmio_version > 1) {
 651                 switch (length) {
 652                 case 1:
 653                         *(uint8_t *)dst = vtmmio_read_config_1(sc, off);
 654                         break;
 655                 case 2:
 656                         *(uint16_t *)dst =
 657                             le16toh(vtmmio_read_config_2(sc, off));
 658                         break;
 659                 case 4:
 660                         *(uint32_t *)dst =
 661                             le32toh(vtmmio_read_config_4(sc, off));
 662                         break;
 663                 case 8:
 664                         low32 = le32toh(vtmmio_read_config_4(sc, off));
 665                         high32 = le32toh(vtmmio_read_config_4(sc, off + 4));
 666                         *(uint64_t *)dst = (high32 << 32) | low32;
 667                         break;
 668                 default:
 669                         panic("%s: invalid length %d\n", __func__, length);
 670                 }
 671
 672                 return;
 673         }
 674
 675         for (d = dst; length > 0; d += size, off += size, length -= size) {
 676 #ifdef ALLOW_WORD_ALIGNED_ACCESS
 677                 if (length >= 4) {
 678                         size = 4;
 679                         *(uint32_t *)d = vtmmio_read_config_4(sc, off);
 680                 } else if (length >= 2) {
 681                         size = 2;
 682                         *(uint16_t *)d = vtmmio_read_config_2(sc, off);
 683                 } else
 684 #endif
 685                 {
 686                         size = 1;
 687                         *d = vtmmio_read_config_1(sc, off);
 688                 }
 689         }
 690 }
 691
 692 static void
 693 vtmmio_write_dev_config(device_t dev, bus_size_t offset,
 694     void *src, int length)
 695 {
 696         struct vtmmio_softc *sc;
 697         bus_size_t off;
 698         uint8_t *s;
 699         int size;
 700
 701         sc = device_get_softc(dev);
 702         off = VIRTIO_MMIO_CONFIG + offset;
 703
 704         /*
 705          * The non-legacy MMIO specification adds size and alignment
 706          * restrctions. It also changes the endianness from native-endian to
 707          * little-endian. See vtmmio_read_dev_config.
 708          */
 709         if (sc->vtmmio_version > 1) {
 710                 switch (length) {
 711                 case 1:
 712                         vtmmio_write_config_1(sc, off, *(uint8_t *)src);
 713                         break;
 714                 case 2:
 715                         vtmmio_write_config_2(sc, off,
 716                             htole16(*(uint16_t *)src));
 717                         break;
 718                 case 4:
 719                         vtmmio_write_config_4(sc, off,
 720                             htole32(*(uint32_t *)src));
 721                         break;
 722                 case 8:
 723                         vtmmio_write_config_4(sc, off,
 724                             htole32(*(uint64_t *)src));
 725                         vtmmio_write_config_4(sc, off + 4,
 726                             htole32((*(uint64_t *)src) >> 32));
 727                         break;
 728                 default:
 729                         panic("%s: invalid length %d\n", __func__, length);
 730                 }
 731
 732                 return;
 733         }
 734
 735         for (s = src; length > 0; s += size, off += size, length -= size) {
 736 #ifdef ALLOW_WORD_ALIGNED_ACCESS
 737                 if (length >= 4) {
 738                         size = 4;
 739                         vtmmio_write_config_4(sc, off, *(uint32_t *)s);
 740                 } else if (length >= 2) {
 741                         size = 2;
 742                         vtmmio_write_config_2(sc, off, *(uint16_t *)s);
 743                 } else
 744 #endif
 745                 {
 746                         size = 1;
 747                         vtmmio_write_config_1(sc, off, *s);
 748                 }
 749         }
 750 }
 751
 752 static void
 753 vtmmio_describe_features(struct vtmmio_softc *sc, const char *msg,
 754     uint64_t features)
 755 {
 756         device_t dev, child;
 757
 758         dev = sc->dev;
 759         child = sc->vtmmio_child_dev;
 760
 761         if (device_is_attached(child) || bootverbose == 0)
 762                 return;
 763
 764         virtio_describe(dev, msg, features, sc->vtmmio_child_feat_desc);
 765 }
 766
 767 static void
 768 vtmmio_probe_and_attach_child(struct vtmmio_softc *sc)
 769 {
 770         device_t dev, child;
 771
 772         dev = sc->dev;
 773         child = sc->vtmmio_child_dev;
 774
 775         if (child == NULL)
 776                 return;
 777
 778         if (device_get_state(child) != DS_NOTPRESENT) {
 779                 return;
 780         }
 781
 782         if (device_probe(child) != 0) {
 783                 return;
 784         }
 785
 786         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
 787         if (device_attach(child) != 0) {
 788                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
 789                 vtmmio_reset(sc);
 790                 vtmmio_release_child_resources(sc);
 791                 /* Reset status for future attempt. */
 792                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
 793         } else {
 794                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
 795                 VIRTIO_ATTACH_COMPLETED(child);
 796         }
 797 }
 798
 799 static int
 800 vtmmio_reinit_virtqueue(struct vtmmio_softc *sc, int idx)
 801 {
 802         struct vtmmio_virtqueue *vqx;
 803         struct virtqueue *vq;
 804         int error;
 805         uint16_t size;
 806
 807         vqx = &sc->vtmmio_vqs[idx];
 808         vq = vqx->vtv_vq;
 809
 810         KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));
 811
 812         vtmmio_select_virtqueue(sc, idx);
 813         size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX);
 814
 815         error = virtqueue_reinit(vq, size);
 816         if (error)
 817                 return (error);
 818
 819         vtmmio_set_virtqueue(sc, vq, size);
 820
 821         return (0);
 822 }
 823
 824 static void
 825 vtmmio_free_interrupts(struct vtmmio_softc *sc)
 826 {
 827
 828         if (sc->ih != NULL)
 829                 bus_teardown_intr(sc->dev, sc->res[1], sc->ih);
 830
 831         if (sc->res[1] != NULL)
 832                 bus_release_resource(sc->dev, SYS_RES_IRQ, 0, sc->res[1]);
 833 }
 834
 835 static void
 836 vtmmio_free_virtqueues(struct vtmmio_softc *sc)
 837 {
 838         struct vtmmio_virtqueue *vqx;
 839         int idx;
 840
 841         for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
 842                 vqx = &sc->vtmmio_vqs[idx];
 843
 844                 vtmmio_select_virtqueue(sc, idx);
 845                 if (sc->vtmmio_version == 1)
 846                         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 0);
 847                 else
 848                         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_READY, 0);
 849
 850                 virtqueue_free(vqx->vtv_vq);
 851                 vqx->vtv_vq = NULL;
 852         }
 853
 854         free(sc->vtmmio_vqs, M_DEVBUF);
 855         sc->vtmmio_vqs = NULL;
 856         sc->vtmmio_nvqs = 0;
 857 }
 858
 859 static void
 860 vtmmio_release_child_resources(struct vtmmio_softc *sc)
 861 {
 862
 863         vtmmio_free_interrupts(sc);
 864         vtmmio_free_virtqueues(sc);
 865 }
 866
 867 static void
 868 vtmmio_reset(struct vtmmio_softc *sc)
 869 {
 870
 871         /*
 872          * Setting the status to RESET sets the host device to
 873          * the original, uninitialized state.
 874          */
 875         vtmmio_set_status(sc->dev, VIRTIO_CONFIG_STATUS_RESET);
 876 }
 877
 878 static void
 879 vtmmio_select_virtqueue(struct vtmmio_softc *sc, int idx)
 880 {
 881
 882         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx);
 883 }
 884
 885 static void
 886 vtmmio_vq_intr(void *arg)
 887 {
 888         struct vtmmio_virtqueue *vqx;
 889         struct vtmmio_softc *sc;
 890         struct virtqueue *vq;
 891         uint32_t status;
 892         int idx;
 893
 894         sc = arg;
 895
 896         status = vtmmio_read_config_4(sc, VIRTIO_MMIO_INTERRUPT_STATUS);
 897         vtmmio_write_config_4(sc, VIRTIO_MMIO_INTERRUPT_ACK, status);
 898
 899         /* The config changed */
 900         if (status & VIRTIO_MMIO_INT_CONFIG)
 901                 if (sc->vtmmio_child_dev != NULL)
 902                         VIRTIO_CONFIG_CHANGE(sc->vtmmio_child_dev);
 903
 904         /* Notify all virtqueues. */
 905         if (status & VIRTIO_MMIO_INT_VRING) {
 906                 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
 907                         vqx = &sc->vtmmio_vqs[idx];
 908                         if (vqx->vtv_no_intr == 0) {
 909                                 vq = vqx->vtv_vq;
 910                                 virtqueue_intr(vq);
 911                         }
 912                 }
 913         }
 914 }