sys/dev/virtio/mmio/virtio_mmio.c

   1 /*-
   2  * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com>
   3  * Copyright (c) 2014 The FreeBSD Foundation
   4  * All rights reserved.
   5  *
   6  * This software was developed by SRI International and the University of
   7  * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
   8  * ("CTSRD"), as part of the DARPA CRASH research programme.
   9  *
  10  * Portions of this software were developed by Andrew Turner
  11  * under sponsorship from the FreeBSD Foundation.
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  * 1. Redistributions of source code must retain the above copyright
  17  *    notice, this list of conditions and the following disclaimer.
  18  * 2. Redistributions in binary form must reproduce the above copyright
  19  *    notice, this list of conditions and the following disclaimer in the
  20  *    documentation and/or other materials provided with the distribution.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34
  35 /*
  36  * VirtIO MMIO interface.
  37  * This driver is heavily based on VirtIO PCI interface driver.
  38  */
  39
  40 #include <sys/cdefs.h>
  41 __FBSDID("$FreeBSD$");
  42
  43 #include <sys/param.h>
  44 #include <sys/systm.h>
  45 #include <sys/bus.h>
  46 #include <sys/kernel.h>
  47 #include <sys/module.h>
  48 #include <sys/malloc.h>
  49 #include <sys/rman.h>
  50 #include <sys/endian.h>
  51
  52 #include <machine/bus.h>
  53 #include <machine/resource.h>
  54
  55 #include <dev/virtio/virtio.h>
  56 #include <dev/virtio/virtqueue.h>
  57 #include <dev/virtio/mmio/virtio_mmio.h>
  58
  59 #include "virtio_mmio_if.h"
  60 #include "virtio_bus_if.h"
  61 #include "virtio_if.h"
  62
  63 #define PAGE_SHIFT      12
  64
  65 struct vtmmio_virtqueue {
  66         struct virtqueue        *vtv_vq;
  67         int                      vtv_no_intr;
  68 };
  69
  70 static int      vtmmio_detach(device_t);
  71 static int      vtmmio_suspend(device_t);
  72 static int      vtmmio_resume(device_t);
  73 static int      vtmmio_shutdown(device_t);
  74 static void     vtmmio_driver_added(device_t, driver_t *);
  75 static void     vtmmio_child_detached(device_t, device_t);
  76 static int      vtmmio_read_ivar(device_t, device_t, int, uintptr_t *);
  77 static int      vtmmio_write_ivar(device_t, device_t, int, uintptr_t);
  78 static uint64_t vtmmio_negotiate_features(device_t, uint64_t);
  79 static int      vtmmio_with_feature(device_t, uint64_t);
  80 static void     vtmmio_set_virtqueue(struct vtmmio_softc *sc,
  81                     struct virtqueue *vq, uint32_t size);
  82 static int      vtmmio_alloc_virtqueues(device_t, int, int,
  83                     struct vq_alloc_info *);
  84 static int      vtmmio_setup_intr(device_t, enum intr_type);
  85 static void     vtmmio_stop(device_t);
  86 static void     vtmmio_poll(device_t);
  87 static int      vtmmio_reinit(device_t, uint64_t);
  88 static void     vtmmio_reinit_complete(device_t);
  89 static void     vtmmio_notify_virtqueue(device_t, uint16_t);
  90 static uint8_t  vtmmio_get_status(device_t);
  91 static void     vtmmio_set_status(device_t, uint8_t);
  92 static void     vtmmio_read_dev_config(device_t, bus_size_t, void *, int);
  93 static void     vtmmio_write_dev_config(device_t, bus_size_t, void *, int);
  94 static void     vtmmio_describe_features(struct vtmmio_softc *, const char *,
  95                     uint64_t);
  96 static void     vtmmio_probe_and_attach_child(struct vtmmio_softc *);
  97 static int      vtmmio_reinit_virtqueue(struct vtmmio_softc *, int);
  98 static void     vtmmio_free_interrupts(struct vtmmio_softc *);
  99 static void     vtmmio_free_virtqueues(struct vtmmio_softc *);
 100 static void     vtmmio_release_child_resources(struct vtmmio_softc *);
 101 static void     vtmmio_reset(struct vtmmio_softc *);
 102 static void     vtmmio_select_virtqueue(struct vtmmio_softc *, int);
 103 static void     vtmmio_vq_intr(void *);
 104
 105 /*
 106  * I/O port read/write wrappers.
 107  */
 108 #define vtmmio_write_config_1(sc, o, v)                         \
 109 do {                                                            \
 110         if (sc->platform != NULL)                               \
 111                 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v));   \
 112         bus_write_1((sc)->res[0], (o), (v));                    \
 113         if (sc->platform != NULL)                               \
 114                 VIRTIO_MMIO_NOTE(sc->platform, (o), (v));       \
 115 } while (0)
 116 #define vtmmio_write_config_2(sc, o, v)                         \
 117 do {                                                            \
 118         if (sc->platform != NULL)                               \
 119                 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v));   \
 120         bus_write_2((sc)->res[0], (o), (v));                    \
 121         if (sc->platform != NULL)                               \
 122                 VIRTIO_MMIO_NOTE(sc->platform, (o), (v));       \
 123 } while (0)
 124 #define vtmmio_write_config_4(sc, o, v)                         \
 125 do {                                                            \
 126         if (sc->platform != NULL)                               \
 127                 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v));   \
 128         bus_write_4((sc)->res[0], (o), (v));                    \
 129         if (sc->platform != NULL)                               \
 130                 VIRTIO_MMIO_NOTE(sc->platform, (o), (v));       \
 131 } while (0)
 132
 133 #define vtmmio_read_config_1(sc, o) \
 134         bus_read_1((sc)->res[0], (o))
 135 #define vtmmio_read_config_2(sc, o) \
 136         bus_read_2((sc)->res[0], (o))
 137 #define vtmmio_read_config_4(sc, o) \
 138         bus_read_4((sc)->res[0], (o))
 139
 140 static device_method_t vtmmio_methods[] = {
 141         /* Device interface. */
 142         DEVMETHOD(device_attach,                  vtmmio_attach),
 143         DEVMETHOD(device_detach,                  vtmmio_detach),
 144         DEVMETHOD(device_suspend,                 vtmmio_suspend),
 145         DEVMETHOD(device_resume,                  vtmmio_resume),
 146         DEVMETHOD(device_shutdown,                vtmmio_shutdown),
 147
 148         /* Bus interface. */
 149         DEVMETHOD(bus_driver_added,               vtmmio_driver_added),
 150         DEVMETHOD(bus_child_detached,             vtmmio_child_detached),
 151         DEVMETHOD(bus_child_pnpinfo_str,          virtio_child_pnpinfo_str),
 152         DEVMETHOD(bus_read_ivar,                  vtmmio_read_ivar),
 153         DEVMETHOD(bus_write_ivar,                 vtmmio_write_ivar),
 154
 155         /* VirtIO bus interface. */
 156         DEVMETHOD(virtio_bus_negotiate_features,  vtmmio_negotiate_features),
 157         DEVMETHOD(virtio_bus_with_feature,        vtmmio_with_feature),
 158         DEVMETHOD(virtio_bus_alloc_virtqueues,    vtmmio_alloc_virtqueues),
 159         DEVMETHOD(virtio_bus_setup_intr,          vtmmio_setup_intr),
 160         DEVMETHOD(virtio_bus_stop,                vtmmio_stop),
 161         DEVMETHOD(virtio_bus_poll,                vtmmio_poll),
 162         DEVMETHOD(virtio_bus_reinit,              vtmmio_reinit),
 163         DEVMETHOD(virtio_bus_reinit_complete,     vtmmio_reinit_complete),
 164         DEVMETHOD(virtio_bus_notify_vq,           vtmmio_notify_virtqueue),
 165         DEVMETHOD(virtio_bus_read_device_config,  vtmmio_read_dev_config),
 166         DEVMETHOD(virtio_bus_write_device_config, vtmmio_write_dev_config),
 167
 168         DEVMETHOD_END
 169 };
 170
 171 DEFINE_CLASS_0(virtio_mmio, vtmmio_driver, vtmmio_methods,
 172     sizeof(struct vtmmio_softc));
 173
 174 MODULE_VERSION(virtio_mmio, 1);
 175
 176 static int
 177 vtmmio_setup_intr(device_t dev, enum intr_type type)
 178 {
 179         struct vtmmio_softc *sc;
 180         int rid;
 181         int err;
 182
 183         sc = device_get_softc(dev);
 184
 185         if (sc->platform != NULL) {
 186                 err = VIRTIO_MMIO_SETUP_INTR(sc->platform, sc->dev,
 187                                         vtmmio_vq_intr, sc);
 188                 if (err == 0) {
 189                         /* Okay we have backend-specific interrupts */
 190                         return (0);
 191                 }
 192         }
 193
 194         rid = 0;
 195         sc->res[1] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 196                 RF_ACTIVE);
 197         if (!sc->res[1]) {
 198                 device_printf(dev, "Can't allocate interrupt\n");
 199                 return (ENXIO);
 200         }
 201
 202         if (bus_setup_intr(dev, sc->res[1], type | INTR_MPSAFE,
 203                 NULL, vtmmio_vq_intr, sc, &sc->ih)) {
 204                 device_printf(dev, "Can't setup the interrupt\n");
 205                 return (ENXIO);
 206         }
 207
 208         return (0);
 209 }
 210
 211 int
 212 vtmmio_attach(device_t dev)
 213 {
 214         struct vtmmio_softc *sc;
 215         device_t child;
 216         int rid;
 217
 218         sc = device_get_softc(dev);
 219         sc->dev = dev;
 220
 221         rid = 0;
 222         sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 223                         RF_ACTIVE);
 224         if (!sc->res[0]) {
 225                 device_printf(dev, "Cannot allocate memory window.\n");
 226                 return (ENXIO);
 227         }
 228
 229         sc->vtmmio_version = vtmmio_read_config_4(sc, VIRTIO_MMIO_VERSION);
 230         if (sc->vtmmio_version < 1 || sc->vtmmio_version > 2) {
 231                 device_printf(dev, "Unsupported version: %x\n",
 232                     sc->vtmmio_version);
 233                 bus_release_resource(dev, SYS_RES_MEMORY, 0,
 234                     sc->res[0]);
 235                 sc->res[0] = NULL;
 236                 return (ENXIO);
 237         }
 238
 239         vtmmio_reset(sc);
 240
 241         /* Tell the host we've noticed this device. */
 242         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
 243
 244         if ((child = device_add_child(dev, NULL, -1)) == NULL) {
 245                 device_printf(dev, "Cannot create child device.\n");
 246                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
 247                 vtmmio_detach(dev);
 248                 return (ENOMEM);
 249         }
 250
 251         sc->vtmmio_child_dev = child;
 252         vtmmio_probe_and_attach_child(sc);
 253
 254         return (0);
 255 }
 256
 257 static int
 258 vtmmio_detach(device_t dev)
 259 {
 260         struct vtmmio_softc *sc;
 261         device_t child;
 262         int error;
 263
 264         sc = device_get_softc(dev);
 265
 266         if ((child = sc->vtmmio_child_dev) != NULL) {
 267                 error = device_delete_child(dev, child);
 268                 if (error)
 269                         return (error);
 270                 sc->vtmmio_child_dev = NULL;
 271         }
 272
 273         vtmmio_reset(sc);
 274
 275         if (sc->res[0] != NULL) {
 276                 bus_release_resource(dev, SYS_RES_MEMORY, 0,
 277                     sc->res[0]);
 278                 sc->res[0] = NULL;
 279         }
 280
 281         return (0);
 282 }
 283
 284 static int
 285 vtmmio_suspend(device_t dev)
 286 {
 287
 288         return (bus_generic_suspend(dev));
 289 }
 290
 291 static int
 292 vtmmio_resume(device_t dev)
 293 {
 294
 295         return (bus_generic_resume(dev));
 296 }
 297
 298 static int
 299 vtmmio_shutdown(device_t dev)
 300 {
 301
 302         (void) bus_generic_shutdown(dev);
 303
 304         /* Forcibly stop the host device. */
 305         vtmmio_stop(dev);
 306
 307         return (0);
 308 }
 309
 310 static void
 311 vtmmio_driver_added(device_t dev, driver_t *driver)
 312 {
 313         struct vtmmio_softc *sc;
 314
 315         sc = device_get_softc(dev);
 316
 317         vtmmio_probe_and_attach_child(sc);
 318 }
 319
 320 static void
 321 vtmmio_child_detached(device_t dev, device_t child)
 322 {
 323         struct vtmmio_softc *sc;
 324
 325         sc = device_get_softc(dev);
 326
 327         vtmmio_reset(sc);
 328         vtmmio_release_child_resources(sc);
 329 }
 330
 331 static int
 332 vtmmio_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
 333 {
 334         struct vtmmio_softc *sc;
 335
 336         sc = device_get_softc(dev);
 337
 338         if (sc->vtmmio_child_dev != child)
 339                 return (ENOENT);
 340
 341         switch (index) {
 342         case VIRTIO_IVAR_DEVTYPE:
 343         case VIRTIO_IVAR_SUBDEVICE:
 344                 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_DEVICE_ID);
 345                 break;
 346         case VIRTIO_IVAR_VENDOR:
 347                 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_VENDOR_ID);
 348                 break;
 349         case VIRTIO_IVAR_SUBVENDOR:
 350         case VIRTIO_IVAR_DEVICE:
 351                 /*
 352                  * Dummy value for fields not present in this bus.  Used by
 353                  * bus-agnostic virtio_child_pnpinfo_str.
 354                  */
 355                 *result = 0;
 356                 break;
 357         default:
 358                 return (ENOENT);
 359         }
 360
 361         return (0);
 362 }
 363
 364 static int
 365 vtmmio_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
 366 {
 367         struct vtmmio_softc *sc;
 368
 369         sc = device_get_softc(dev);
 370
 371         if (sc->vtmmio_child_dev != child)
 372                 return (ENOENT);
 373
 374         switch (index) {
 375         case VIRTIO_IVAR_FEATURE_DESC:
 376                 sc->vtmmio_child_feat_desc = (void *) value;
 377                 break;
 378         default:
 379                 return (ENOENT);
 380         }
 381
 382         return (0);
 383 }
 384
 385 static uint64_t
 386 vtmmio_negotiate_features(device_t dev, uint64_t child_features)
 387 {
 388         struct vtmmio_softc *sc;
 389         uint64_t host_features, features;
 390
 391         sc = device_get_softc(dev);
 392
 393         host_features = vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES);
 394         vtmmio_describe_features(sc, "host", host_features);
 395
 396         /*
 397          * Limit negotiated features to what the driver, virtqueue, and
 398          * host all support.
 399          */
 400         features = host_features & child_features;
 401         features = virtqueue_filter_features(features);
 402         sc->vtmmio_features = features;
 403
 404         vtmmio_describe_features(sc, "negotiated", features);
 405         vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features);
 406
 407         return (features);
 408 }
 409
 410 static int
 411 vtmmio_with_feature(device_t dev, uint64_t feature)
 412 {
 413         struct vtmmio_softc *sc;
 414
 415         sc = device_get_softc(dev);
 416
 417         return ((sc->vtmmio_features & feature) != 0);
 418 }
 419
 420 static void
 421 vtmmio_set_virtqueue(struct vtmmio_softc *sc, struct virtqueue *vq,
 422     uint32_t size)
 423 {
 424         vm_paddr_t paddr;
 425
 426         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size);
 427 #if 0
 428         device_printf(dev, "virtqueue paddr 0x%08lx\n",
 429             (uint64_t)paddr);
 430 #endif
 431         if (sc->vtmmio_version == 1) {
 432                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN,
 433                     VIRTIO_MMIO_VRING_ALIGN);
 434                 paddr = virtqueue_paddr(vq);
 435                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN,
 436                     paddr >> PAGE_SHIFT);
 437         } else {
 438                 paddr = virtqueue_desc_paddr(vq);
 439                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_DESC_LOW,
 440                     paddr);
 441                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_DESC_HIGH,
 442                     ((uint64_t)paddr) >> 32);
 443
 444                 paddr = virtqueue_avail_paddr(vq);
 445                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_AVAIL_LOW,
 446                     paddr);
 447                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_AVAIL_HIGH,
 448                     ((uint64_t)paddr) >> 32);
 449
 450                 paddr = virtqueue_used_paddr(vq);
 451                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_USED_LOW,
 452                     paddr);
 453                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_USED_HIGH,
 454                     ((uint64_t)paddr) >> 32);
 455
 456                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_READY, 1);
 457         }
 458 }
 459
 460 static int
 461 vtmmio_alloc_virtqueues(device_t dev, int flags, int nvqs,
 462     struct vq_alloc_info *vq_info)
 463 {
 464         struct vtmmio_virtqueue *vqx;
 465         struct vq_alloc_info *info;
 466         struct vtmmio_softc *sc;
 467         struct virtqueue *vq;
 468         uint32_t size;
 469         int idx, error;
 470
 471         sc = device_get_softc(dev);
 472
 473         if (sc->vtmmio_nvqs != 0)
 474                 return (EALREADY);
 475         if (nvqs <= 0)
 476                 return (EINVAL);
 477
 478         sc->vtmmio_vqs = malloc(nvqs * sizeof(struct vtmmio_virtqueue),
 479             M_DEVBUF, M_NOWAIT | M_ZERO);
 480         if (sc->vtmmio_vqs == NULL)
 481                 return (ENOMEM);
 482
 483         vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE,
 484             (1 << PAGE_SHIFT));
 485
 486         for (idx = 0; idx < nvqs; idx++) {
 487                 vqx = &sc->vtmmio_vqs[idx];
 488                 info = &vq_info[idx];
 489
 490                 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx);
 491
 492                 vtmmio_select_virtqueue(sc, idx);
 493                 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX);
 494
 495                 error = virtqueue_alloc(dev, idx, size,
 496                     VIRTIO_MMIO_VRING_ALIGN, ~(vm_paddr_t)0, info, &vq);
 497                 if (error) {
 498                         device_printf(dev,
 499                             "cannot allocate virtqueue %d: %d\n",
 500                             idx, error);
 501                         break;
 502                 }
 503
 504                 vtmmio_set_virtqueue(sc, vq, size);
 505
 506                 vqx->vtv_vq = *info->vqai_vq = vq;
 507                 vqx->vtv_no_intr = info->vqai_intr == NULL;
 508
 509                 sc->vtmmio_nvqs++;
 510         }
 511
 512         if (error)
 513                 vtmmio_free_virtqueues(sc);
 514
 515         return (error);
 516 }
 517
 518 static void
 519 vtmmio_stop(device_t dev)
 520 {
 521
 522         vtmmio_reset(device_get_softc(dev));
 523 }
 524
 525 static void
 526 vtmmio_poll(device_t dev)
 527 {
 528         struct vtmmio_softc *sc;
 529
 530         sc = device_get_softc(dev);
 531
 532         if (sc->platform != NULL)
 533                 VIRTIO_MMIO_POLL(sc->platform);
 534 }
 535
 536 static int
 537 vtmmio_reinit(device_t dev, uint64_t features)
 538 {
 539         struct vtmmio_softc *sc;
 540         int idx, error;
 541
 542         sc = device_get_softc(dev);
 543
 544         if (vtmmio_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
 545                 vtmmio_stop(dev);
 546
 547         /*
 548          * Quickly drive the status through ACK and DRIVER. The device
 549          * does not become usable again until vtmmio_reinit_complete().
 550          */
 551         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
 552         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
 553
 554         vtmmio_negotiate_features(dev, features);
 555
 556         vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE,
 557             (1 << PAGE_SHIFT));
 558
 559         for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
 560                 error = vtmmio_reinit_virtqueue(sc, idx);
 561                 if (error)
 562                         return (error);
 563         }
 564
 565         return (0);
 566 }
 567
 568 static void
 569 vtmmio_reinit_complete(device_t dev)
 570 {
 571
 572         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
 573 }
 574
 575 static void
 576 vtmmio_notify_virtqueue(device_t dev, uint16_t queue)
 577 {
 578         struct vtmmio_softc *sc;
 579
 580         sc = device_get_softc(dev);
 581
 582         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NOTIFY, queue);
 583 }
 584
 585 static uint8_t
 586 vtmmio_get_status(device_t dev)
 587 {
 588         struct vtmmio_softc *sc;
 589
 590         sc = device_get_softc(dev);
 591
 592         return (vtmmio_read_config_4(sc, VIRTIO_MMIO_STATUS));
 593 }
 594
 595 static void
 596 vtmmio_set_status(device_t dev, uint8_t status)
 597 {
 598         struct vtmmio_softc *sc;
 599
 600         sc = device_get_softc(dev);
 601
 602         if (status != VIRTIO_CONFIG_STATUS_RESET)
 603                 status |= vtmmio_get_status(dev);
 604
 605         vtmmio_write_config_4(sc, VIRTIO_MMIO_STATUS, status);
 606 }
 607
 608 static void
 609 vtmmio_read_dev_config(device_t dev, bus_size_t offset,
 610     void *dst, int length)
 611 {
 612         struct vtmmio_softc *sc;
 613         bus_size_t off;
 614         uint8_t *d;
 615         int size;
 616         uint64_t low32, high32;
 617
 618         sc = device_get_softc(dev);
 619         off = VIRTIO_MMIO_CONFIG + offset;
 620
 621         /*
 622          * The non-legacy MMIO specification adds the following restriction:
 623          *
 624          *   4.2.2.2: For the device-specific configuration space, the driver
 625          *   MUST use 8 bit wide accesses for 8 bit wide fields, 16 bit wide
 626          *   and aligned accesses for 16 bit wide fields and 32 bit wide and
 627          *   aligned accesses for 32 and 64 bit wide fields.
 628          *
 629          * The endianness also varies between non-legacy and legacy:
 630          *
 631          *   2.4: Note: The device configuration space uses the little-endian
 632          *   format for multi-byte fields.
 633          *
 634          *   2.4.3: Note that for legacy interfaces, device configuration space
 635          *   is generally the guest’s native endian, rather than PCI’s
 636          *   little-endian. The correct endian-ness is documented for each
 637          *   device.
 638          */
 639         if (sc->vtmmio_version > 1) {
 640                 switch (length) {
 641                 case 1:
 642                         *(uint8_t *)dst = vtmmio_read_config_1(sc, off);
 643                         break;
 644                 case 2:
 645                         *(uint16_t *)dst =
 646                             le16toh(vtmmio_read_config_2(sc, off));
 647                         break;
 648                 case 4:
 649                         *(uint32_t *)dst =
 650                             le32toh(vtmmio_read_config_4(sc, off));
 651                         break;
 652                 case 8:
 653                         low32 = le32toh(vtmmio_read_config_4(sc, off));
 654                         high32 = le32toh(vtmmio_read_config_4(sc, off + 4));
 655                         *(uint64_t *)dst = (high32 << 32) | low32;
 656                         break;
 657                 default:
 658                         panic("%s: invalid length %d\n", __func__, length);
 659                 }
 660
 661                 return;
 662         }
 663
 664         for (d = dst; length > 0; d += size, off += size, length -= size) {
 665 #ifdef ALLOW_WORD_ALIGNED_ACCESS
 666                 if (length >= 4) {
 667                         size = 4;
 668                         *(uint32_t *)d = vtmmio_read_config_4(sc, off);
 669                 } else if (length >= 2) {
 670                         size = 2;
 671                         *(uint16_t *)d = vtmmio_read_config_2(sc, off);
 672                 } else
 673 #endif
 674                 {
 675                         size = 1;
 676                         *d = vtmmio_read_config_1(sc, off);
 677                 }
 678         }
 679 }
 680
 681 static void
 682 vtmmio_write_dev_config(device_t dev, bus_size_t offset,
 683     void *src, int length)
 684 {
 685         struct vtmmio_softc *sc;
 686         bus_size_t off;
 687         uint8_t *s;
 688         int size;
 689
 690         sc = device_get_softc(dev);
 691         off = VIRTIO_MMIO_CONFIG + offset;
 692
 693         /*
 694          * The non-legacy MMIO specification adds size and alignment
 695          * restrctions. It also changes the endianness from native-endian to
 696          * little-endian. See vtmmio_read_dev_config.
 697          */
 698         if (sc->vtmmio_version > 1) {
 699                 switch (length) {
 700                 case 1:
 701                         vtmmio_write_config_1(sc, off, *(uint8_t *)src);
 702                         break;
 703                 case 2:
 704                         vtmmio_write_config_2(sc, off,
 705                             htole16(*(uint16_t *)src));
 706                         break;
 707                 case 4:
 708                         vtmmio_write_config_4(sc, off,
 709                             htole32(*(uint32_t *)src));
 710                         break;
 711                 case 8:
 712                         vtmmio_write_config_4(sc, off,
 713                             htole32(*(uint64_t *)src));
 714                         vtmmio_write_config_4(sc, off + 4,
 715                             htole32((*(uint64_t *)src) >> 32));
 716                         break;
 717                 default:
 718                         panic("%s: invalid length %d\n", __func__, length);
 719                 }
 720
 721                 return;
 722         }
 723
 724         for (s = src; length > 0; s += size, off += size, length -= size) {
 725 #ifdef ALLOW_WORD_ALIGNED_ACCESS
 726                 if (length >= 4) {
 727                         size = 4;
 728                         vtmmio_write_config_4(sc, off, *(uint32_t *)s);
 729                 } else if (length >= 2) {
 730                         size = 2;
 731                         vtmmio_write_config_2(sc, off, *(uint16_t *)s);
 732                 } else
 733 #endif
 734                 {
 735                         size = 1;
 736                         vtmmio_write_config_1(sc, off, *s);
 737                 }
 738         }
 739 }
 740
 741 static void
 742 vtmmio_describe_features(struct vtmmio_softc *sc, const char *msg,
 743     uint64_t features)
 744 {
 745         device_t dev, child;
 746
 747         dev = sc->dev;
 748         child = sc->vtmmio_child_dev;
 749
 750         if (device_is_attached(child) || bootverbose == 0)
 751                 return;
 752
 753         virtio_describe(dev, msg, features, sc->vtmmio_child_feat_desc);
 754 }
 755
 756 static void
 757 vtmmio_probe_and_attach_child(struct vtmmio_softc *sc)
 758 {
 759         device_t dev, child;
 760
 761         dev = sc->dev;
 762         child = sc->vtmmio_child_dev;
 763
 764         if (child == NULL)
 765                 return;
 766
 767         if (device_get_state(child) != DS_NOTPRESENT) {
 768                 return;
 769         }
 770
 771         if (device_probe(child) != 0) {
 772                 return;
 773         }
 774
 775         vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
 776         if (device_attach(child) != 0) {
 777                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
 778                 vtmmio_reset(sc);
 779                 vtmmio_release_child_resources(sc);
 780                 /* Reset status for future attempt. */
 781                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
 782         } else {
 783                 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
 784                 VIRTIO_ATTACH_COMPLETED(child);
 785         }
 786 }
 787
 788 static int
 789 vtmmio_reinit_virtqueue(struct vtmmio_softc *sc, int idx)
 790 {
 791         struct vtmmio_virtqueue *vqx;
 792         struct virtqueue *vq;
 793         int error;
 794         uint16_t size;
 795
 796         vqx = &sc->vtmmio_vqs[idx];
 797         vq = vqx->vtv_vq;
 798
 799         KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));
 800
 801         vtmmio_select_virtqueue(sc, idx);
 802         size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX);
 803
 804         error = virtqueue_reinit(vq, size);
 805         if (error)
 806                 return (error);
 807
 808         vtmmio_set_virtqueue(sc, vq, size);
 809
 810         return (0);
 811 }
 812
 813 static void
 814 vtmmio_free_interrupts(struct vtmmio_softc *sc)
 815 {
 816
 817         if (sc->ih != NULL)
 818                 bus_teardown_intr(sc->dev, sc->res[1], sc->ih);
 819
 820         if (sc->res[1] != NULL)
 821                 bus_release_resource(sc->dev, SYS_RES_IRQ, 0, sc->res[1]);
 822 }
 823
 824 static void
 825 vtmmio_free_virtqueues(struct vtmmio_softc *sc)
 826 {
 827         struct vtmmio_virtqueue *vqx;
 828         int idx;
 829
 830         for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
 831                 vqx = &sc->vtmmio_vqs[idx];
 832
 833                 vtmmio_select_virtqueue(sc, idx);
 834                 if (sc->vtmmio_version == 1)
 835                         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 0);
 836                 else
 837                         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_READY, 0);
 838
 839                 virtqueue_free(vqx->vtv_vq);
 840                 vqx->vtv_vq = NULL;
 841         }
 842
 843         free(sc->vtmmio_vqs, M_DEVBUF);
 844         sc->vtmmio_vqs = NULL;
 845         sc->vtmmio_nvqs = 0;
 846 }
 847
 848 static void
 849 vtmmio_release_child_resources(struct vtmmio_softc *sc)
 850 {
 851
 852         vtmmio_free_interrupts(sc);
 853         vtmmio_free_virtqueues(sc);
 854 }
 855
 856 static void
 857 vtmmio_reset(struct vtmmio_softc *sc)
 858 {
 859
 860         /*
 861          * Setting the status to RESET sets the host device to
 862          * the original, uninitialized state.
 863          */
 864         vtmmio_set_status(sc->dev, VIRTIO_CONFIG_STATUS_RESET);
 865 }
 866
 867 static void
 868 vtmmio_select_virtqueue(struct vtmmio_softc *sc, int idx)
 869 {
 870
 871         vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx);
 872 }
 873
 874 static void
 875 vtmmio_vq_intr(void *arg)
 876 {
 877         struct vtmmio_virtqueue *vqx;
 878         struct vtmmio_softc *sc;
 879         struct virtqueue *vq;
 880         uint32_t status;
 881         int idx;
 882
 883         sc = arg;
 884
 885         status = vtmmio_read_config_4(sc, VIRTIO_MMIO_INTERRUPT_STATUS);
 886         vtmmio_write_config_4(sc, VIRTIO_MMIO_INTERRUPT_ACK, status);
 887
 888         /* The config changed */
 889         if (status & VIRTIO_MMIO_INT_CONFIG)
 890                 if (sc->vtmmio_child_dev != NULL)
 891                         VIRTIO_CONFIG_CHANGE(sc->vtmmio_child_dev);
 892
 893         /* Notify all virtqueues. */
 894         if (status & VIRTIO_MMIO_INT_VRING) {
 895                 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
 896                         vqx = &sc->vtmmio_vqs[idx];
 897                         if (vqx->vtv_no_intr == 0) {
 898                                 vq = vqx->vtv_vq;
 899                                 virtqueue_intr(vq);
 900                         }
 901                 }
 902         }
 903 }