sys/dev/vmware/vmci/vmci.c

   1 /*-
   2  * Copyright (c) 2018 VMware, Inc.
   3  *
   4  * SPDX-License-Identifier: (BSD-2-Clause OR GPL-2.0)
   5  */
   6
   7 /* Driver for VMware Virtual Machine Communication Interface (VMCI) device. */
   8
   9 #include <sys/cdefs.h>
  10 __FBSDID("$FreeBSD$");
  11
  12 #include <sys/types.h>
  13 #include <sys/bus.h>
  14 #include <sys/kernel.h>
  15 #include <sys/malloc.h>
  16 #include <sys/module.h>
  17 #include <sys/rman.h>
  18 #include <sys/systm.h>
  19
  20 #include <dev/pci/pcireg.h>
  21 #include <dev/pci/pcivar.h>
  22
  23 #include <machine/bus.h>
  24
  25 #include "vmci.h"
  26 #include "vmci_doorbell.h"
  27 #include "vmci_driver.h"
  28 #include "vmci_kernel_defs.h"
  29 #include "vmci_queue_pair.h"
  30
  31 static int      vmci_probe(device_t);
  32 static int      vmci_attach(device_t);
  33 static int      vmci_detach(device_t);
  34 static int      vmci_shutdown(device_t);
  35
  36 static int      vmci_map_bars(struct vmci_softc *);
  37 static void     vmci_unmap_bars(struct vmci_softc *);
  38
  39 static int      vmci_config_capabilities(struct vmci_softc *);
  40
  41 static int      vmci_dma_malloc_int(struct vmci_softc *, bus_size_t,
  42                     bus_size_t, struct vmci_dma_alloc *);
  43 static void     vmci_dma_free_int(struct vmci_softc *,
  44                     struct vmci_dma_alloc *);
  45
  46 static int      vmci_config_interrupts(struct vmci_softc *);
  47 static int      vmci_config_interrupt(struct vmci_softc *);
  48 static int      vmci_check_intr_cnt(struct vmci_softc *);
  49 static int      vmci_allocate_interrupt_resources(struct vmci_softc *);
  50 static int      vmci_setup_interrupts(struct vmci_softc *);
  51 static void     vmci_dismantle_interrupts(struct vmci_softc *);
  52 static void     vmci_interrupt(void *);
  53 static void     vmci_interrupt_bm(void *);
  54 static void     dispatch_datagrams(void *, int);
  55 static void     process_bitmap(void *, int);
  56
  57 static void     vmci_delayed_work_fn_cb(void *context, int data);
  58
  59 static device_method_t vmci_methods[] = {
  60         /* Device interface. */
  61         DEVMETHOD(device_probe,         vmci_probe),
  62         DEVMETHOD(device_attach,        vmci_attach),
  63         DEVMETHOD(device_detach,        vmci_detach),
  64         DEVMETHOD(device_shutdown,      vmci_shutdown),
  65
  66         DEVMETHOD_END
  67 };
  68
  69 static driver_t vmci_driver = {
  70         "vmci", vmci_methods, sizeof(struct vmci_softc)
  71 };
  72
  73 static devclass_t vmci_devclass;
  74 DRIVER_MODULE(vmci, pci, vmci_driver, vmci_devclass, 0, 0);
  75 MODULE_VERSION(vmci, VMCI_VERSION);
  76
  77 MODULE_DEPEND(vmci, pci, 1, 1, 1);
  78
  79 static struct vmci_softc *vmci_sc;
  80
  81 #define LGPFX   "vmci: "
  82 /*
  83  * Allocate a buffer for incoming datagrams globally to avoid repeated
  84  * allocation in the interrupt handler's atomic context.
  85  */
  86 static uint8_t *data_buffer = NULL;
  87 static uint32_t data_buffer_size = VMCI_MAX_DG_SIZE;
  88
  89 struct vmci_delayed_work_info {
  90         vmci_work_fn    *work_fn;
  91         void            *data;
  92         vmci_list_item(vmci_delayed_work_info) entry;
  93 };
  94
  95 /*
  96  *------------------------------------------------------------------------------
  97  *
  98  * vmci_probe --
  99  *
 100  *     Probe to see if the VMCI device is present.
 101  *
 102  * Results:
 103  *     BUS_PROBE_DEFAULT if device exists, ENXIO otherwise.
 104  *
 105  * Side effects:
 106  *     None.
 107  *
 108  *------------------------------------------------------------------------------
 109  */
 110
 111 static int
 112 vmci_probe(device_t dev)
 113 {
 114
 115         if (pci_get_vendor(dev) == VMCI_VMWARE_VENDOR_ID &&
 116             pci_get_device(dev) == VMCI_VMWARE_DEVICE_ID) {
 117                 device_set_desc(dev,
 118                     "VMware Virtual Machine Communication Interface");
 119
 120                 return (BUS_PROBE_DEFAULT);
 121         }
 122
 123         return (ENXIO);
 124 }
 125
 126 /*
 127  *------------------------------------------------------------------------------
 128  *
 129  * vmci_attach --
 130  *
 131  *     Attach VMCI device to the system after vmci_probe() has been called and
 132  *     the device has been detected.
 133  *
 134  * Results:
 135  *     0 if success, ENXIO otherwise.
 136  *
 137  * Side effects:
 138  *     None.
 139  *
 140  *------------------------------------------------------------------------------
 141  */
 142
 143 static int
 144 vmci_attach(device_t dev)
 145 {
 146         struct vmci_softc *sc;
 147         int error, i;
 148
 149         sc = device_get_softc(dev);
 150         sc->vmci_dev = dev;
 151         vmci_sc = sc;
 152
 153         data_buffer = NULL;
 154         sc->vmci_num_intr = 0;
 155         for (i = 0; i < VMCI_MAX_INTRS; i++) {
 156                 sc->vmci_intrs[i].vmci_irq = NULL;
 157                 sc->vmci_intrs[i].vmci_handler = NULL;
 158         }
 159
 160         TASK_INIT(&sc->vmci_interrupt_dq_task, 0, dispatch_datagrams, sc);
 161         TASK_INIT(&sc->vmci_interrupt_bm_task, 0, process_bitmap, sc);
 162
 163         TASK_INIT(&sc->vmci_delayed_work_task, 0, vmci_delayed_work_fn_cb, sc);
 164
 165         pci_enable_busmaster(dev);
 166
 167         mtx_init(&sc->vmci_spinlock, "VMCI Spinlock", NULL, MTX_SPIN);
 168         mtx_init(&sc->vmci_delayed_work_lock, "VMCI Delayed Work Lock",
 169             NULL, MTX_DEF);
 170
 171         error = vmci_map_bars(sc);
 172         if (error) {
 173                 VMCI_LOG_ERROR(LGPFX"Failed to map PCI BARs.\n");
 174                 goto fail;
 175         }
 176
 177         error = vmci_config_capabilities(sc);
 178         if (error) {
 179                 VMCI_LOG_ERROR(LGPFX"Failed to configure capabilities.\n");
 180                 goto fail;
 181         }
 182
 183         vmci_list_init(&sc->vmci_delayed_work_infos);
 184
 185         vmci_components_init();
 186         vmci_util_init();
 187         error = vmci_qp_guest_endpoints_init();
 188         if (error) {
 189                 VMCI_LOG_ERROR(LGPFX"vmci_qp_guest_endpoints_init failed.\n");
 190                 goto fail;
 191         }
 192
 193         error = vmci_config_interrupts(sc);
 194         if (error)
 195                 VMCI_LOG_ERROR(LGPFX"Failed to enable interrupts.\n");
 196
 197 fail:
 198         if (error) {
 199                 vmci_detach(dev);
 200                 return (ENXIO);
 201         }
 202
 203         return (0);
 204 }
 205
 206 /*
 207  *------------------------------------------------------------------------------
 208  *
 209  * vmci_detach --
 210  *
 211  *     Detach the VMCI device.
 212  *
 213  * Results:
 214  *     0
 215  *
 216  * Side effects:
 217  *     None.
 218  *
 219  *------------------------------------------------------------------------------
 220  */
 221
 222 static int
 223 vmci_detach(device_t dev)
 224 {
 225         struct vmci_softc *sc;
 226
 227         sc = device_get_softc(dev);
 228
 229         vmci_qp_guest_endpoints_exit();
 230         vmci_util_exit();
 231
 232         vmci_dismantle_interrupts(sc);
 233
 234         vmci_components_cleanup();
 235
 236         taskqueue_drain(taskqueue_thread, &sc->vmci_delayed_work_task);
 237         mtx_destroy(&sc->vmci_delayed_work_lock);
 238
 239         if (sc->vmci_res0 != NULL)
 240                 bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0,
 241                     VMCI_CONTROL_ADDR, VMCI_CONTROL_RESET);
 242
 243         if (sc->vmci_notifications_bitmap.dma_vaddr != NULL)
 244                 vmci_dma_free(&sc->vmci_notifications_bitmap);
 245
 246         vmci_unmap_bars(sc);
 247
 248         mtx_destroy(&sc->vmci_spinlock);
 249
 250         pci_disable_busmaster(dev);
 251
 252         return (0);
 253 }
 254
 255 /*
 256  *------------------------------------------------------------------------------
 257  *
 258  * vmci_shutdown --
 259  *
 260  *     This function is called during system shutdown. We don't do anything.
 261  *
 262  * Results:
 263  *     0
 264  *
 265  * Side effects:
 266  *     None.
 267  *
 268  *------------------------------------------------------------------------------
 269  */
 270
 271 static int
 272 vmci_shutdown(device_t dev)
 273 {
 274
 275         return (0);
 276 }
 277
 278 /*
 279  *------------------------------------------------------------------------------
 280  *
 281  * vmci_map_bars --
 282  *
 283  *     Maps the PCI I/O and MMIO BARs.
 284  *
 285  * Results:
 286  *     0 on success, ENXIO otherwise.
 287  *
 288  * Side effects:
 289  *     None.
 290  *
 291  *------------------------------------------------------------------------------
 292  */
 293
 294 static int
 295 vmci_map_bars(struct vmci_softc *sc)
 296 {
 297         int rid;
 298
 299         /* Map the PCI I/O BAR: BAR0 */
 300         rid = PCIR_BAR(0);
 301         sc->vmci_res0 = bus_alloc_resource_any(sc->vmci_dev, SYS_RES_IOPORT,
 302             &rid, RF_ACTIVE);
 303         if (sc->vmci_res0 == NULL) {
 304                 VMCI_LOG_ERROR(LGPFX"Could not map: BAR0\n");
 305                 return (ENXIO);
 306         }
 307
 308         sc->vmci_iot0 = rman_get_bustag(sc->vmci_res0);
 309         sc->vmci_ioh0 = rman_get_bushandle(sc->vmci_res0);
 310         sc->vmci_ioaddr = rman_get_start(sc->vmci_res0);
 311
 312         /* Map the PCI MMIO BAR: BAR1 */
 313         rid = PCIR_BAR(1);
 314         sc->vmci_res1 = bus_alloc_resource_any(sc->vmci_dev, SYS_RES_MEMORY,
 315             &rid, RF_ACTIVE);
 316         if (sc->vmci_res1 == NULL) {
 317                 VMCI_LOG_ERROR(LGPFX"Could not map: BAR1\n");
 318                 return (ENXIO);
 319         }
 320
 321         sc->vmci_iot1 = rman_get_bustag(sc->vmci_res1);
 322         sc->vmci_ioh1 = rman_get_bushandle(sc->vmci_res1);
 323
 324         return (0);
 325 }
 326
 327 /*
 328  *------------------------------------------------------------------------------
 329  *
 330  * vmci_unmap_bars --
 331  *
 332  *     Unmaps the VMCI PCI I/O and MMIO BARs.
 333  *
 334  * Results:
 335  *     None.
 336  *
 337  * Side effects:
 338  *     None.
 339  *
 340  *------------------------------------------------------------------------------
 341  */
 342
 343 static void
 344 vmci_unmap_bars(struct vmci_softc *sc)
 345 {
 346         int rid;
 347
 348         if (sc->vmci_res0 != NULL) {
 349                 rid = PCIR_BAR(0);
 350                 bus_release_resource(sc->vmci_dev, SYS_RES_IOPORT, rid,
 351                     sc->vmci_res0);
 352                 sc->vmci_res0 = NULL;
 353         }
 354
 355         if (sc->vmci_res1 != NULL) {
 356                 rid = PCIR_BAR(1);
 357                 bus_release_resource(sc->vmci_dev, SYS_RES_MEMORY, rid,
 358                     sc->vmci_res1);
 359                 sc->vmci_res1 = NULL;
 360         }
 361 }
 362
 363 /*
 364  *------------------------------------------------------------------------------
 365  *
 366  * vmci_config_capabilities --
 367  *
 368  *     Check the VMCI device capabilities and configure the device accordingly.
 369  *
 370  * Results:
 371  *     0 if success, ENODEV otherwise.
 372  *
 373  * Side effects:
 374  *     Device capabilities are enabled.
 375  *
 376  *------------------------------------------------------------------------------
 377  */
 378
 379 static int
 380 vmci_config_capabilities(struct vmci_softc *sc)
 381 {
 382         unsigned long bitmap_PPN;
 383         int error;
 384
 385         /*
 386          * Verify that the VMCI device supports the capabilities that we
 387          * need. Datagrams are necessary and notifications will be used
 388          * if the device supports it.
 389          */
 390         sc->capabilities = bus_space_read_4(sc->vmci_iot0, sc->vmci_ioh0,
 391             VMCI_CAPS_ADDR);
 392
 393         if ((sc->capabilities & VMCI_CAPS_DATAGRAM) == 0) {
 394                 VMCI_LOG_ERROR(LGPFX"VMCI device does not support "
 395                     "datagrams.\n");
 396                 return (ENODEV);
 397         }
 398
 399         if (sc->capabilities & VMCI_CAPS_NOTIFICATIONS) {
 400                 sc->capabilities = VMCI_CAPS_DATAGRAM;
 401                 error = vmci_dma_malloc(PAGE_SIZE, 1,
 402                     &sc->vmci_notifications_bitmap);
 403                 if (error)
 404                         VMCI_LOG_ERROR(LGPFX"Failed to alloc memory for "
 405                             "notification bitmap.\n");
 406                 else {
 407                         memset(sc->vmci_notifications_bitmap.dma_vaddr, 0,
 408                             PAGE_SIZE);
 409                         sc->capabilities |= VMCI_CAPS_NOTIFICATIONS;
 410                 }
 411         } else
 412                 sc->capabilities = VMCI_CAPS_DATAGRAM;
 413
 414         /* Let the host know which capabilities we intend to use. */
 415         bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0,
 416             VMCI_CAPS_ADDR, sc->capabilities);
 417
 418         /*
 419          * Register notification bitmap with device if that capability is
 420          * used.
 421          */
 422         if (sc->capabilities & VMCI_CAPS_NOTIFICATIONS) {
 423                 bitmap_PPN =
 424                     sc->vmci_notifications_bitmap.dma_paddr >> PAGE_SHIFT;
 425                 vmci_register_notification_bitmap(bitmap_PPN);
 426         }
 427
 428         /* Check host capabilities. */
 429         if (!vmci_check_host_capabilities())
 430                 return (ENODEV);
 431
 432         return (0);
 433 }
 434
 435 /*
 436  *------------------------------------------------------------------------------
 437  *
 438  * vmci_dmamap_cb --
 439  *
 440  *     Callback to receive mapping information resulting from the load of a
 441  *     bus_dmamap_t via bus_dmamap_load()
 442  *
 443  * Results:
 444  *     None.
 445  *
 446  * Side effects:
 447  *     None.
 448  *
 449  *------------------------------------------------------------------------------
 450  */
 451
 452 static void
 453 vmci_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 454 {
 455         bus_addr_t *baddr = arg;
 456
 457         if (error == 0)
 458                 *baddr = segs->ds_addr;
 459 }
 460
 461 /*
 462  *------------------------------------------------------------------------------
 463  *
 464  * vmci_dma_malloc_int --
 465  *
 466  *     Internal function that allocates DMA memory.
 467  *
 468  * Results:
 469  *     0 if success.
 470  *     ENOMEM if insufficient memory.
 471  *     EINPROGRESS if mapping is deferred.
 472  *     EINVAL if the request was invalid.
 473  *
 474  * Side effects:
 475  *     DMA memory is allocated.
 476  *
 477  *------------------------------------------------------------------------------
 478  */
 479
 480 static int
 481 vmci_dma_malloc_int(struct vmci_softc *sc, bus_size_t size, bus_size_t align,
 482     struct vmci_dma_alloc *dma)
 483 {
 484         int error;
 485
 486         bzero(dma, sizeof(struct vmci_dma_alloc));
 487
 488         error = bus_dma_tag_create(bus_get_dma_tag(vmci_sc->vmci_dev),
 489             align, 0,           /* alignment, bounds */
 490             BUS_SPACE_MAXADDR,  /* lowaddr */
 491             BUS_SPACE_MAXADDR,  /* highaddr */
 492             NULL, NULL,         /* filter, filterarg */
 493             size,               /* maxsize */
 494             1,                  /* nsegments */
 495             size,               /* maxsegsize */
 496             BUS_DMA_ALLOCNOW,   /* flags */
 497             NULL,               /* lockfunc */
 498             NULL,               /* lockfuncarg */
 499             &dma->dma_tag);
 500         if (error) {
 501                 VMCI_LOG_ERROR(LGPFX"bus_dma_tag_create failed: %d\n", error);
 502                 goto fail;
 503         }
 504
 505         error = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
 506             BUS_DMA_ZERO | BUS_DMA_NOWAIT, &dma->dma_map);
 507         if (error) {
 508                 VMCI_LOG_ERROR(LGPFX"bus_dmamem_alloc failed: %d\n", error);
 509                 goto fail;
 510         }
 511
 512         error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
 513             size, vmci_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT);
 514         if (error) {
 515                 VMCI_LOG_ERROR(LGPFX"bus_dmamap_load failed: %d\n", error);
 516                 goto fail;
 517         }
 518
 519         dma->dma_size = size;
 520
 521 fail:
 522         if (error)
 523                 vmci_dma_free(dma);
 524
 525         return (error);
 526 }
 527
 528 /*
 529  *------------------------------------------------------------------------------
 530  *
 531  * vmci_dma_malloc --
 532  *
 533  *     This function is a wrapper around vmci_dma_malloc_int for callers
 534  *     outside of this module. Since we only support a single VMCI device, this
 535  *     wrapper provides access to the device softc structure.
 536  *
 537  * Results:
 538  *     0 if success.
 539  *     ENOMEM if insufficient memory.
 540  *     EINPROGRESS if mapping is deferred.
 541  *     EINVAL if the request was invalid.
 542  *
 543  * Side effects:
 544  *     DMA memory is allocated.
 545  *
 546  *------------------------------------------------------------------------------
 547  */
 548
 549 int
 550 vmci_dma_malloc(bus_size_t size, bus_size_t align, struct vmci_dma_alloc *dma)
 551 {
 552
 553         return (vmci_dma_malloc_int(vmci_sc, size, align, dma));
 554 }
 555
 556 /*
 557  *------------------------------------------------------------------------------
 558  *
 559  * vmci_dma_free_int --
 560  *
 561  *     Internal function that frees DMA memory.
 562  *
 563  * Results:
 564  *     None.
 565  *
 566  * Side effects:
 567  *     Frees DMA memory.
 568  *
 569  *------------------------------------------------------------------------------
 570  */
 571
 572 static void
 573 vmci_dma_free_int(struct vmci_softc *sc, struct vmci_dma_alloc *dma)
 574 {
 575
 576         if (dma->dma_tag != NULL) {
 577                 if (dma->dma_paddr != 0) {
 578                         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
 579                             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 580                         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
 581                 }
 582
 583                 if (dma->dma_vaddr != NULL)
 584                         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr,
 585                             dma->dma_map);
 586
 587                 bus_dma_tag_destroy(dma->dma_tag);
 588         }
 589         bzero(dma, sizeof(struct vmci_dma_alloc));
 590 }
 591
 592 /*
 593  *------------------------------------------------------------------------------
 594  *
 595  * vmci_dma_free --
 596  *
 597  *     This function is a wrapper around vmci_dma_free_int for callers outside
 598  *     of this module. Since we only support a single VMCI device, this wrapper
 599  *     provides access to the device softc structure.
 600  *
 601  * Results:
 602  *     None.
 603  *
 604  * Side effects:
 605  *     Frees DMA memory.
 606  *
 607  *------------------------------------------------------------------------------
 608  */
 609
 610 void
 611 vmci_dma_free(struct vmci_dma_alloc *dma)
 612 {
 613
 614         vmci_dma_free_int(vmci_sc, dma);
 615 }
 616
 617 /*
 618  *------------------------------------------------------------------------------
 619  *
 620  * vmci_config_interrupts --
 621  *
 622  *     Configures and enables interrupts. Try to configure MSI-X. If this fails,
 623  *     try to configure MSI. If even this fails, try legacy interrupts.
 624  *
 625  * Results:
 626  *     0 if success.
 627  *     ENOMEM if insufficient memory.
 628  *     ENODEV if the device doesn't support interrupts.
 629  *     ENXIO if the device configuration failed.
 630  *
 631  * Side effects:
 632  *     Interrupts get enabled if successful.
 633  *
 634  *------------------------------------------------------------------------------
 635  */
 636
 637 static int
 638 vmci_config_interrupts(struct vmci_softc *sc)
 639 {
 640         int error;
 641
 642         data_buffer = malloc(data_buffer_size, M_DEVBUF, M_ZERO | M_NOWAIT);
 643         if (data_buffer == NULL)
 644                 return (ENOMEM);
 645
 646         sc->vmci_intr_type = VMCI_INTR_TYPE_MSIX;
 647         error = vmci_config_interrupt(sc);
 648         if (error) {
 649                 sc->vmci_intr_type = VMCI_INTR_TYPE_MSI;
 650                 error = vmci_config_interrupt(sc);
 651         }
 652         if (error) {
 653                 sc->vmci_intr_type = VMCI_INTR_TYPE_INTX;
 654                 error = vmci_config_interrupt(sc);
 655         }
 656         if (error)
 657                 return (error);
 658
 659         /* Enable specific interrupt bits. */
 660         if (sc->capabilities & VMCI_CAPS_NOTIFICATIONS)
 661                 bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0,
 662                     VMCI_IMR_ADDR, VMCI_IMR_DATAGRAM | VMCI_IMR_NOTIFICATION);
 663         else
 664                 bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0,
 665                     VMCI_IMR_ADDR, VMCI_IMR_DATAGRAM);
 666
 667         /* Enable interrupts. */
 668         bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0,
 669             VMCI_CONTROL_ADDR, VMCI_CONTROL_INT_ENABLE);
 670
 671         return (0);
 672 }
 673
 674 /*
 675  *------------------------------------------------------------------------------
 676  *
 677  * vmci_config_interrupt --
 678  *
 679  *     Check the number of interrupts supported, allocate resources and setup
 680  *     interrupts.
 681  *
 682  * Results:
 683  *     0 if success.
 684  *     ENOMEM if insufficient memory.
 685  *     ENODEV if the device doesn't support interrupts.
 686  *     ENXIO if the device configuration failed.
 687  *
 688  * Side effects:
 689  *     Resources get allocated and interrupts get setup (but not enabled) if
 690  *     successful.
 691  *
 692  *------------------------------------------------------------------------------
 693  */
 694
 695 static int
 696 vmci_config_interrupt(struct vmci_softc *sc)
 697 {
 698         int error;
 699
 700         error = vmci_check_intr_cnt(sc);
 701         if (error)
 702                 return (error);
 703
 704         error = vmci_allocate_interrupt_resources(sc);
 705         if (error)
 706                 return (error);
 707
 708         error = vmci_setup_interrupts(sc);
 709         if (error)
 710                 return (error);
 711
 712         return (0);
 713 }
 714
 715 /*
 716  *------------------------------------------------------------------------------
 717  *
 718  * vmci_check_intr_cnt --
 719  *
 720  *     Check the number of interrupts supported by the device and ask PCI bus
 721  *     to allocate appropriate number of interrupts.
 722  *
 723  * Results:
 724  *     0 if success.
 725  *     ENODEV if the device doesn't support any interrupts.
 726  *     ENXIO if the device configuration failed.
 727  *
 728  * Side effects:
 729  *     Resources get allocated on success.
 730  *
 731  *------------------------------------------------------------------------------
 732  */
 733
 734 static int
 735 vmci_check_intr_cnt(struct vmci_softc *sc)
 736 {
 737
 738         if (sc->vmci_intr_type == VMCI_INTR_TYPE_INTX) {
 739                 sc->vmci_num_intr = 1;
 740                 return (0);
 741         }
 742
 743         /*
 744          * Make sure that the device supports the required number of MSI/MSI-X
 745          * messages. We try for 2 MSI-X messages but 1 is good too. We need at
 746          * least 1 MSI message.
 747          */
 748         sc->vmci_num_intr = (sc->vmci_intr_type == VMCI_INTR_TYPE_MSIX) ?
 749             pci_msix_count(sc->vmci_dev) : pci_msi_count(sc->vmci_dev);
 750
 751         if (!sc->vmci_num_intr) {
 752                 VMCI_LOG_ERROR(LGPFX"Device does not support any interrupt"
 753                     " messages");
 754                 return (ENODEV);
 755         }
 756
 757         sc->vmci_num_intr = (sc->vmci_intr_type == VMCI_INTR_TYPE_MSIX) ?
 758             VMCI_MAX_INTRS : 1;
 759         if (sc->vmci_intr_type == VMCI_INTR_TYPE_MSIX) {
 760                 if (pci_alloc_msix(sc->vmci_dev, &sc->vmci_num_intr))
 761                         return (ENXIO);
 762         } else if (sc->vmci_intr_type == VMCI_INTR_TYPE_MSI) {
 763                 if (pci_alloc_msi(sc->vmci_dev, &sc->vmci_num_intr))
 764                         return (ENXIO);
 765         }
 766
 767         return (0);
 768 }
 769
 770 /*
 771  *------------------------------------------------------------------------------
 772  *
 773  * vmci_allocate_interrupt_resources --
 774  *
 775  *     Allocate resources necessary for interrupts.
 776  *
 777  * Results:
 778  *     0 if success, ENXIO otherwise.
 779  *
 780  * Side effects:
 781  *     Resources get allocated on success.
 782  *
 783  *------------------------------------------------------------------------------
 784  */
 785
 786 static int
 787 vmci_allocate_interrupt_resources(struct vmci_softc *sc)
 788 {
 789         struct resource *irq;
 790         int flags, i, rid;
 791
 792         flags = RF_ACTIVE;
 793         flags |= (sc->vmci_num_intr == 1) ? RF_SHAREABLE : 0;
 794         rid = (sc->vmci_intr_type == VMCI_INTR_TYPE_INTX) ? 0 : 1;
 795
 796         for (i = 0; i < sc->vmci_num_intr; i++, rid++) {
 797                 irq = bus_alloc_resource_any(sc->vmci_dev, SYS_RES_IRQ, &rid,
 798                     flags);
 799                 if (irq == NULL)
 800                         return (ENXIO);
 801                 sc->vmci_intrs[i].vmci_irq = irq;
 802                 sc->vmci_intrs[i].vmci_rid = rid;
 803         }
 804
 805         return (0);
 806 }
 807
 808 /*
 809  *------------------------------------------------------------------------------
 810  *
 811  * vmci_setup_interrupts --
 812  *
 813  *     Sets up the interrupts.
 814  *
 815  * Results:
 816  *     0 if success, appropriate error code from bus_setup_intr otherwise.
 817  *
 818  * Side effects:
 819  *     Interrupt handler gets attached.
 820  *
 821  *------------------------------------------------------------------------------
 822  */
 823
 824 static int
 825 vmci_setup_interrupts(struct vmci_softc *sc)
 826 {
 827         struct vmci_interrupt *intr;
 828         int error, flags;
 829
 830         flags = INTR_TYPE_NET | INTR_MPSAFE;
 831         if (sc->vmci_num_intr > 1)
 832                 flags |= INTR_EXCL;
 833
 834         intr = &sc->vmci_intrs[0];
 835         error = bus_setup_intr(sc->vmci_dev, intr->vmci_irq, flags, NULL,
 836             vmci_interrupt, NULL, &intr->vmci_handler);
 837         if (error)
 838                 return (error);
 839         bus_describe_intr(sc->vmci_dev, intr->vmci_irq, intr->vmci_handler,
 840             "vmci_interrupt");
 841
 842         if (sc->vmci_num_intr == 2) {
 843                 intr = &sc->vmci_intrs[1];
 844                 error = bus_setup_intr(sc->vmci_dev, intr->vmci_irq, flags,
 845                     NULL, vmci_interrupt_bm, NULL, &intr->vmci_handler);
 846                 if (error)
 847                         return (error);
 848                 bus_describe_intr(sc->vmci_dev, intr->vmci_irq,
 849                     intr->vmci_handler, "vmci_interrupt_bm");
 850         }
 851
 852         return (0);
 853 }
 854
 855 /*
 856  *------------------------------------------------------------------------------
 857  *
 858  * vmci_interrupt --
 859  *
 860  *     Interrupt handler for legacy or MSI interrupt, or for first MSI-X
 861  *     interrupt (vector VMCI_INTR_DATAGRAM).
 862  *
 863  * Results:
 864  *     None.
 865  *
 866  * Side effects:
 867  *     None.
 868  *
 869  *------------------------------------------------------------------------------
 870  */
 871
 872 static void
 873 vmci_interrupt(void *arg)
 874 {
 875
 876         if (vmci_sc->vmci_num_intr == 2)
 877                 taskqueue_enqueue(taskqueue_swi,
 878                     &vmci_sc->vmci_interrupt_dq_task);
 879         else {
 880                 unsigned int icr;
 881
 882                 icr = inl(vmci_sc->vmci_ioaddr + VMCI_ICR_ADDR);
 883                 if (icr == 0 || icr == 0xffffffff)
 884                         return;
 885                 if (icr & VMCI_ICR_DATAGRAM) {
 886                         taskqueue_enqueue(taskqueue_swi,
 887                             &vmci_sc->vmci_interrupt_dq_task);
 888                         icr &= ~VMCI_ICR_DATAGRAM;
 889                 }
 890                 if (icr & VMCI_ICR_NOTIFICATION) {
 891                         taskqueue_enqueue(taskqueue_swi,
 892                             &vmci_sc->vmci_interrupt_bm_task);
 893                         icr &= ~VMCI_ICR_NOTIFICATION;
 894                 }
 895                 if (icr != 0)
 896                         VMCI_LOG_INFO(LGPFX"Ignoring unknown interrupt "
 897                             "cause");
 898         }
 899 }
 900
 901 /*
 902  *------------------------------------------------------------------------------
 903  *
 904  * vmci_interrupt_bm --
 905  *
 906  *     Interrupt handler for MSI-X interrupt vector VMCI_INTR_NOTIFICATION,
 907  *     which is for the notification bitmap. Will only get called if we are
 908  *     using MSI-X with exclusive vectors.
 909  *
 910  * Results:
 911  *     None.
 912  *
 913  * Side effects:
 914  *     None.
 915  *
 916  *------------------------------------------------------------------------------
 917  */
 918
 919 static void
 920 vmci_interrupt_bm(void *arg)
 921 {
 922
 923         ASSERT(vmci_sc->vmci_num_intr == 2);
 924         taskqueue_enqueue(taskqueue_swi, &vmci_sc->vmci_interrupt_bm_task);
 925 }
 926
 927 /*
 928  *------------------------------------------------------------------------------
 929  *
 930  * dispatch_datagrams --
 931  *
 932  *     Reads and dispatches incoming datagrams.
 933  *
 934  * Results:
 935  *     None.
 936  *
 937  * Side effects:
 938  *     Reads data from the device.
 939  *
 940  *------------------------------------------------------------------------------
 941  */
 942
 943 static void
 944 dispatch_datagrams(void *context, int data)
 945 {
 946
 947         if (data_buffer == NULL)
 948                 VMCI_LOG_INFO(LGPFX"dispatch_datagrams(): no buffer "
 949                     "present");
 950
 951         vmci_read_datagrams_from_port((vmci_io_handle) 0,
 952             vmci_sc->vmci_ioaddr + VMCI_DATA_IN_ADDR,
 953             data_buffer, data_buffer_size);
 954 }
 955
 956 /*
 957  *------------------------------------------------------------------------------
 958  *
 959  * process_bitmap --
 960  *
 961  *     Scans the notification bitmap for raised flags, clears them and handles
 962  *     the notifications.
 963  *
 964  * Results:
 965  *     None.
 966  *
 967  * Side effects:
 968  *     None.
 969  *
 970  *------------------------------------------------------------------------------
 971  */
 972
 973 static void
 974 process_bitmap(void *context, int data)
 975 {
 976
 977         if (vmci_sc->vmci_notifications_bitmap.dma_vaddr == NULL)
 978                 VMCI_LOG_INFO(LGPFX"process_bitmaps(): no bitmap present");
 979
 980         vmci_scan_notification_bitmap(
 981             vmci_sc->vmci_notifications_bitmap.dma_vaddr);
 982 }
 983
 984 /*
 985  *------------------------------------------------------------------------------
 986  *
 987  * vmci_dismantle_interrupts --
 988  *
 989  *     Releases resources, detaches the interrupt handler and drains the task
 990  *     queue.
 991  *
 992  * Results:
 993  *     None.
 994  *
 995  * Side effects:
 996  *     No more interrupts.
 997  *
 998  *------------------------------------------------------------------------------
 999  */
1000
1001 static void
1002 vmci_dismantle_interrupts(struct vmci_softc *sc)
1003 {
1004         struct vmci_interrupt *intr;
1005         int i;
1006
1007         for (i = 0; i < sc->vmci_num_intr; i++) {
1008                 intr = &sc->vmci_intrs[i];
1009                 if (intr->vmci_handler != NULL) {
1010                         bus_teardown_intr(sc->vmci_dev, intr->vmci_irq,
1011                             intr->vmci_handler);
1012                         intr->vmci_handler = NULL;
1013                 }
1014                 if (intr->vmci_irq != NULL) {
1015                         bus_release_resource(sc->vmci_dev, SYS_RES_IRQ,
1016                             intr->vmci_rid, intr->vmci_irq);
1017                         intr->vmci_irq = NULL;
1018                         intr->vmci_rid = -1;
1019                 }
1020         }
1021
1022         if ((sc->vmci_intr_type != VMCI_INTR_TYPE_INTX) &&
1023             (sc->vmci_num_intr))
1024                 pci_release_msi(sc->vmci_dev);
1025
1026         taskqueue_drain(taskqueue_swi, &sc->vmci_interrupt_dq_task);
1027         taskqueue_drain(taskqueue_swi, &sc->vmci_interrupt_bm_task);
1028
1029         if (data_buffer != NULL)
1030                 free(data_buffer, M_DEVBUF);
1031 }
1032
1033 /*
1034  *------------------------------------------------------------------------------
1035  *
1036  * vmci_delayed_work_fn_cb --
1037  *
1038  *     Callback function that executes the queued up delayed work functions.
1039  *
1040  * Results:
1041  *     None.
1042  *
1043  * Side effects:
1044  *     None.
1045  *
1046  *------------------------------------------------------------------------------
1047  */
1048
1049 static void
1050 vmci_delayed_work_fn_cb(void *context, int data)
1051 {
1052         vmci_list(vmci_delayed_work_info) temp_list;
1053
1054         vmci_list_init(&temp_list);
1055
1056         /*
1057          * Swap vmci_delayed_work_infos list with the empty temp_list while
1058          * holding a lock. vmci_delayed_work_infos would then be an empty list
1059          * and temp_list would contain the elements from the original
1060          * vmci_delayed_work_infos. Finally, iterate through temp_list
1061          * executing the delayed callbacks.
1062          */
1063
1064         mtx_lock(&vmci_sc->vmci_delayed_work_lock);
1065         vmci_list_swap(&temp_list, &vmci_sc->vmci_delayed_work_infos,
1066             vmci_delayed_work_info, entry);
1067         mtx_unlock(&vmci_sc->vmci_delayed_work_lock);
1068
1069         while (!vmci_list_empty(&temp_list)) {
1070                 struct vmci_delayed_work_info *delayed_work_info =
1071                     vmci_list_first(&temp_list);
1072
1073                 delayed_work_info->work_fn(delayed_work_info->data);
1074
1075                 vmci_list_remove(delayed_work_info, entry);
1076                 vmci_free_kernel_mem(delayed_work_info,
1077                     sizeof(*delayed_work_info));
1078         }
1079 }
1080
1081 /*
1082  *------------------------------------------------------------------------------
1083  *
1084  * vmci_schedule_delayed_work_fn --
1085  *
1086  *     Schedule the specified callback.
1087  *
1088  * Results:
1089  *     0 if success, error code otherwise.
1090  *
1091  * Side effects:
1092  *     None.
1093  *
1094  *------------------------------------------------------------------------------
1095  */
1096
1097 int
1098 vmci_schedule_delayed_work_fn(vmci_work_fn *work_fn, void *data)
1099 {
1100         struct vmci_delayed_work_info *delayed_work_info;
1101
1102         delayed_work_info = vmci_alloc_kernel_mem(sizeof(*delayed_work_info),
1103             VMCI_MEMORY_ATOMIC);
1104
1105         if (!delayed_work_info)
1106                 return (VMCI_ERROR_NO_MEM);
1107
1108         delayed_work_info->work_fn = work_fn;
1109         delayed_work_info->data = data;
1110         mtx_lock(&vmci_sc->vmci_delayed_work_lock);
1111         vmci_list_insert(&vmci_sc->vmci_delayed_work_infos,
1112             delayed_work_info, entry);
1113         mtx_unlock(&vmci_sc->vmci_delayed_work_lock);
1114
1115         taskqueue_enqueue(taskqueue_thread,
1116             &vmci_sc->vmci_delayed_work_task);
1117
1118         return (VMCI_SUCCESS);
1119 }
1120
1121 /*
1122  *------------------------------------------------------------------------------
1123  *
1124  * vmci_send_datagram --
1125  *
1126  *     VM to hypervisor call mechanism.
1127  *
1128  * Results:
1129  *     The result of the hypercall.
1130  *
1131  * Side effects:
1132  *     None.
1133  *
1134  *------------------------------------------------------------------------------
1135  */
1136
1137 int
1138 vmci_send_datagram(struct vmci_datagram *dg)
1139 {
1140         int result;
1141
1142         if (dg == NULL)
1143                 return (VMCI_ERROR_INVALID_ARGS);
1144
1145         /*
1146          * Need to acquire spinlock on the device because
1147          * the datagram data may be spread over multiple pages and the monitor
1148          * may interleave device user rpc calls from multiple VCPUs. Acquiring
1149          * the spinlock precludes that possibility. Disabling interrupts to
1150          * avoid incoming datagrams during a "rep out" and possibly landing up
1151          * in this function.
1152          */
1153         mtx_lock_spin(&vmci_sc->vmci_spinlock);
1154
1155         /*
1156          * Send the datagram and retrieve the return value from the result
1157          * register.
1158          */
1159         __asm__ __volatile__(
1160             "cld\n\t"
1161             "rep outsb\n\t"
1162             : /* No output. */
1163             : "d"(vmci_sc->vmci_ioaddr + VMCI_DATA_OUT_ADDR),
1164             "c"(VMCI_DG_SIZE(dg)), "S"(dg)
1165             );
1166
1167         /*
1168          * XXX: Should read result high port as well when updating handlers to
1169          * return 64bit.
1170          */
1171
1172         result = bus_space_read_4(vmci_sc->vmci_iot0,
1173             vmci_sc->vmci_ioh0, VMCI_RESULT_LOW_ADDR);
1174         mtx_unlock_spin(&vmci_sc->vmci_spinlock);
1175
1176         return (result);
1177 }