sys/dev/vmware/vmci/vmci_queue_pair.c

   1 /*-
   2  * Copyright (c) 2018 VMware, Inc.
   3  *
   4  * SPDX-License-Identifier: (BSD-2-Clause OR GPL-2.0)
   5  */
   6
   7 /* VMCI QueuePair API implementation. */
   8
   9 #include <sys/cdefs.h>
  10 __FBSDID("$FreeBSD$");
  11
  12 #include "vmci.h"
  13 #include "vmci_driver.h"
  14 #include "vmci_event.h"
  15 #include "vmci_kernel_api.h"
  16 #include "vmci_kernel_defs.h"
  17 #include "vmci_queue_pair.h"
  18
  19 #define LGPFX   "vmci_queue_pair: "
  20
  21 struct queue_pair_entry {
  22         vmci_list_item(queue_pair_entry) list_item;
  23         struct vmci_handle handle;
  24         vmci_id         peer;
  25         uint32_t        flags;
  26         uint64_t        produce_size;
  27         uint64_t        consume_size;
  28         uint32_t        ref_count;
  29 };
  30
  31 struct qp_guest_endpoint {
  32         struct queue_pair_entry qp;
  33         uint64_t        num_ppns;
  34         void            *produce_q;
  35         void            *consume_q;
  36         bool            hibernate_failure;
  37         struct ppn_set  ppn_set;
  38 };
  39
  40 struct queue_pair_list {
  41         vmci_list(queue_pair_entry) head;
  42         volatile int    hibernate;
  43         vmci_mutex      mutex;
  44 };
  45
  46 #define QPE_NUM_PAGES(_QPE)                                             \
  47         ((uint32_t)(CEILING(_QPE.produce_size, PAGE_SIZE) +             \
  48         CEILING(_QPE.consume_size, PAGE_SIZE) + 2))
  49
  50 static struct queue_pair_list qp_guest_endpoints;
  51
  52 static struct   queue_pair_entry *queue_pair_list_find_entry(
  53                     struct queue_pair_list *qp_list, struct vmci_handle handle);
  54 static void     queue_pair_list_add_entry(struct queue_pair_list *qp_list,
  55                     struct queue_pair_entry *entry);
  56 static void     queue_pair_list_remove_entry(struct queue_pair_list *qp_list,
  57                     struct queue_pair_entry *entry);
  58 static struct   queue_pair_entry *queue_pair_list_get_head(
  59                     struct queue_pair_list *qp_list);
  60 static int      queue_pair_notify_peer_local(bool attach,
  61                     struct vmci_handle handle);
  62 static struct   qp_guest_endpoint *qp_guest_endpoint_create(
  63                     struct vmci_handle handle, vmci_id peer, uint32_t flags,
  64                     uint64_t produce_size, uint64_t consume_size,
  65                     void *produce_q, void *consume_q);
  66 static void     qp_guest_endpoint_destroy(struct qp_guest_endpoint *entry);
  67 static int      vmci_queue_pair_alloc_hypercall(
  68                     const struct qp_guest_endpoint *entry);
  69 static int      vmci_queue_pair_alloc_guest_work(struct vmci_handle *handle,
  70                     struct vmci_queue **produce_q, uint64_t produce_size,
  71                     struct vmci_queue **consume_q, uint64_t consume_size,
  72                     vmci_id peer, uint32_t flags,
  73                     vmci_privilege_flags priv_flags);
  74 static int      vmci_queue_pair_detach_guest_work(struct vmci_handle handle);
  75 static int      vmci_queue_pair_detach_hypercall(struct vmci_handle handle);
  76
  77 /*
  78  *------------------------------------------------------------------------------
  79  *
  80  * vmci_queue_pair_alloc --
  81  *
  82  *     Allocates a VMCI QueuePair. Only checks validity of input arguments. The
  83  *     real work is done in the host or guest specific function.
  84  *
  85  * Results:
  86  *     VMCI_SUCCESS on success, appropriate error code otherwise.
  87  *
  88  * Side effects:
  89  *     None.
  90  *
  91  *------------------------------------------------------------------------------
  92  */
  93
  94 int
  95 vmci_queue_pair_alloc(struct vmci_handle *handle, struct vmci_queue **produce_q,
  96     uint64_t produce_size, struct vmci_queue **consume_q, uint64_t consume_size,
  97     vmci_id peer, uint32_t flags, vmci_privilege_flags priv_flags)
  98 {
  99
 100         if (!handle || !produce_q || !consume_q ||
 101             (!produce_size && !consume_size) || (flags & ~VMCI_QP_ALL_FLAGS))
 102                 return (VMCI_ERROR_INVALID_ARGS);
 103
 104         return (vmci_queue_pair_alloc_guest_work(handle, produce_q,
 105             produce_size, consume_q, consume_size, peer, flags, priv_flags));
 106 }
 107
 108 /*
 109  *------------------------------------------------------------------------------
 110  *
 111  * vmci_queue_pair_detach --
 112  *
 113  *     Detaches from a VMCI QueuePair. Only checks validity of input argument.
 114  *     Real work is done in the host or guest specific function.
 115  *
 116  * Results:
 117  *     Success or failure.
 118  *
 119  * Side effects:
 120  *     Memory is freed.
 121  *
 122  *------------------------------------------------------------------------------
 123  */
 124
 125 int
 126 vmci_queue_pair_detach(struct vmci_handle handle)
 127 {
 128
 129         if (VMCI_HANDLE_INVALID(handle))
 130                 return (VMCI_ERROR_INVALID_ARGS);
 131
 132         return (vmci_queue_pair_detach_guest_work(handle));
 133 }
 134
 135 /*
 136  *------------------------------------------------------------------------------
 137  *
 138  * queue_pair_list_init --
 139  *
 140  *     Initializes the list of QueuePairs.
 141  *
 142  * Results:
 143  *     Success or failure.
 144  *
 145  * Side effects:
 146  *     None.
 147  *
 148  *------------------------------------------------------------------------------
 149  */
 150
 151 static inline int
 152 queue_pair_list_init(struct queue_pair_list *qp_list)
 153 {
 154         int ret;
 155
 156         vmci_list_init(&qp_list->head);
 157         atomic_store_int(&qp_list->hibernate, 0);
 158         ret = vmci_mutex_init(&qp_list->mutex, "VMCI QP List lock");
 159         return (ret);
 160 }
 161
 162 /*
 163  *------------------------------------------------------------------------------
 164  *
 165  * queue_pair_list_destroy --
 166  *
 167  *     Destroy the list's mutex.
 168  *
 169  * Results:
 170  *     None.
 171  *
 172  * Side effects:
 173  *     None.
 174  *
 175  *------------------------------------------------------------------------------
 176  */
 177
 178 static inline void
 179 queue_pair_list_destroy(struct queue_pair_list *qp_list)
 180 {
 181
 182         vmci_mutex_destroy(&qp_list->mutex);
 183         vmci_list_init(&qp_list->head);
 184 }
 185
 186 /*
 187  *------------------------------------------------------------------------------
 188  *
 189  * queue_pair_list_find_entry --
 190  *
 191  *     Finds the entry in the list corresponding to a given handle. Assumes that
 192  *     the list is locked.
 193  *
 194  * Results:
 195  *     Pointer to entry.
 196  *
 197  * Side effects:
 198  *     None.
 199  *
 200  *------------------------------------------------------------------------------
 201  */
 202
 203 static struct queue_pair_entry *
 204 queue_pair_list_find_entry(struct queue_pair_list *qp_list,
 205     struct vmci_handle handle)
 206 {
 207         struct queue_pair_entry *next;
 208
 209         if (VMCI_HANDLE_INVALID(handle))
 210                 return (NULL);
 211
 212         vmci_list_scan(next, &qp_list->head, list_item) {
 213                 if (VMCI_HANDLE_EQUAL(next->handle, handle))
 214                         return (next);
 215         }
 216
 217         return (NULL);
 218 }
 219
 220 /*
 221  *------------------------------------------------------------------------------
 222  *
 223  * queue_pair_list_add_entry --
 224  *
 225  *     Adds the given entry to the list. Assumes that the list is locked.
 226  *
 227  * Results:
 228  *     None.
 229  *
 230  * Side effects:
 231  *     None.
 232  *
 233  *------------------------------------------------------------------------------
 234  */
 235
 236 static void
 237 queue_pair_list_add_entry(struct queue_pair_list *qp_list,
 238     struct queue_pair_entry *entry)
 239 {
 240
 241         if (entry)
 242                 vmci_list_insert(&qp_list->head, entry, list_item);
 243 }
 244
 245 /*
 246  *------------------------------------------------------------------------------
 247  *
 248  * queue_pair_list_remove_entry --
 249  *
 250  *     Removes the given entry from the list. Assumes that the list is locked.
 251  *
 252  * Results:
 253  *     None.
 254  *
 255  * Side effects:
 256  *     None.
 257  *
 258  *------------------------------------------------------------------------------
 259  */
 260
 261 static void
 262 queue_pair_list_remove_entry(struct queue_pair_list *qp_list,
 263     struct queue_pair_entry *entry)
 264 {
 265
 266         if (entry)
 267                 vmci_list_remove(entry, list_item);
 268 }
 269
 270 /*
 271  *------------------------------------------------------------------------------
 272  *
 273  * queue_pair_list_get_head --
 274  *
 275  *     Returns the entry from the head of the list. Assumes that the list is
 276  *     locked.
 277  *
 278  * Results:
 279  *     Pointer to entry.
 280  *
 281  * Side effects:
 282  *     None.
 283  *
 284  *------------------------------------------------------------------------------
 285  */
 286
 287 static struct queue_pair_entry *
 288 queue_pair_list_get_head(struct queue_pair_list *qp_list)
 289 {
 290
 291         return (vmci_list_first(&qp_list->head));
 292 }
 293
 294 /*
 295  *------------------------------------------------------------------------------
 296  *
 297  * vmci_qp_guest_endpoints_init --
 298  *
 299  *     Initalizes data structure state keeping track of queue pair guest
 300  *     endpoints.
 301  *
 302  * Results:
 303  *     VMCI_SUCCESS on success and appropriate failure code otherwise.
 304  *
 305  * Side effects:
 306  *     None.
 307  *
 308  *------------------------------------------------------------------------------
 309  */
 310
 311 int
 312 vmci_qp_guest_endpoints_init(void)
 313 {
 314
 315         return (queue_pair_list_init(&qp_guest_endpoints));
 316 }
 317
 318 /*
 319  *------------------------------------------------------------------------------
 320  *
 321  * vmci_qp_guest_endpoints_exit --
 322  *
 323  *     Destroys all guest queue pair endpoints. If active guest queue pairs
 324  *     still exist, hypercalls to attempt detach from these queue pairs will be
 325  *     made. Any failure to detach is silently ignored.
 326  *
 327  * Results:
 328  *     None.
 329  *
 330  * Side effects:
 331  *     None.
 332  *
 333  *------------------------------------------------------------------------------
 334  */
 335
 336 void
 337 vmci_qp_guest_endpoints_exit(void)
 338 {
 339         struct qp_guest_endpoint *entry;
 340
 341         vmci_mutex_acquire(&qp_guest_endpoints.mutex);
 342
 343         while ((entry =
 344             (struct qp_guest_endpoint *)queue_pair_list_get_head(
 345             &qp_guest_endpoints)) != NULL) {
 346                 /*
 347                  * Don't make a hypercall for local QueuePairs.
 348                  */
 349                 if (!(entry->qp.flags & VMCI_QPFLAG_LOCAL))
 350                         vmci_queue_pair_detach_hypercall(entry->qp.handle);
 351                 /*
 352                  * We cannot fail the exit, so let's reset ref_count.
 353                  */
 354                 entry->qp.ref_count = 0;
 355                 queue_pair_list_remove_entry(&qp_guest_endpoints, &entry->qp);
 356                 qp_guest_endpoint_destroy(entry);
 357         }
 358
 359         atomic_store_int(&qp_guest_endpoints.hibernate, 0);
 360         vmci_mutex_release(&qp_guest_endpoints.mutex);
 361         queue_pair_list_destroy(&qp_guest_endpoints);
 362 }
 363
 364 /*
 365  *------------------------------------------------------------------------------
 366  *
 367  * vmci_qp_guest_endpoints_sync --
 368  *
 369  *     Use this as a synchronization point when setting globals, for example,
 370  *     during device shutdown.
 371  *
 372  * Results:
 373  *     true.
 374  *
 375  * Side effects:
 376  *     None.
 377  *
 378  *------------------------------------------------------------------------------
 379  */
 380
 381 void
 382 vmci_qp_guest_endpoints_sync(void)
 383 {
 384
 385         vmci_mutex_acquire(&qp_guest_endpoints.mutex);
 386         vmci_mutex_release(&qp_guest_endpoints.mutex);
 387 }
 388
 389 /*
 390  *------------------------------------------------------------------------------
 391  *
 392  * qp_guest_endpoint_create --
 393  *
 394  *     Allocates and initializes a qp_guest_endpoint structure. Allocates a
 395  *     QueuePair rid (and handle) iff the given entry has an invalid handle.
 396  *     0 through VMCI_RESERVED_RESOURCE_ID_MAX are reserved handles. Assumes
 397  *     that the QP list mutex is held by the caller.
 398  *
 399  * Results:
 400  *     Pointer to structure intialized.
 401  *
 402  * Side effects:
 403  *     None.
 404  *
 405  *------------------------------------------------------------------------------
 406  */
 407
 408 struct qp_guest_endpoint *
 409 qp_guest_endpoint_create(struct vmci_handle handle, vmci_id peer,
 410     uint32_t flags, uint64_t produce_size, uint64_t consume_size,
 411     void *produce_q, void *consume_q)
 412 {
 413         struct qp_guest_endpoint *entry;
 414         static vmci_id queue_pair_rid;
 415         const uint64_t num_ppns = CEILING(produce_size, PAGE_SIZE) +
 416             CEILING(consume_size, PAGE_SIZE) +
 417             2; /* One page each for the queue headers. */
 418
 419         queue_pair_rid = VMCI_RESERVED_RESOURCE_ID_MAX + 1;
 420
 421         ASSERT((produce_size || consume_size) && produce_q && consume_q);
 422
 423         if (VMCI_HANDLE_INVALID(handle)) {
 424                 vmci_id context_id = vmci_get_context_id();
 425                 vmci_id old_rid = queue_pair_rid;
 426
 427                 /*
 428                  * Generate a unique QueuePair rid.  Keep on trying until we
 429                  * wrap around in the RID space.
 430                  */
 431                 ASSERT(old_rid > VMCI_RESERVED_RESOURCE_ID_MAX);
 432                 do {
 433                         handle = VMCI_MAKE_HANDLE(context_id, queue_pair_rid);
 434                         entry =
 435                             (struct qp_guest_endpoint *)
 436                             queue_pair_list_find_entry(&qp_guest_endpoints,
 437                             handle);
 438                         queue_pair_rid++;
 439                         if (UNLIKELY(!queue_pair_rid)) {
 440                                 /*
 441                                  * Skip the reserved rids.
 442                                  */
 443                                 queue_pair_rid =
 444                                     VMCI_RESERVED_RESOURCE_ID_MAX + 1;
 445                         }
 446                 } while (entry && queue_pair_rid != old_rid);
 447
 448                 if (UNLIKELY(entry != NULL)) {
 449                         ASSERT(queue_pair_rid == old_rid);
 450                         /*
 451                          * We wrapped around --- no rids were free.
 452                          */
 453                         return (NULL);
 454                 }
 455         }
 456
 457         ASSERT(!VMCI_HANDLE_INVALID(handle) &&
 458             queue_pair_list_find_entry(&qp_guest_endpoints, handle) == NULL);
 459         entry = vmci_alloc_kernel_mem(sizeof(*entry), VMCI_MEMORY_NORMAL);
 460         if (entry) {
 461                 entry->qp.handle = handle;
 462                 entry->qp.peer = peer;
 463                 entry->qp.flags = flags;
 464                 entry->qp.produce_size = produce_size;
 465                 entry->qp.consume_size = consume_size;
 466                 entry->qp.ref_count = 0;
 467                 entry->num_ppns = num_ppns;
 468                 memset(&entry->ppn_set, 0, sizeof(entry->ppn_set));
 469                 entry->produce_q = produce_q;
 470                 entry->consume_q = consume_q;
 471         }
 472         return (entry);
 473 }
 474
 475 /*
 476  *------------------------------------------------------------------------------
 477  *
 478  * qp_guest_endpoint_destroy --
 479  *
 480  *     Frees a qp_guest_endpoint structure.
 481  *
 482  * Results:
 483  *     None.
 484  *
 485  * Side effects:
 486  *     None.
 487  *
 488  *------------------------------------------------------------------------------
 489  */
 490
 491 void
 492 qp_guest_endpoint_destroy(struct qp_guest_endpoint *entry)
 493 {
 494
 495         ASSERT(entry);
 496         ASSERT(entry->qp.ref_count == 0);
 497
 498         vmci_free_ppn_set(&entry->ppn_set);
 499         vmci_free_queue(entry->produce_q, entry->qp.produce_size);
 500         vmci_free_queue(entry->consume_q, entry->qp.consume_size);
 501         vmci_free_kernel_mem(entry, sizeof(*entry));
 502 }
 503
 504 /*
 505  *------------------------------------------------------------------------------
 506  *
 507  * vmci_queue_pair_alloc_hypercall --
 508  *
 509  *     Helper to make a QueuePairAlloc hypercall when the driver is
 510  *     supporting a guest device.
 511  *
 512  * Results:
 513  *     Result of the hypercall.
 514  *
 515  * Side effects:
 516  *     Memory is allocated & freed.
 517  *
 518  *------------------------------------------------------------------------------
 519  */
 520 static int
 521 vmci_queue_pair_alloc_hypercall(const struct qp_guest_endpoint *entry)
 522 {
 523         struct vmci_queue_pair_alloc_msg *alloc_msg;
 524         size_t msg_size;
 525         int result;
 526
 527         if (!entry || entry->num_ppns <= 2)
 528                 return (VMCI_ERROR_INVALID_ARGS);
 529
 530         ASSERT(!(entry->qp.flags & VMCI_QPFLAG_LOCAL));
 531
 532         msg_size = sizeof(*alloc_msg) + (size_t)entry->num_ppns * sizeof(PPN);
 533         alloc_msg = vmci_alloc_kernel_mem(msg_size, VMCI_MEMORY_NORMAL);
 534         if (!alloc_msg)
 535                 return (VMCI_ERROR_NO_MEM);
 536
 537         alloc_msg->hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
 538             VMCI_QUEUEPAIR_ALLOC);
 539         alloc_msg->hdr.src = VMCI_ANON_SRC_HANDLE;
 540         alloc_msg->hdr.payload_size = msg_size - VMCI_DG_HEADERSIZE;
 541         alloc_msg->handle = entry->qp.handle;
 542         alloc_msg->peer = entry->qp.peer;
 543         alloc_msg->flags = entry->qp.flags;
 544         alloc_msg->produce_size = entry->qp.produce_size;
 545         alloc_msg->consume_size = entry->qp.consume_size;
 546         alloc_msg->num_ppns = entry->num_ppns;
 547         result = vmci_populate_ppn_list((uint8_t *)alloc_msg +
 548             sizeof(*alloc_msg), &entry->ppn_set);
 549         if (result == VMCI_SUCCESS)
 550                 result = vmci_send_datagram((struct vmci_datagram *)alloc_msg);
 551         vmci_free_kernel_mem(alloc_msg, msg_size);
 552
 553         return (result);
 554 }
 555
 556 /*
 557  *------------------------------------------------------------------------------
 558  *
 559  * vmci_queue_pair_alloc_guest_work --
 560  *
 561  *     This functions handles the actual allocation of a VMCI queue pair guest
 562  *     endpoint. Allocates physical pages for the queue pair. It makes OS
 563  *     dependent calls through generic wrappers.
 564  *
 565  * Results:
 566  *     Success or failure.
 567  *
 568  * Side effects:
 569  *     Memory is allocated.
 570  *
 571  *------------------------------------------------------------------------------
 572  */
 573
 574 static int
 575 vmci_queue_pair_alloc_guest_work(struct vmci_handle *handle,
 576     struct vmci_queue **produce_q, uint64_t produce_size,
 577     struct vmci_queue **consume_q, uint64_t consume_size, vmci_id peer,
 578     uint32_t flags, vmci_privilege_flags priv_flags)
 579 {
 580         struct qp_guest_endpoint *queue_pair_entry = NULL;
 581         void *my_consume_q = NULL;
 582         void *my_produce_q = NULL;
 583         const uint64_t num_consume_pages = CEILING(consume_size, PAGE_SIZE) + 1;
 584         const uint64_t num_produce_pages = CEILING(produce_size, PAGE_SIZE) + 1;
 585         int result;
 586
 587         ASSERT(handle && produce_q && consume_q &&
 588             (produce_size || consume_size));
 589
 590         if (priv_flags != VMCI_NO_PRIVILEGE_FLAGS)
 591                 return (VMCI_ERROR_NO_ACCESS);
 592
 593         vmci_mutex_acquire(&qp_guest_endpoints.mutex);
 594
 595         if ((atomic_load_int(&qp_guest_endpoints.hibernate) == 1) &&
 596                  !(flags & VMCI_QPFLAG_LOCAL)) {
 597                 /*
 598                  * While guest OS is in hibernate state, creating non-local
 599                  * queue pairs is not allowed after the point where the VMCI
 600                  * guest driver converted the existing queue pairs to local
 601                  * ones.
 602                  */
 603
 604                 result = VMCI_ERROR_UNAVAILABLE;
 605                 goto error;
 606         }
 607
 608         if ((queue_pair_entry =
 609             (struct qp_guest_endpoint *)queue_pair_list_find_entry(
 610             &qp_guest_endpoints, *handle)) != NULL) {
 611                 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) {
 612                         /* Local attach case. */
 613                         if (queue_pair_entry->qp.ref_count > 1) {
 614                                 VMCI_LOG_DEBUG(LGPFX"Error attempting to "
 615                                     "attach more than once.\n");
 616                                 result = VMCI_ERROR_UNAVAILABLE;
 617                                 goto error_keep_entry;
 618                         }
 619
 620                         if (queue_pair_entry->qp.produce_size != consume_size ||
 621                             queue_pair_entry->qp.consume_size != produce_size ||
 622                             queue_pair_entry->qp.flags !=
 623                             (flags & ~VMCI_QPFLAG_ATTACH_ONLY)) {
 624                                 VMCI_LOG_DEBUG(LGPFX"Error mismatched "
 625                                     "queue pair in local attach.\n");
 626                                 result = VMCI_ERROR_QUEUEPAIR_MISMATCH;
 627                                 goto error_keep_entry;
 628                         }
 629
 630                         /*
 631                          * Do a local attach. We swap the consume and produce
 632                          * queues for the attacher and deliver an attach event.
 633                          */
 634                         result = queue_pair_notify_peer_local(true, *handle);
 635                         if (result < VMCI_SUCCESS)
 636                                 goto error_keep_entry;
 637                         my_produce_q = queue_pair_entry->consume_q;
 638                         my_consume_q = queue_pair_entry->produce_q;
 639                         goto out;
 640                 }
 641                 result = VMCI_ERROR_ALREADY_EXISTS;
 642                 goto error_keep_entry;
 643         }
 644
 645         my_produce_q = vmci_alloc_queue(produce_size, flags);
 646         if (!my_produce_q) {
 647                 VMCI_LOG_WARNING(LGPFX"Error allocating pages for produce "
 648                     "queue.\n");
 649                 result = VMCI_ERROR_NO_MEM;
 650                 goto error;
 651         }
 652
 653         my_consume_q = vmci_alloc_queue(consume_size, flags);
 654         if (!my_consume_q) {
 655                 VMCI_LOG_WARNING(LGPFX"Error allocating pages for consume "
 656                     "queue.\n");
 657                 result = VMCI_ERROR_NO_MEM;
 658                 goto error;
 659         }
 660
 661         queue_pair_entry = qp_guest_endpoint_create(*handle, peer, flags,
 662             produce_size, consume_size, my_produce_q, my_consume_q);
 663         if (!queue_pair_entry) {
 664                 VMCI_LOG_WARNING(LGPFX"Error allocating memory in %s.\n",
 665                     __FUNCTION__);
 666                 result = VMCI_ERROR_NO_MEM;
 667                 goto error;
 668         }
 669
 670         result = vmci_alloc_ppn_set(my_produce_q, num_produce_pages,
 671             my_consume_q, num_consume_pages, &queue_pair_entry->ppn_set);
 672         if (result < VMCI_SUCCESS) {
 673                 VMCI_LOG_WARNING(LGPFX"vmci_alloc_ppn_set failed.\n");
 674                 goto error;
 675         }
 676
 677         /*
 678          * It's only necessary to notify the host if this queue pair will be
 679          * attached to from another context.
 680          */
 681         if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) {
 682                 /* Local create case. */
 683                 vmci_id context_id = vmci_get_context_id();
 684
 685                 /*
 686                  * Enforce similar checks on local queue pairs as we do for
 687                  * regular ones. The handle's context must match the creator
 688                  * or attacher context id (here they are both the current
 689                  * context id) and the attach-only flag cannot exist during
 690                  * create. We also ensure specified peer is this context or
 691                  * an invalid one.
 692                  */
 693                 if (queue_pair_entry->qp.handle.context != context_id ||
 694                     (queue_pair_entry->qp.peer != VMCI_INVALID_ID &&
 695                     queue_pair_entry->qp.peer != context_id)) {
 696                         result = VMCI_ERROR_NO_ACCESS;
 697                         goto error;
 698                 }
 699
 700                 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_ATTACH_ONLY) {
 701                         result = VMCI_ERROR_NOT_FOUND;
 702                         goto error;
 703                 }
 704         } else {
 705                 result = vmci_queue_pair_alloc_hypercall(queue_pair_entry);
 706                 if (result < VMCI_SUCCESS) {
 707                         VMCI_LOG_WARNING(
 708                             LGPFX"vmci_queue_pair_alloc_hypercall result = "
 709                             "%d.\n", result);
 710                         goto error;
 711                 }
 712         }
 713
 714         queue_pair_list_add_entry(&qp_guest_endpoints, &queue_pair_entry->qp);
 715
 716 out:
 717         queue_pair_entry->qp.ref_count++;
 718         *handle = queue_pair_entry->qp.handle;
 719         *produce_q = (struct vmci_queue *)my_produce_q;
 720         *consume_q = (struct vmci_queue *)my_consume_q;
 721
 722         /*
 723          * We should initialize the queue pair header pages on a local queue
 724          * pair create. For non-local queue pairs, the hypervisor initializes
 725          * the header pages in the create step.
 726          */
 727         if ((queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) &&
 728             queue_pair_entry->qp.ref_count == 1) {
 729                 vmci_queue_header_init((*produce_q)->q_header, *handle);
 730                 vmci_queue_header_init((*consume_q)->q_header, *handle);
 731         }
 732
 733         vmci_mutex_release(&qp_guest_endpoints.mutex);
 734
 735         return (VMCI_SUCCESS);
 736
 737 error:
 738         vmci_mutex_release(&qp_guest_endpoints.mutex);
 739         if (queue_pair_entry) {
 740                 /* The queues will be freed inside the destroy routine. */
 741                 qp_guest_endpoint_destroy(queue_pair_entry);
 742         } else {
 743                 if (my_produce_q)
 744                         vmci_free_queue(my_produce_q, produce_size);
 745                 if (my_consume_q)
 746                         vmci_free_queue(my_consume_q, consume_size);
 747         }
 748         return (result);
 749
 750 error_keep_entry:
 751         /* This path should only be used when an existing entry was found. */
 752         ASSERT(queue_pair_entry->qp.ref_count > 0);
 753         vmci_mutex_release(&qp_guest_endpoints.mutex);
 754         return (result);
 755 }
 756
 757 /*
 758  *------------------------------------------------------------------------------
 759  *
 760  * vmci_queue_pair_detach_hypercall --
 761  *
 762  *     Helper to make a QueuePairDetach hypercall when the driver is supporting
 763  *     a guest device.
 764  *
 765  * Results:
 766  *     Result of the hypercall.
 767  *
 768  * Side effects:
 769  *     None.
 770  *
 771  *------------------------------------------------------------------------------
 772  */
 773
 774 int
 775 vmci_queue_pair_detach_hypercall(struct vmci_handle handle)
 776 {
 777         struct vmci_queue_pair_detach_msg detach_msg;
 778
 779         detach_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
 780             VMCI_QUEUEPAIR_DETACH);
 781         detach_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
 782         detach_msg.hdr.payload_size = sizeof(handle);
 783         detach_msg.handle = handle;
 784
 785         return (vmci_send_datagram((struct vmci_datagram *)&detach_msg));
 786 }
 787
 788 /*
 789  *------------------------------------------------------------------------------
 790  *
 791  * vmci_queue_pair_detach_guest_work --
 792  *
 793  *     Helper for VMCI QueuePair detach interface. Frees the physical pages for
 794  *     the queue pair.
 795  *
 796  * Results:
 797  *     Success or failure.
 798  *
 799  * Side effects:
 800  *     Memory may be freed.
 801  *
 802  *------------------------------------------------------------------------------
 803  */
 804
 805 static int
 806 vmci_queue_pair_detach_guest_work(struct vmci_handle handle)
 807 {
 808         struct qp_guest_endpoint *entry;
 809         int result;
 810         uint32_t ref_count;
 811
 812         ASSERT(!VMCI_HANDLE_INVALID(handle));
 813
 814         vmci_mutex_acquire(&qp_guest_endpoints.mutex);
 815
 816         entry = (struct qp_guest_endpoint *)queue_pair_list_find_entry(
 817             &qp_guest_endpoints, handle);
 818         if (!entry) {
 819                 vmci_mutex_release(&qp_guest_endpoints.mutex);
 820                 return (VMCI_ERROR_NOT_FOUND);
 821         }
 822
 823         ASSERT(entry->qp.ref_count >= 1);
 824
 825         if (entry->qp.flags & VMCI_QPFLAG_LOCAL) {
 826                 result = VMCI_SUCCESS;
 827
 828                 if (entry->qp.ref_count > 1) {
 829                         result = queue_pair_notify_peer_local(false, handle);
 830
 831                         /*
 832                          * We can fail to notify a local queuepair because we
 833                          * can't allocate. We still want to release the entry
 834                          * if that happens, so don't bail out yet.
 835                          */
 836                 }
 837         } else {
 838                 result = vmci_queue_pair_detach_hypercall(handle);
 839                 if (entry->hibernate_failure) {
 840                         if (result == VMCI_ERROR_NOT_FOUND) {
 841                                 /*
 842                                  * If a queue pair detach failed when entering
 843                                  * hibernation, the guest driver and the device
 844                                  * may disagree on its existence when coming
 845                                  * out of hibernation. The guest driver will
 846                                  * regard it as a non-local queue pair, but
 847                                  * the device state is gone, since the device
 848                                  * has been powered off. In this case, we
 849                                  * treat the queue pair as a local queue pair
 850                                  * with no peer.
 851                                  */
 852
 853                                 ASSERT(entry->qp.ref_count == 1);
 854                                 result = VMCI_SUCCESS;
 855                         }
 856                 }
 857                 if (result < VMCI_SUCCESS) {
 858                         /*
 859                          * We failed to notify a non-local queuepair. That other
 860                          * queuepair might still be accessing the shared
 861                          * memory, so don't release the entry yet. It will get
 862                          * cleaned up by vmci_queue_pair_Exit() if necessary
 863                          * (assuming we are going away, otherwise why did this
 864                          * fail?).
 865                          */
 866
 867                         vmci_mutex_release(&qp_guest_endpoints.mutex);
 868                         return (result);
 869                 }
 870         }
 871
 872         /*
 873          * If we get here then we either failed to notify a local queuepair, or
 874          * we succeeded in all cases.  Release the entry if required.
 875          */
 876
 877         entry->qp.ref_count--;
 878         if (entry->qp.ref_count == 0)
 879                 queue_pair_list_remove_entry(&qp_guest_endpoints, &entry->qp);
 880
 881         /* If we didn't remove the entry, this could change once we unlock. */
 882         ref_count = entry ? entry->qp.ref_count :
 883             0xffffffff; /*
 884                          * Value does not matter, silence the
 885                          * compiler.
 886                          */
 887
 888         vmci_mutex_release(&qp_guest_endpoints.mutex);
 889
 890         if (ref_count == 0)
 891                 qp_guest_endpoint_destroy(entry);
 892         return (result);
 893 }
 894
 895 /*
 896  *------------------------------------------------------------------------------
 897  *
 898  * queue_pair_notify_peer_local --
 899  *
 900  *     Dispatches a queue pair event message directly into the local event
 901  *     queue.
 902  *
 903  * Results:
 904  *     VMCI_SUCCESS on success, error code otherwise
 905  *
 906  * Side effects:
 907  *     None.
 908  *
 909  *------------------------------------------------------------------------------
 910  */
 911
 912 static int
 913 queue_pair_notify_peer_local(bool attach, struct vmci_handle handle)
 914 {
 915         struct vmci_event_msg *e_msg;
 916         struct vmci_event_payload_qp *e_payload;
 917         /* buf is only 48 bytes. */
 918         vmci_id context_id;
 919         context_id = vmci_get_context_id();
 920         char buf[sizeof(*e_msg) + sizeof(*e_payload)];
 921
 922         e_msg = (struct vmci_event_msg *)buf;
 923         e_payload = vmci_event_msg_payload(e_msg);
 924
 925         e_msg->hdr.dst = VMCI_MAKE_HANDLE(context_id, VMCI_EVENT_HANDLER);
 926         e_msg->hdr.src = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
 927             VMCI_CONTEXT_RESOURCE_ID);
 928         e_msg->hdr.payload_size = sizeof(*e_msg) + sizeof(*e_payload) -
 929             sizeof(e_msg->hdr);
 930         e_msg->event_data.event = attach ? VMCI_EVENT_QP_PEER_ATTACH :
 931             VMCI_EVENT_QP_PEER_DETACH;
 932         e_payload->peer_id = context_id;
 933         e_payload->handle = handle;
 934
 935         return (vmci_event_dispatch((struct vmci_datagram *)e_msg));
 936 }