sys/dev/vmware/vmci/vmci_doorbell.c

   1 /*-
   2  * Copyright (c) 2018 VMware, Inc.
   3  *
   4  * SPDX-License-Identifier: (BSD-2-Clause OR GPL-2.0)
   5  */
   6
   7 /* This file implements the VMCI doorbell API. */
   8
   9 #include <sys/cdefs.h>
  10 __FBSDID("$FreeBSD$");
  11
  12 #include <sys/types.h>
  13
  14 #include "vmci_doorbell.h"
  15 #include "vmci_driver.h"
  16 #include "vmci_kernel_api.h"
  17 #include "vmci_kernel_defs.h"
  18 #include "vmci_resource.h"
  19 #include "vmci_utils.h"
  20
  21 #define LGPFX                           "vmci_doorbell: "
  22
  23 #define VMCI_DOORBELL_INDEX_TABLE_SIZE  64
  24 #define VMCI_DOORBELL_HASH(_idx)                                        \
  25         vmci_hash_id((_idx), VMCI_DOORBELL_INDEX_TABLE_SIZE)
  26
  27 /* Describes a doorbell notification handle allocated by the host. */
  28 struct vmci_doorbell_entry {
  29         struct vmci_resource                    resource;
  30         uint32_t                                idx;
  31         vmci_list_item(vmci_doorbell_entry)     idx_list_item;
  32         vmci_privilege_flags                    priv_flags;
  33         bool                                    is_doorbell;
  34         bool                                    run_delayed;
  35         vmci_callback                           notify_cb;
  36         void                                    *client_data;
  37         vmci_event                              destroy_event;
  38         volatile int                            active;
  39 };
  40
  41 struct vmci_doorbell_index_table {
  42         vmci_lock                       lock;
  43         vmci_list(vmci_doorbell_entry)  entries[VMCI_DOORBELL_INDEX_TABLE_SIZE];
  44 };
  45
  46 /* The VMCI index table keeps track of currently registered doorbells. */
  47 static struct vmci_doorbell_index_table vmci_doorbell_it;
  48
  49 /*
  50  * The max_notify_idx is one larger than the currently known bitmap index in
  51  * use, and is used to determine how much of the bitmap needs to be scanned.
  52  */
  53 static uint32_t max_notify_idx;
  54
  55 /*
  56  * The notify_idx_count is used for determining whether there are free entries
  57  * within the bitmap (if notify_idx_count + 1 < max_notify_idx).
  58  */
  59 static uint32_t notify_idx_count;
  60
  61 /*
  62  * The last_notify_idx_reserved is used to track the last index handed out - in
  63  * the case where multiple handles share a notification index, we hand out
  64  * indexes round robin based on last_notify_idx_reserved.
  65  */
  66 static uint32_t last_notify_idx_reserved;
  67
  68 /* This is a one entry cache used to by the index allocation. */
  69 static uint32_t last_notify_idx_released = PAGE_SIZE;
  70
  71 static void     vmci_doorbell_free_cb(void *client_data);
  72 static int      vmci_doorbell_release_cb(void *client_data);
  73 static void     vmci_doorbell_delayed_dispatch_cb(void *data);
  74
  75 /*
  76  *------------------------------------------------------------------------------
  77  *
  78  * vmci_doorbell_init --
  79  *
  80  *     General init code.
  81  *
  82  * Result:
  83  *     VMCI_SUCCESS on success, lock allocation error otherwise.
  84  *
  85  * Side effects:
  86  *     None.
  87  *
  88  *------------------------------------------------------------------------------
  89  */
  90
  91 int
  92 vmci_doorbell_init(void)
  93 {
  94         uint32_t bucket;
  95
  96         for (bucket = 0; bucket < ARRAYSIZE(vmci_doorbell_it.entries);
  97             ++bucket)
  98                 vmci_list_init(&vmci_doorbell_it.entries[bucket]);
  99
 100         return (vmci_init_lock(&vmci_doorbell_it.lock,
 101             "VMCI Doorbell index table lock"));
 102 }
 103
 104 /*
 105  *------------------------------------------------------------------------------
 106  *
 107  * vmci_doorbell_exit --
 108  *
 109  *     General exit code.
 110  *
 111  * Result:
 112  *     None.
 113  *
 114  * Side effects:
 115  *     None.
 116  *
 117  *------------------------------------------------------------------------------
 118  */
 119
 120 void
 121 vmci_doorbell_exit(void)
 122 {
 123
 124         vmci_cleanup_lock(&vmci_doorbell_it.lock);
 125 }
 126
 127 /*
 128  *------------------------------------------------------------------------------
 129  *
 130  * vmci_doorbell_free_cb --
 131  *
 132  *     Callback to free doorbell entry structure when resource is no longer used,
 133  *     i.e. the reference count reached 0.  The entry is freed in
 134  *     vmci_doorbell_destroy(), which is waiting on the signal that gets fired
 135  *     here.
 136  *
 137  * Result:
 138  *     None.
 139  *
 140  * Side effects:
 141  *     Signals VMCI event.
 142  *
 143  *------------------------------------------------------------------------------
 144  */
 145
 146 static void
 147 vmci_doorbell_free_cb(void *client_data)
 148 {
 149         struct vmci_doorbell_entry *entry;
 150
 151         entry = (struct vmci_doorbell_entry *)client_data;
 152         ASSERT(entry);
 153         vmci_signal_event(&entry->destroy_event);
 154 }
 155
 156 /*
 157  *------------------------------------------------------------------------------
 158  *
 159  * vmci_doorbell_release_cb --
 160  *
 161  *     Callback to release the resource reference. It is called by the
 162  *     vmci_wait_on_event function before it blocks.
 163  *
 164  * Result:
 165  *     Always 0.
 166  *
 167  * Side effects:
 168  *     None.
 169  *
 170  *------------------------------------------------------------------------------
 171  */
 172
 173 static int
 174 vmci_doorbell_release_cb(void *client_data)
 175 {
 176         struct vmci_doorbell_entry *entry;
 177
 178         entry  = (struct vmci_doorbell_entry *)client_data;
 179         ASSERT(entry);
 180         vmci_resource_release(&entry->resource);
 181         return (0);
 182 }
 183
 184 /*
 185  *------------------------------------------------------------------------------
 186  *
 187  * vmci_doorbell_get_priv_flags --
 188  *
 189  *     Utility function that retrieves the privilege flags associated with a
 190  *     given doorbell handle. For guest endpoints, the privileges are determined
 191  *     by the context ID, but for host endpoints privileges are associated with
 192  *     the complete handle. Hypervisor endpoints are not yet supported.
 193  *
 194  * Result:
 195  *     VMCI_SUCCESS on success,
 196  *     VMCI_ERROR_NOT_FOUND if handle isn't found,
 197  *     VMCI_ERROR_INVALID_ARGS if handle is invalid.
 198  *
 199  * Side effects:
 200  *     None.
 201  *
 202  *------------------------------------------------------------------------------
 203  */
 204
 205 int
 206 vmci_doorbell_get_priv_flags(struct vmci_handle handle,
 207     vmci_privilege_flags *priv_flags)
 208 {
 209
 210         if (priv_flags == NULL || handle.context == VMCI_INVALID_ID)
 211                 return (VMCI_ERROR_INVALID_ARGS);
 212
 213         if (handle.context == VMCI_HOST_CONTEXT_ID) {
 214                 struct vmci_doorbell_entry *entry;
 215                 struct vmci_resource *resource;
 216
 217                 resource = vmci_resource_get(handle,
 218                     VMCI_RESOURCE_TYPE_DOORBELL);
 219                 if (resource == NULL)
 220                         return (VMCI_ERROR_NOT_FOUND);
 221                 entry = RESOURCE_CONTAINER(
 222                     resource, struct vmci_doorbell_entry, resource);
 223                 *priv_flags = entry->priv_flags;
 224                 vmci_resource_release(resource);
 225         } else if (handle.context == VMCI_HYPERVISOR_CONTEXT_ID) {
 226                 /* Hypervisor endpoints for notifications are not supported. */
 227                 return (VMCI_ERROR_INVALID_ARGS);
 228         } else
 229                 *priv_flags = VMCI_NO_PRIVILEGE_FLAGS;
 230
 231         return (VMCI_SUCCESS);
 232 }
 233
 234 /*
 235  *------------------------------------------------------------------------------
 236  *
 237  * vmci_doorbell_index_table_find --
 238  *
 239  *     Find doorbell entry by bitmap index.
 240  *
 241  * Results:
 242  *     Entry if found, NULL if not.
 243  *
 244  * Side effects:
 245  *     None.
 246  *
 247  *------------------------------------------------------------------------------
 248  */
 249
 250 static struct vmci_doorbell_entry *
 251 vmci_doorbell_index_table_find(uint32_t idx)
 252 {
 253         struct vmci_doorbell_entry *iter;
 254         uint32_t bucket;
 255
 256         bucket = VMCI_DOORBELL_HASH(idx);
 257
 258         vmci_list_scan(iter, &vmci_doorbell_it.entries[bucket], idx_list_item) {
 259                 if (idx == iter->idx)
 260                         return (iter);
 261         }
 262
 263         return (NULL);
 264 }
 265
 266 /*
 267  *------------------------------------------------------------------------------
 268  *
 269  * vmci_doorbell_index_table_add --
 270  *
 271  *     Add the given entry to the index table. This will hold() the entry's
 272  *     resource so that the entry is not deleted before it is removed from the
 273  *     table.
 274  *
 275  * Results:
 276  *     None.
 277  *
 278  * Side effects:
 279  *     None.
 280  *
 281  *------------------------------------------------------------------------------
 282  */
 283
 284 static void
 285 vmci_doorbell_index_table_add(struct vmci_doorbell_entry *entry)
 286 {
 287         uint32_t bucket;
 288         uint32_t new_notify_idx;
 289
 290         ASSERT(entry);
 291
 292         vmci_resource_hold(&entry->resource);
 293
 294         vmci_grab_lock_bh(&vmci_doorbell_it.lock);
 295
 296         /*
 297          * Below we try to allocate an index in the notification bitmap with
 298          * "not too much" sharing between resources. If we use less that the
 299          * full bitmap, we either add to the end if there are no unused flags
 300          * within the currently used area, or we search for unused ones. If we
 301          * use the full bitmap, we allocate the index round robin.
 302          */
 303
 304         if (max_notify_idx < PAGE_SIZE || notify_idx_count < PAGE_SIZE) {
 305                 if (last_notify_idx_released < max_notify_idx &&
 306                     !vmci_doorbell_index_table_find(last_notify_idx_released)) {
 307                         new_notify_idx = last_notify_idx_released;
 308                         last_notify_idx_released = PAGE_SIZE;
 309                 } else {
 310                         bool reused = false;
 311                         new_notify_idx = last_notify_idx_reserved;
 312                         if (notify_idx_count + 1 < max_notify_idx) {
 313                                 do {
 314                                         if (!vmci_doorbell_index_table_find(
 315                                             new_notify_idx)) {
 316                                                 reused = true;
 317                                                 break;
 318                                         }
 319                                         new_notify_idx = (new_notify_idx + 1) %
 320                                             max_notify_idx;
 321                                 } while (new_notify_idx !=
 322                                     last_notify_idx_released);
 323                         }
 324                         if (!reused) {
 325                                 new_notify_idx = max_notify_idx;
 326                                 max_notify_idx++;
 327                         }
 328                 }
 329         } else {
 330                 new_notify_idx = (last_notify_idx_reserved + 1) % PAGE_SIZE;
 331         }
 332         last_notify_idx_reserved = new_notify_idx;
 333         notify_idx_count++;
 334
 335         entry->idx = new_notify_idx;
 336         bucket = VMCI_DOORBELL_HASH(entry->idx);
 337         vmci_list_insert(&vmci_doorbell_it.entries[bucket], entry,
 338             idx_list_item);
 339
 340         vmci_release_lock_bh(&vmci_doorbell_it.lock);
 341 }
 342
 343 /*
 344  *------------------------------------------------------------------------------
 345  *
 346  * vmci_doorbell_index_table_remove --
 347  *
 348  *     Remove the given entry from the index table. This will release() the
 349  *     entry's resource.
 350  *
 351  * Results:
 352  *     None.
 353  *
 354  * Side effects:
 355  *     None.
 356  *
 357  *------------------------------------------------------------------------------
 358  */
 359
 360 static void
 361 vmci_doorbell_index_table_remove(struct vmci_doorbell_entry *entry)
 362 {
 363         ASSERT(entry);
 364
 365         vmci_grab_lock_bh(&vmci_doorbell_it.lock);
 366
 367         vmci_list_remove(entry, idx_list_item);
 368
 369         notify_idx_count--;
 370         if (entry->idx == max_notify_idx - 1) {
 371                 /*
 372                  * If we delete an entry with the maximum known notification
 373                  * index, we take the opportunity to prune the current max. As
 374                  * there might be other unused indices immediately below, we
 375                  * lower the maximum until we hit an index in use
 376                  */
 377
 378                 while (max_notify_idx > 0 &&
 379                     !vmci_doorbell_index_table_find(max_notify_idx - 1))
 380                         max_notify_idx--;
 381         }
 382         last_notify_idx_released = entry->idx;
 383
 384         vmci_release_lock_bh(&vmci_doorbell_it.lock);
 385
 386         vmci_resource_release(&entry->resource);
 387 }
 388
 389 /*
 390  *------------------------------------------------------------------------------
 391  *
 392  * vmci_doorbell_link --
 393  *
 394  *     Creates a link between the given doorbell handle and the given index in
 395  *     the bitmap in the device backend.
 396  *
 397  * Results:
 398  *     VMCI_SUCCESS if success, appropriate error code otherwise.
 399  *
 400  * Side effects:
 401  *     Notification state is created in hypervisor.
 402  *
 403  *------------------------------------------------------------------------------
 404  */
 405
 406 static int
 407 vmci_doorbell_link(struct vmci_handle handle, bool is_doorbell,
 408     uint32_t notify_idx)
 409 {
 410         struct vmci_doorbell_link_msg link_msg;
 411         vmci_id resource_id;
 412
 413         ASSERT(!VMCI_HANDLE_INVALID(handle));
 414
 415         if (is_doorbell)
 416                 resource_id = VMCI_DOORBELL_LINK;
 417         else {
 418                 ASSERT(false);
 419                 return (VMCI_ERROR_UNAVAILABLE);
 420         }
 421
 422         link_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
 423             resource_id);
 424         link_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
 425         link_msg.hdr.payload_size = sizeof(link_msg) - VMCI_DG_HEADERSIZE;
 426         link_msg.handle = handle;
 427         link_msg.notify_idx = notify_idx;
 428
 429         return (vmci_send_datagram((struct vmci_datagram *)&link_msg));
 430 }
 431
 432 /*
 433  *------------------------------------------------------------------------------
 434  *
 435  * vmci_doorbell_unlink --
 436  *
 437  *     Unlinks the given doorbell handle from an index in the bitmap in the
 438  *     device backend.
 439  *
 440  * Results:
 441  *     VMCI_SUCCESS if success, appropriate error code otherwise.
 442  *
 443  * Side effects:
 444  *     Notification state is destroyed in hypervisor.
 445  *
 446  *------------------------------------------------------------------------------
 447  */
 448
 449 static int
 450 vmci_doorbell_unlink(struct vmci_handle handle, bool is_doorbell)
 451 {
 452         struct vmci_doorbell_unlink_msg unlink_msg;
 453         vmci_id resource_id;
 454
 455         ASSERT(!VMCI_HANDLE_INVALID(handle));
 456
 457         if (is_doorbell)
 458                 resource_id = VMCI_DOORBELL_UNLINK;
 459         else {
 460                 ASSERT(false);
 461                 return (VMCI_ERROR_UNAVAILABLE);
 462         }
 463
 464         unlink_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
 465             resource_id);
 466         unlink_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
 467         unlink_msg.hdr.payload_size = sizeof(unlink_msg) - VMCI_DG_HEADERSIZE;
 468         unlink_msg.handle = handle;
 469
 470         return (vmci_send_datagram((struct vmci_datagram *)&unlink_msg));
 471 }
 472
 473 /*
 474  *------------------------------------------------------------------------------
 475  *
 476  * vmci_doorbell_create --
 477  *
 478  *     Creates a doorbell with the given callback. If the handle is
 479  *     VMCI_INVALID_HANDLE, a free handle will be assigned, if possible. The
 480  *     callback can be run immediately (potentially with locks held - the
 481  *     default) or delayed (in a kernel thread) by specifying the flag
 482  *     VMCI_FLAG_DELAYED_CB. If delayed execution is selected, a given callback
 483  *     may not be run if the kernel is unable to allocate memory for the delayed
 484  *     execution (highly unlikely).
 485  *
 486  * Results:
 487  *     VMCI_SUCCESS on success, appropriate error code otherwise.
 488  *
 489  * Side effects:
 490  *     None.
 491  *
 492  *------------------------------------------------------------------------------
 493  */
 494
 495 int
 496 vmci_doorbell_create(struct vmci_handle *handle, uint32_t flags,
 497     vmci_privilege_flags priv_flags, vmci_callback notify_cb, void *client_data)
 498 {
 499         struct vmci_doorbell_entry *entry;
 500         struct vmci_handle new_handle;
 501         int result;
 502
 503         if (!handle || !notify_cb || flags & ~VMCI_FLAG_DELAYED_CB ||
 504             priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS)
 505                 return (VMCI_ERROR_INVALID_ARGS);
 506
 507         entry = vmci_alloc_kernel_mem(sizeof(*entry), VMCI_MEMORY_NORMAL);
 508         if (entry == NULL) {
 509                 VMCI_LOG_WARNING(LGPFX"Failed allocating memory for datagram "
 510                     "entry.\n");
 511                 return (VMCI_ERROR_NO_MEM);
 512         }
 513
 514         if (!vmci_can_schedule_delayed_work() &&
 515             (flags & VMCI_FLAG_DELAYED_CB)) {
 516                 result = VMCI_ERROR_INVALID_ARGS;
 517                 goto free_mem;
 518         }
 519
 520         if (VMCI_HANDLE_INVALID(*handle)) {
 521                 vmci_id context_id;
 522
 523                 context_id = vmci_get_context_id();
 524                 vmci_id resource_id = vmci_resource_get_id(context_id);
 525                 if (resource_id == VMCI_INVALID_ID) {
 526                         result = VMCI_ERROR_NO_HANDLE;
 527                         goto free_mem;
 528                 }
 529                 new_handle = VMCI_MAKE_HANDLE(context_id, resource_id);
 530         } else {
 531                 if (VMCI_INVALID_ID == handle->resource) {
 532                         VMCI_LOG_DEBUG(LGPFX"Invalid argument "
 533                             "(handle=0x%x:0x%x).\n", handle->context,
 534                             handle->resource);
 535                         result = VMCI_ERROR_INVALID_ARGS;
 536                         goto free_mem;
 537                 }
 538                 new_handle = *handle;
 539         }
 540
 541         entry->idx = 0;
 542         entry->priv_flags = priv_flags;
 543         entry->is_doorbell = true;
 544         entry->run_delayed = (flags & VMCI_FLAG_DELAYED_CB) ? true : false;
 545         entry->notify_cb = notify_cb;
 546         entry->client_data = client_data;
 547         atomic_store_int(&entry->active, 0);
 548         vmci_create_event(&entry->destroy_event);
 549
 550         result = vmci_resource_add(&entry->resource,
 551             VMCI_RESOURCE_TYPE_DOORBELL, new_handle, vmci_doorbell_free_cb,
 552             entry);
 553         if (result != VMCI_SUCCESS) {
 554                 VMCI_LOG_WARNING(LGPFX"Failed to add new resource "
 555                     "(handle=0x%x:0x%x).\n", new_handle.context,
 556                     new_handle.resource);
 557                 if (result == VMCI_ERROR_DUPLICATE_ENTRY)
 558                         result = VMCI_ERROR_ALREADY_EXISTS;
 559
 560                 goto destroy;
 561         }
 562
 563         vmci_doorbell_index_table_add(entry);
 564         result = vmci_doorbell_link(new_handle, entry->is_doorbell, entry->idx);
 565         if (VMCI_SUCCESS != result)
 566                 goto destroy_resource;
 567         atomic_store_int(&entry->active, 1);
 568
 569         if (VMCI_HANDLE_INVALID(*handle))
 570                 *handle = new_handle;
 571
 572         return (result);
 573
 574 destroy_resource:
 575         vmci_doorbell_index_table_remove(entry);
 576         vmci_resource_remove(new_handle, VMCI_RESOURCE_TYPE_DOORBELL);
 577 destroy:
 578         vmci_destroy_event(&entry->destroy_event);
 579 free_mem:
 580         vmci_free_kernel_mem(entry, sizeof(*entry));
 581         return (result);
 582 }
 583
 584 /*
 585  *------------------------------------------------------------------------------
 586  *
 587  * vmci_doorbell_destroy --
 588  *
 589  *     Destroys a doorbell previously created with vmci_doorbell_create. This
 590  *     operation may block waiting for a callback to finish.
 591  *
 592  * Results:
 593  *     VMCI_SUCCESS on success, appropriate error code otherwise.
 594  *
 595  * Side effects:
 596  *     May block.
 597  *
 598  *------------------------------------------------------------------------------
 599  */
 600
 601 int
 602 vmci_doorbell_destroy(struct vmci_handle handle)
 603 {
 604         struct vmci_doorbell_entry *entry;
 605         struct vmci_resource *resource;
 606         int result;
 607
 608         if (VMCI_HANDLE_INVALID(handle))
 609                 return (VMCI_ERROR_INVALID_ARGS);
 610
 611         resource = vmci_resource_get(handle, VMCI_RESOURCE_TYPE_DOORBELL);
 612         if (resource == NULL) {
 613                 VMCI_LOG_DEBUG(LGPFX"Failed to destroy doorbell "
 614                     "(handle=0x%x:0x%x).\n", handle.context, handle.resource);
 615                 return (VMCI_ERROR_NOT_FOUND);
 616         }
 617         entry = RESOURCE_CONTAINER(resource, struct vmci_doorbell_entry,
 618             resource);
 619
 620         vmci_doorbell_index_table_remove(entry);
 621
 622         result = vmci_doorbell_unlink(handle, entry->is_doorbell);
 623         if (VMCI_SUCCESS != result) {
 624
 625                 /*
 626                  * The only reason this should fail would be an inconsistency
 627                  * between guest and hypervisor state, where the guest believes
 628                  * it has an active registration whereas the hypervisor doesn't.
 629                  * One case where this may happen is if a doorbell is
 630                  * unregistered following a hibernation at a time where the
 631                  * doorbell state hasn't been restored on the hypervisor side
 632                  * yet. Since the handle has now been removed in the guest,
 633                  * we just print a warning and return success.
 634                  */
 635
 636                 VMCI_LOG_DEBUG(LGPFX"Unlink of %s (handle=0x%x:0x%x) unknown "
 637                     "by hypervisor (error=%d).\n",
 638                     entry->is_doorbell ? "doorbell" : "queuepair",
 639                     handle.context, handle.resource, result);
 640         }
 641
 642         /*
 643          * Now remove the resource from the table.  It might still be in use
 644          * after this, in a callback or still on the delayed work queue.
 645          */
 646
 647         vmci_resource_remove(handle, VMCI_RESOURCE_TYPE_DOORBELL);
 648
 649         /*
 650          * We now wait on the destroyEvent and release the reference we got
 651          * above.
 652          */
 653
 654         vmci_wait_on_event(&entry->destroy_event, vmci_doorbell_release_cb,
 655             entry);
 656
 657         /*
 658          * We know that we are now the only reference to the above entry so
 659          * can safely free it.
 660          */
 661
 662         vmci_destroy_event(&entry->destroy_event);
 663         vmci_free_kernel_mem(entry, sizeof(*entry));
 664
 665         return (VMCI_SUCCESS);
 666 }
 667
 668 /*
 669  *------------------------------------------------------------------------------
 670  *
 671  * vmci_doorbell_notify_as_guest --
 672  *
 673  *     Notify another guest or the host. We send a datagram down to the host
 674  *     via the hypervisor with the notification info.
 675  *
 676  * Results:
 677  *     VMCI_SUCCESS on success, appropriate error code otherwise.
 678  *
 679  * Side effects:
 680  *     May do a hypercall.
 681  *
 682  *------------------------------------------------------------------------------
 683  */
 684
 685 static int
 686 vmci_doorbell_notify_as_guest(struct vmci_handle handle,
 687     vmci_privilege_flags priv_flags)
 688 {
 689         struct vmci_doorbell_notify_msg notify_msg;
 690
 691         notify_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
 692             VMCI_DOORBELL_NOTIFY);
 693         notify_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
 694         notify_msg.hdr.payload_size = sizeof(notify_msg) - VMCI_DG_HEADERSIZE;
 695         notify_msg.handle = handle;
 696
 697         return (vmci_send_datagram((struct vmci_datagram *)&notify_msg));
 698 }
 699
 700 /*
 701  *------------------------------------------------------------------------------
 702  *
 703  * vmci_doorbell_notify --
 704  *
 705  *     Generates a notification on the doorbell identified by the handle. For
 706  *     host side generation of notifications, the caller can specify what the
 707  *     privilege of the calling side is.
 708  *
 709  * Results:
 710  *     VMCI_SUCCESS on success, appropriate error code otherwise.
 711  *
 712  * Side effects:
 713  *     May do a hypercall.
 714  *
 715  *------------------------------------------------------------------------------
 716  */
 717
 718 int
 719 vmci_doorbell_notify(struct vmci_handle dst, vmci_privilege_flags priv_flags)
 720 {
 721         struct vmci_handle src;
 722
 723         if (VMCI_HANDLE_INVALID(dst) ||
 724             (priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS))
 725                 return (VMCI_ERROR_INVALID_ARGS);
 726
 727         src = VMCI_INVALID_HANDLE;
 728
 729         return (vmci_doorbell_notify_as_guest(dst, priv_flags));
 730 }
 731
 732 /*
 733  *------------------------------------------------------------------------------
 734  *
 735  * vmci_doorbell_delayed_dispatch_cb --
 736  *
 737  *     Calls the specified callback in a delayed context.
 738  *
 739  * Results:
 740  *     None.
 741  *
 742  * Side effects:
 743  *     None.
 744  *
 745  *------------------------------------------------------------------------------
 746  */
 747
 748 static void
 749 vmci_doorbell_delayed_dispatch_cb(void *data)
 750 {
 751         struct vmci_doorbell_entry *entry = (struct vmci_doorbell_entry *)data;
 752
 753         ASSERT(data);
 754
 755         entry->notify_cb(entry->client_data);
 756
 757         vmci_resource_release(&entry->resource);
 758 }
 759
 760 /*
 761  *------------------------------------------------------------------------------
 762  *
 763  * vmci_doorbell_sync --
 764  *
 765  *     Use this as a synchronization point when setting globals, for example,
 766  *     during device shutdown.
 767  *
 768  * Results:
 769  *     None.
 770  *
 771  * Side effects:
 772  *     None.
 773  *
 774  *------------------------------------------------------------------------------
 775  */
 776
 777 void
 778 vmci_doorbell_sync(void)
 779 {
 780
 781         vmci_grab_lock_bh(&vmci_doorbell_it.lock);
 782         vmci_release_lock_bh(&vmci_doorbell_it.lock);
 783         vmci_resource_sync();
 784 }
 785
 786 /*
 787  *------------------------------------------------------------------------------
 788  *
 789  * vmci_register_notification_bitmap --
 790  *
 791  *     Register the notification bitmap with the host.
 792  *
 793  * Results:
 794  *     true if the bitmap is registered successfully with the device, false
 795  *     otherwise.
 796  *
 797  * Side effects:
 798  *     None.
 799  *
 800  *------------------------------------------------------------------------------
 801  */
 802
 803 bool
 804 vmci_register_notification_bitmap(PPN bitmap_ppn)
 805 {
 806         struct vmci_notify_bitmap_set_msg bitmap_set_msg;
 807         int result;
 808
 809         /*
 810          * Do not ASSERT() on the guest device here. This function can get
 811          * called during device initialization, so the ASSERT() will fail even
 812          * though the device is (almost) up.
 813          */
 814
 815         bitmap_set_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
 816             VMCI_SET_NOTIFY_BITMAP);
 817         bitmap_set_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
 818         bitmap_set_msg.hdr.payload_size =
 819             sizeof(bitmap_set_msg) - VMCI_DG_HEADERSIZE;
 820         bitmap_set_msg.bitmap_ppn = bitmap_ppn;
 821
 822         result = vmci_send_datagram((struct vmci_datagram *)&bitmap_set_msg);
 823         if (result != VMCI_SUCCESS) {
 824                 VMCI_LOG_DEBUG(LGPFX"Failed to register (PPN=%u) as "
 825                     "notification bitmap (error=%d).\n",
 826                     bitmap_ppn, result);
 827                 return (false);
 828         }
 829         return (true);
 830 }
 831
 832 /*
 833  *------------------------------------------------------------------------------
 834  *
 835  * vmci_doorbell_fire_entries --
 836  *
 837  *     Executes or schedules the handlers for a given notify index.
 838  *
 839  * Result:
 840  *     Notification hash entry if found. NULL otherwise.
 841  *
 842  * Side effects:
 843  *     Whatever the side effects of the handlers are.
 844  *
 845  *------------------------------------------------------------------------------
 846  */
 847
 848 static void
 849 vmci_doorbell_fire_entries(uint32_t notify_idx)
 850 {
 851         struct vmci_doorbell_entry *iter;
 852         uint32_t bucket = VMCI_DOORBELL_HASH(notify_idx);
 853
 854         vmci_grab_lock_bh(&vmci_doorbell_it.lock);
 855
 856         vmci_list_scan(iter, &vmci_doorbell_it.entries[bucket], idx_list_item) {
 857                 if (iter->idx == notify_idx &&
 858                     atomic_load_int(&iter->active) == 1) {
 859                         ASSERT(iter->notify_cb);
 860                         if (iter->run_delayed) {
 861                                 int err;
 862
 863                                 vmci_resource_hold(&iter->resource);
 864                                 err = vmci_schedule_delayed_work(
 865                                     vmci_doorbell_delayed_dispatch_cb, iter);
 866                                 if (err != VMCI_SUCCESS) {
 867                                         vmci_resource_release(&iter->resource);
 868                                         goto out;
 869                                 }
 870                         } else
 871                                 iter->notify_cb(iter->client_data);
 872                 }
 873         }
 874
 875 out:
 876         vmci_release_lock_bh(&vmci_doorbell_it.lock);
 877 }
 878
 879 /*
 880  *------------------------------------------------------------------------------
 881  *
 882  * vmci_scan_notification_bitmap --
 883  *
 884  *     Scans the notification bitmap, collects pending notifications, resets
 885  *     the bitmap and invokes appropriate callbacks.
 886  *
 887  * Results:
 888  *     None.
 889  *
 890  * Side effects:
 891  *     May schedule tasks, allocate memory and run callbacks.
 892  *
 893  *------------------------------------------------------------------------------
 894  */
 895
 896 void
 897 vmci_scan_notification_bitmap(uint8_t *bitmap)
 898 {
 899         uint32_t idx;
 900
 901         ASSERT(bitmap);
 902
 903         for (idx = 0; idx < max_notify_idx; idx++) {
 904                 if (bitmap[idx] & 0x1) {
 905                         bitmap[idx] &= ~1;
 906                         vmci_doorbell_fire_entries(idx);
 907                 }
 908         }
 909 }