2 * Copyright (c) 2018 VMware, Inc.
4 * SPDX-License-Identifier: (BSD-2-Clause OR GPL-2.0)
7 /* This file implements the VMCI doorbell API. */
10 __FBSDID("$FreeBSD$");
12 #include <sys/types.h>
14 #include "vmci_doorbell.h"
15 #include "vmci_driver.h"
16 #include "vmci_kernel_api.h"
17 #include "vmci_kernel_defs.h"
18 #include "vmci_resource.h"
19 #include "vmci_utils.h"
21 #define LGPFX "vmci_doorbell: "
23 #define VMCI_DOORBELL_INDEX_TABLE_SIZE 64
24 #define VMCI_DOORBELL_HASH(_idx) \
25 vmci_hash_id((_idx), VMCI_DOORBELL_INDEX_TABLE_SIZE)
27 /* Describes a doorbell notification handle allocated by the host. */
28 struct vmci_doorbell_entry {
29 struct vmci_resource resource;
31 vmci_list_item(vmci_doorbell_entry) idx_list_item;
32 vmci_privilege_flags priv_flags;
35 vmci_callback notify_cb;
37 vmci_event destroy_event;
41 struct vmci_doorbell_index_table {
43 vmci_list(vmci_doorbell_entry) entries[VMCI_DOORBELL_INDEX_TABLE_SIZE];
46 /* The VMCI index table keeps track of currently registered doorbells. */
47 static struct vmci_doorbell_index_table vmci_doorbell_it;
50 * The max_notify_idx is one larger than the currently known bitmap index in
51 * use, and is used to determine how much of the bitmap needs to be scanned.
53 static uint32_t max_notify_idx;
56 * The notify_idx_count is used for determining whether there are free entries
57 * within the bitmap (if notify_idx_count + 1 < max_notify_idx).
59 static uint32_t notify_idx_count;
62 * The last_notify_idx_reserved is used to track the last index handed out - in
63 * the case where multiple handles share a notification index, we hand out
64 * indexes round robin based on last_notify_idx_reserved.
66 static uint32_t last_notify_idx_reserved;
68 /* This is a one entry cache used to by the index allocation. */
69 static uint32_t last_notify_idx_released = PAGE_SIZE;
71 static void vmci_doorbell_free_cb(void *client_data);
72 static int vmci_doorbell_release_cb(void *client_data);
73 static void vmci_doorbell_delayed_dispatch_cb(void *data);
76 *------------------------------------------------------------------------------
78 * vmci_doorbell_init --
83 * VMCI_SUCCESS on success, lock allocation error otherwise.
88 *------------------------------------------------------------------------------
92 vmci_doorbell_init(void)
96 for (bucket = 0; bucket < ARRAYSIZE(vmci_doorbell_it.entries);
98 vmci_list_init(&vmci_doorbell_it.entries[bucket]);
100 return (vmci_init_lock(&vmci_doorbell_it.lock,
101 "VMCI Doorbell index table lock"));
105 *------------------------------------------------------------------------------
107 * vmci_doorbell_exit --
117 *------------------------------------------------------------------------------
121 vmci_doorbell_exit(void)
124 vmci_cleanup_lock(&vmci_doorbell_it.lock);
128 *------------------------------------------------------------------------------
130 * vmci_doorbell_free_cb --
132 * Callback to free doorbell entry structure when resource is no longer used,
133 * i.e. the reference count reached 0. The entry is freed in
134 * vmci_doorbell_destroy(), which is waiting on the signal that gets fired
141 * Signals VMCI event.
143 *------------------------------------------------------------------------------
147 vmci_doorbell_free_cb(void *client_data)
149 struct vmci_doorbell_entry *entry;
151 entry = (struct vmci_doorbell_entry *)client_data;
153 vmci_signal_event(&entry->destroy_event);
157 *------------------------------------------------------------------------------
159 * vmci_doorbell_release_cb --
161 * Callback to release the resource reference. It is called by the
162 * vmci_wait_on_event function before it blocks.
170 *------------------------------------------------------------------------------
174 vmci_doorbell_release_cb(void *client_data)
176 struct vmci_doorbell_entry *entry;
178 entry = (struct vmci_doorbell_entry *)client_data;
180 vmci_resource_release(&entry->resource);
185 *------------------------------------------------------------------------------
187 * vmci_doorbell_get_priv_flags --
189 * Utility function that retrieves the privilege flags associated with a
190 * given doorbell handle. For guest endpoints, the privileges are determined
191 * by the context ID, but for host endpoints privileges are associated with
192 * the complete handle. Hypervisor endpoints are not yet supported.
195 * VMCI_SUCCESS on success,
196 * VMCI_ERROR_NOT_FOUND if handle isn't found,
197 * VMCI_ERROR_INVALID_ARGS if handle is invalid.
202 *------------------------------------------------------------------------------
206 vmci_doorbell_get_priv_flags(struct vmci_handle handle,
207 vmci_privilege_flags *priv_flags)
210 if (priv_flags == NULL || handle.context == VMCI_INVALID_ID)
211 return (VMCI_ERROR_INVALID_ARGS);
213 if (handle.context == VMCI_HOST_CONTEXT_ID) {
214 struct vmci_doorbell_entry *entry;
215 struct vmci_resource *resource;
217 resource = vmci_resource_get(handle,
218 VMCI_RESOURCE_TYPE_DOORBELL);
219 if (resource == NULL)
220 return (VMCI_ERROR_NOT_FOUND);
221 entry = RESOURCE_CONTAINER(
222 resource, struct vmci_doorbell_entry, resource);
223 *priv_flags = entry->priv_flags;
224 vmci_resource_release(resource);
225 } else if (handle.context == VMCI_HYPERVISOR_CONTEXT_ID) {
226 /* Hypervisor endpoints for notifications are not supported. */
227 return (VMCI_ERROR_INVALID_ARGS);
229 *priv_flags = VMCI_NO_PRIVILEGE_FLAGS;
231 return (VMCI_SUCCESS);
235 *------------------------------------------------------------------------------
237 * vmci_doorbell_index_table_find --
239 * Find doorbell entry by bitmap index.
242 * Entry if found, NULL if not.
247 *------------------------------------------------------------------------------
250 static struct vmci_doorbell_entry *
251 vmci_doorbell_index_table_find(uint32_t idx)
253 struct vmci_doorbell_entry *iter;
256 bucket = VMCI_DOORBELL_HASH(idx);
258 vmci_list_scan(iter, &vmci_doorbell_it.entries[bucket], idx_list_item) {
259 if (idx == iter->idx)
267 *------------------------------------------------------------------------------
269 * vmci_doorbell_index_table_add --
271 * Add the given entry to the index table. This will hold() the entry's
272 * resource so that the entry is not deleted before it is removed from the
281 *------------------------------------------------------------------------------
285 vmci_doorbell_index_table_add(struct vmci_doorbell_entry *entry)
288 uint32_t new_notify_idx;
292 vmci_resource_hold(&entry->resource);
294 vmci_grab_lock_bh(&vmci_doorbell_it.lock);
297 * Below we try to allocate an index in the notification bitmap with
298 * "not too much" sharing between resources. If we use less that the
299 * full bitmap, we either add to the end if there are no unused flags
300 * within the currently used area, or we search for unused ones. If we
301 * use the full bitmap, we allocate the index round robin.
304 if (max_notify_idx < PAGE_SIZE || notify_idx_count < PAGE_SIZE) {
305 if (last_notify_idx_released < max_notify_idx &&
306 !vmci_doorbell_index_table_find(last_notify_idx_released)) {
307 new_notify_idx = last_notify_idx_released;
308 last_notify_idx_released = PAGE_SIZE;
311 new_notify_idx = last_notify_idx_reserved;
312 if (notify_idx_count + 1 < max_notify_idx) {
314 if (!vmci_doorbell_index_table_find(
319 new_notify_idx = (new_notify_idx + 1) %
321 } while (new_notify_idx !=
322 last_notify_idx_released);
325 new_notify_idx = max_notify_idx;
330 new_notify_idx = (last_notify_idx_reserved + 1) % PAGE_SIZE;
332 last_notify_idx_reserved = new_notify_idx;
335 entry->idx = new_notify_idx;
336 bucket = VMCI_DOORBELL_HASH(entry->idx);
337 vmci_list_insert(&vmci_doorbell_it.entries[bucket], entry,
340 vmci_release_lock_bh(&vmci_doorbell_it.lock);
344 *------------------------------------------------------------------------------
346 * vmci_doorbell_index_table_remove --
348 * Remove the given entry from the index table. This will release() the
357 *------------------------------------------------------------------------------
361 vmci_doorbell_index_table_remove(struct vmci_doorbell_entry *entry)
365 vmci_grab_lock_bh(&vmci_doorbell_it.lock);
367 vmci_list_remove(entry, idx_list_item);
370 if (entry->idx == max_notify_idx - 1) {
372 * If we delete an entry with the maximum known notification
373 * index, we take the opportunity to prune the current max. As
374 * there might be other unused indices immediately below, we
375 * lower the maximum until we hit an index in use
378 while (max_notify_idx > 0 &&
379 !vmci_doorbell_index_table_find(max_notify_idx - 1))
382 last_notify_idx_released = entry->idx;
384 vmci_release_lock_bh(&vmci_doorbell_it.lock);
386 vmci_resource_release(&entry->resource);
390 *------------------------------------------------------------------------------
392 * vmci_doorbell_link --
394 * Creates a link between the given doorbell handle and the given index in
395 * the bitmap in the device backend.
398 * VMCI_SUCCESS if success, appropriate error code otherwise.
401 * Notification state is created in hypervisor.
403 *------------------------------------------------------------------------------
407 vmci_doorbell_link(struct vmci_handle handle, bool is_doorbell,
410 struct vmci_doorbell_link_msg link_msg;
413 ASSERT(!VMCI_HANDLE_INVALID(handle));
416 resource_id = VMCI_DOORBELL_LINK;
419 return (VMCI_ERROR_UNAVAILABLE);
422 link_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
424 link_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
425 link_msg.hdr.payload_size = sizeof(link_msg) - VMCI_DG_HEADERSIZE;
426 link_msg.handle = handle;
427 link_msg.notify_idx = notify_idx;
429 return (vmci_send_datagram((struct vmci_datagram *)&link_msg));
433 *------------------------------------------------------------------------------
435 * vmci_doorbell_unlink --
437 * Unlinks the given doorbell handle from an index in the bitmap in the
441 * VMCI_SUCCESS if success, appropriate error code otherwise.
444 * Notification state is destroyed in hypervisor.
446 *------------------------------------------------------------------------------
450 vmci_doorbell_unlink(struct vmci_handle handle, bool is_doorbell)
452 struct vmci_doorbell_unlink_msg unlink_msg;
455 ASSERT(!VMCI_HANDLE_INVALID(handle));
458 resource_id = VMCI_DOORBELL_UNLINK;
461 return (VMCI_ERROR_UNAVAILABLE);
464 unlink_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
466 unlink_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
467 unlink_msg.hdr.payload_size = sizeof(unlink_msg) - VMCI_DG_HEADERSIZE;
468 unlink_msg.handle = handle;
470 return (vmci_send_datagram((struct vmci_datagram *)&unlink_msg));
474 *------------------------------------------------------------------------------
476 * vmci_doorbell_create --
478 * Creates a doorbell with the given callback. If the handle is
479 * VMCI_INVALID_HANDLE, a free handle will be assigned, if possible. The
480 * callback can be run immediately (potentially with locks held - the
481 * default) or delayed (in a kernel thread) by specifying the flag
482 * VMCI_FLAG_DELAYED_CB. If delayed execution is selected, a given callback
483 * may not be run if the kernel is unable to allocate memory for the delayed
484 * execution (highly unlikely).
487 * VMCI_SUCCESS on success, appropriate error code otherwise.
492 *------------------------------------------------------------------------------
496 vmci_doorbell_create(struct vmci_handle *handle, uint32_t flags,
497 vmci_privilege_flags priv_flags, vmci_callback notify_cb, void *client_data)
499 struct vmci_doorbell_entry *entry;
500 struct vmci_handle new_handle;
503 if (!handle || !notify_cb || flags & ~VMCI_FLAG_DELAYED_CB ||
504 priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS)
505 return (VMCI_ERROR_INVALID_ARGS);
507 entry = vmci_alloc_kernel_mem(sizeof(*entry), VMCI_MEMORY_NORMAL);
509 VMCI_LOG_WARNING(LGPFX"Failed allocating memory for datagram "
511 return (VMCI_ERROR_NO_MEM);
514 if (!vmci_can_schedule_delayed_work() &&
515 (flags & VMCI_FLAG_DELAYED_CB)) {
516 result = VMCI_ERROR_INVALID_ARGS;
520 if (VMCI_HANDLE_INVALID(*handle)) {
523 context_id = vmci_get_context_id();
524 vmci_id resource_id = vmci_resource_get_id(context_id);
525 if (resource_id == VMCI_INVALID_ID) {
526 result = VMCI_ERROR_NO_HANDLE;
529 new_handle = VMCI_MAKE_HANDLE(context_id, resource_id);
531 if (VMCI_INVALID_ID == handle->resource) {
532 VMCI_LOG_DEBUG(LGPFX"Invalid argument "
533 "(handle=0x%x:0x%x).\n", handle->context,
535 result = VMCI_ERROR_INVALID_ARGS;
538 new_handle = *handle;
542 entry->priv_flags = priv_flags;
543 entry->is_doorbell = true;
544 entry->run_delayed = (flags & VMCI_FLAG_DELAYED_CB) ? true : false;
545 entry->notify_cb = notify_cb;
546 entry->client_data = client_data;
547 atomic_store_int(&entry->active, 0);
548 vmci_create_event(&entry->destroy_event);
550 result = vmci_resource_add(&entry->resource,
551 VMCI_RESOURCE_TYPE_DOORBELL, new_handle, vmci_doorbell_free_cb,
553 if (result != VMCI_SUCCESS) {
554 VMCI_LOG_WARNING(LGPFX"Failed to add new resource "
555 "(handle=0x%x:0x%x).\n", new_handle.context,
556 new_handle.resource);
557 if (result == VMCI_ERROR_DUPLICATE_ENTRY)
558 result = VMCI_ERROR_ALREADY_EXISTS;
563 vmci_doorbell_index_table_add(entry);
564 result = vmci_doorbell_link(new_handle, entry->is_doorbell, entry->idx);
565 if (VMCI_SUCCESS != result)
566 goto destroy_resource;
567 atomic_store_int(&entry->active, 1);
569 if (VMCI_HANDLE_INVALID(*handle))
570 *handle = new_handle;
575 vmci_doorbell_index_table_remove(entry);
576 vmci_resource_remove(new_handle, VMCI_RESOURCE_TYPE_DOORBELL);
578 vmci_destroy_event(&entry->destroy_event);
580 vmci_free_kernel_mem(entry, sizeof(*entry));
585 *------------------------------------------------------------------------------
587 * vmci_doorbell_destroy --
589 * Destroys a doorbell previously created with vmci_doorbell_create. This
590 * operation may block waiting for a callback to finish.
593 * VMCI_SUCCESS on success, appropriate error code otherwise.
598 *------------------------------------------------------------------------------
602 vmci_doorbell_destroy(struct vmci_handle handle)
604 struct vmci_doorbell_entry *entry;
605 struct vmci_resource *resource;
608 if (VMCI_HANDLE_INVALID(handle))
609 return (VMCI_ERROR_INVALID_ARGS);
611 resource = vmci_resource_get(handle, VMCI_RESOURCE_TYPE_DOORBELL);
612 if (resource == NULL) {
613 VMCI_LOG_DEBUG(LGPFX"Failed to destroy doorbell "
614 "(handle=0x%x:0x%x).\n", handle.context, handle.resource);
615 return (VMCI_ERROR_NOT_FOUND);
617 entry = RESOURCE_CONTAINER(resource, struct vmci_doorbell_entry,
620 vmci_doorbell_index_table_remove(entry);
622 result = vmci_doorbell_unlink(handle, entry->is_doorbell);
623 if (VMCI_SUCCESS != result) {
626 * The only reason this should fail would be an inconsistency
627 * between guest and hypervisor state, where the guest believes
628 * it has an active registration whereas the hypervisor doesn't.
629 * One case where this may happen is if a doorbell is
630 * unregistered following a hibernation at a time where the
631 * doorbell state hasn't been restored on the hypervisor side
632 * yet. Since the handle has now been removed in the guest,
633 * we just print a warning and return success.
636 VMCI_LOG_DEBUG(LGPFX"Unlink of %s (handle=0x%x:0x%x) unknown "
637 "by hypervisor (error=%d).\n",
638 entry->is_doorbell ? "doorbell" : "queuepair",
639 handle.context, handle.resource, result);
643 * Now remove the resource from the table. It might still be in use
644 * after this, in a callback or still on the delayed work queue.
647 vmci_resource_remove(handle, VMCI_RESOURCE_TYPE_DOORBELL);
650 * We now wait on the destroyEvent and release the reference we got
654 vmci_wait_on_event(&entry->destroy_event, vmci_doorbell_release_cb,
658 * We know that we are now the only reference to the above entry so
659 * can safely free it.
662 vmci_destroy_event(&entry->destroy_event);
663 vmci_free_kernel_mem(entry, sizeof(*entry));
665 return (VMCI_SUCCESS);
669 *------------------------------------------------------------------------------
671 * vmci_doorbell_notify_as_guest --
673 * Notify another guest or the host. We send a datagram down to the host
674 * via the hypervisor with the notification info.
677 * VMCI_SUCCESS on success, appropriate error code otherwise.
680 * May do a hypercall.
682 *------------------------------------------------------------------------------
686 vmci_doorbell_notify_as_guest(struct vmci_handle handle,
687 vmci_privilege_flags priv_flags)
689 struct vmci_doorbell_notify_msg notify_msg;
691 notify_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
692 VMCI_DOORBELL_NOTIFY);
693 notify_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
694 notify_msg.hdr.payload_size = sizeof(notify_msg) - VMCI_DG_HEADERSIZE;
695 notify_msg.handle = handle;
697 return (vmci_send_datagram((struct vmci_datagram *)¬ify_msg));
701 *------------------------------------------------------------------------------
703 * vmci_doorbell_notify --
705 * Generates a notification on the doorbell identified by the handle. For
706 * host side generation of notifications, the caller can specify what the
707 * privilege of the calling side is.
710 * VMCI_SUCCESS on success, appropriate error code otherwise.
713 * May do a hypercall.
715 *------------------------------------------------------------------------------
719 vmci_doorbell_notify(struct vmci_handle dst, vmci_privilege_flags priv_flags)
721 struct vmci_handle src;
723 if (VMCI_HANDLE_INVALID(dst) ||
724 (priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS))
725 return (VMCI_ERROR_INVALID_ARGS);
727 src = VMCI_INVALID_HANDLE;
729 return (vmci_doorbell_notify_as_guest(dst, priv_flags));
733 *------------------------------------------------------------------------------
735 * vmci_doorbell_delayed_dispatch_cb --
737 * Calls the specified callback in a delayed context.
745 *------------------------------------------------------------------------------
749 vmci_doorbell_delayed_dispatch_cb(void *data)
751 struct vmci_doorbell_entry *entry = (struct vmci_doorbell_entry *)data;
755 entry->notify_cb(entry->client_data);
757 vmci_resource_release(&entry->resource);
761 *------------------------------------------------------------------------------
763 * vmci_doorbell_sync --
765 * Use this as a synchronization point when setting globals, for example,
766 * during device shutdown.
774 *------------------------------------------------------------------------------
778 vmci_doorbell_sync(void)
781 vmci_grab_lock_bh(&vmci_doorbell_it.lock);
782 vmci_release_lock_bh(&vmci_doorbell_it.lock);
783 vmci_resource_sync();
787 *------------------------------------------------------------------------------
789 * vmci_register_notification_bitmap --
791 * Register the notification bitmap with the host.
794 * true if the bitmap is registered successfully with the device, false
800 *------------------------------------------------------------------------------
804 vmci_register_notification_bitmap(PPN bitmap_ppn)
806 struct vmci_notify_bitmap_set_msg bitmap_set_msg;
810 * Do not ASSERT() on the guest device here. This function can get
811 * called during device initialization, so the ASSERT() will fail even
812 * though the device is (almost) up.
815 bitmap_set_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
816 VMCI_SET_NOTIFY_BITMAP);
817 bitmap_set_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
818 bitmap_set_msg.hdr.payload_size =
819 sizeof(bitmap_set_msg) - VMCI_DG_HEADERSIZE;
820 bitmap_set_msg.bitmap_ppn = bitmap_ppn;
822 result = vmci_send_datagram((struct vmci_datagram *)&bitmap_set_msg);
823 if (result != VMCI_SUCCESS) {
824 VMCI_LOG_DEBUG(LGPFX"Failed to register (PPN=%u) as "
825 "notification bitmap (error=%d).\n",
833 *------------------------------------------------------------------------------
835 * vmci_doorbell_fire_entries --
837 * Executes or schedules the handlers for a given notify index.
840 * Notification hash entry if found. NULL otherwise.
843 * Whatever the side effects of the handlers are.
845 *------------------------------------------------------------------------------
849 vmci_doorbell_fire_entries(uint32_t notify_idx)
851 struct vmci_doorbell_entry *iter;
852 uint32_t bucket = VMCI_DOORBELL_HASH(notify_idx);
854 vmci_grab_lock_bh(&vmci_doorbell_it.lock);
856 vmci_list_scan(iter, &vmci_doorbell_it.entries[bucket], idx_list_item) {
857 if (iter->idx == notify_idx &&
858 atomic_load_int(&iter->active) == 1) {
859 ASSERT(iter->notify_cb);
860 if (iter->run_delayed) {
863 vmci_resource_hold(&iter->resource);
864 err = vmci_schedule_delayed_work(
865 vmci_doorbell_delayed_dispatch_cb, iter);
866 if (err != VMCI_SUCCESS) {
867 vmci_resource_release(&iter->resource);
871 iter->notify_cb(iter->client_data);
876 vmci_release_lock_bh(&vmci_doorbell_it.lock);
880 *------------------------------------------------------------------------------
882 * vmci_scan_notification_bitmap --
884 * Scans the notification bitmap, collects pending notifications, resets
885 * the bitmap and invokes appropriate callbacks.
891 * May schedule tasks, allocate memory and run callbacks.
893 *------------------------------------------------------------------------------
897 vmci_scan_notification_bitmap(uint8_t *bitmap)
903 for (idx = 0; idx < max_notify_idx; idx++) {
904 if (bitmap[idx] & 0x1) {
906 vmci_doorbell_fire_entries(idx);