2 * Copyright (c) 2018 VMware, Inc.
4 * SPDX-License-Identifier: (BSD-2-Clause OR GPL-2.0)
7 /* This file implements Queue accessor methods. */
10 * vmci_qpair is an interface that hides the queue pair internals. Rather than
11 * access each queue in a pair directly, operations are performed on the queue
12 * as a whole. This is simpler and less error-prone, and allows for future
13 * queue pair features to be added under the hood with no change to the client
17 #include <sys/cdefs.h>
18 __FBSDID("$FreeBSD$");
20 #include "vmci_kernel_api.h"
21 #include "vmci_kernel_defs.h"
22 #include "vmci_kernel_if.h"
23 #include "vmci_queue.h"
24 #include "vmci_queue_pair.h"
26 /* This structure is opaque to the clients. */
28 struct vmci_handle handle;
29 struct vmci_queue *produce_q;
30 struct vmci_queue *consume_q;
31 uint64_t produce_q_size;
32 uint64_t consume_q_size;
35 vmci_privilege_flags priv_flags;
40 static void vmci_qpair_get_queue_headers(const struct vmci_qpair *qpair,
41 struct vmci_queue_header **produce_q_header,
42 struct vmci_queue_header **consume_q_header);
45 *------------------------------------------------------------------------------
47 * vmci_queue_add_producer_tail --
49 * Helper routine to increment the Producer Tail.
52 * VMCI_ERROR_NOT_FOUND if the vmm_world registered with the queue cannot
53 * be found. Otherwise VMCI_SUCCESS.
58 *------------------------------------------------------------------------------
62 vmci_queue_add_producer_tail(struct vmci_queue *queue,
63 size_t add, uint64_t queue_size)
66 vmci_queue_header_add_producer_tail(queue->q_header, add, queue_size);
67 return (VMCI_SUCCESS);
71 *------------------------------------------------------------------------------
73 * vmci_queue_add_consumer_head --
75 * Helper routine to increment the Consumer Head.
78 * VMCI_ERROR_NOT_FOUND if the vmm_world registered with the queue cannot
79 * be found. Otherwise VMCI_SUCCESS.
84 *------------------------------------------------------------------------------
88 vmci_queue_add_consumer_head(struct vmci_queue *queue,
89 size_t add, uint64_t queue_size)
92 vmci_queue_header_add_consumer_head(queue->q_header, add, queue_size);
93 return (VMCI_SUCCESS);
97 *------------------------------------------------------------------------------
99 * vmci_qpair_get_queue_headers --
101 * Helper routine that will retrieve the produce and consume headers of a
105 * VMCI_SUCCESS if either current or saved queue headers are found.
106 * Appropriate error code otherwise.
111 *------------------------------------------------------------------------------
115 vmci_qpair_get_queue_headers(const struct vmci_qpair *qpair,
116 struct vmci_queue_header **produce_q_header,
117 struct vmci_queue_header **consume_q_header)
120 ASSERT((qpair->produce_q != NULL) && (qpair->consume_q != NULL));
121 *produce_q_header = qpair->produce_q->q_header;
122 *consume_q_header = qpair->consume_q->q_header;
126 *------------------------------------------------------------------------------
128 * vmci_qpair_alloc --
130 * This is the client interface for allocating the memory for a vmci_qpair
131 * structure and then attaching to the underlying queue. If an error occurs
132 * allocating the memory for the vmci_qpair structure, no attempt is made to
133 * attach. If an error occurs attaching, then there's the vmci_qpair
134 * structure is freed.
142 *------------------------------------------------------------------------------
146 vmci_qpair_alloc(struct vmci_qpair **qpair, struct vmci_handle *handle,
147 uint64_t produce_q_size, uint64_t consume_q_size, vmci_id peer,
148 uint32_t flags, vmci_privilege_flags priv_flags)
150 struct vmci_qpair *my_qpair;
151 vmci_event_release_cb wakeup_cb;
156 * Restrict the size of a queuepair. Though the device enforces a limit
157 * on the total amount of memory that can be allocated to queuepairs for
158 * a guest, we avoid unnecessarily allocating a lot of memory. Also, we
159 * try to allocate this memory before we make the queuepair allocation
162 * (Note that this doesn't prevent all cases; a user with only this much
163 * physical memory could still get into trouble.) The error used by the
164 * device is NO_RESOURCES, so use that here too.
167 if (produce_q_size + consume_q_size <
168 MAX(produce_q_size, consume_q_size) ||
169 produce_q_size + consume_q_size > VMCI_MAX_GUEST_QP_MEMORY)
170 return (VMCI_ERROR_NO_RESOURCES);
172 if (flags & VMCI_QPFLAG_NONBLOCK)
173 return (VMCI_ERROR_INVALID_ARGS);
175 my_qpair = vmci_alloc_kernel_mem(sizeof(*my_qpair), VMCI_MEMORY_NORMAL);
177 return (VMCI_ERROR_NO_MEM);
179 my_qpair->produce_q_size = produce_q_size;
180 my_qpair->consume_q_size = consume_q_size;
181 my_qpair->peer = peer;
182 my_qpair->flags = flags;
183 my_qpair->priv_flags = priv_flags;
188 retval = vmci_queue_pair_alloc(handle, &my_qpair->produce_q,
189 my_qpair->produce_q_size, &my_qpair->consume_q,
190 my_qpair->consume_q_size, my_qpair->peer, my_qpair->flags,
191 my_qpair->priv_flags);
193 if (retval < VMCI_SUCCESS) {
194 vmci_free_kernel_mem(my_qpair, sizeof(*my_qpair));
199 my_qpair->handle = *handle;
205 *------------------------------------------------------------------------------
207 * vmci_qpair_detach --
209 * This is the client interface for detaching from a vmci_qpair. Note that
210 * this routine will free the memory allocated for the vmci_qpair structure,
217 * Will clear the caller's pointer to the vmci_qpair structure.
219 *------------------------------------------------------------------------------
223 vmci_qpair_detach(struct vmci_qpair **qpair)
225 struct vmci_qpair *old_qpair;
228 if (!qpair || !(*qpair))
229 return (VMCI_ERROR_INVALID_ARGS);
232 result = vmci_queue_pair_detach(old_qpair->handle);
235 * The guest can fail to detach for a number of reasons, and if it does
236 * so, it will cleanup the entry (if there is one). We need to release
237 * the qpair struct here; there isn't much the caller can do, and we
238 * don't want to leak.
241 if (old_qpair->flags & VMCI_QPFLAG_LOCAL)
242 vmci_destroy_event(&old_qpair->event);
244 vmci_free_kernel_mem(old_qpair, sizeof(*old_qpair));
251 *------------------------------------------------------------------------------
253 * vmci_qpair_get_produce_indexes --
255 * This is the client interface for getting the current indexes of the
256 * qpair from the point of the view of the caller as the producer.
265 *------------------------------------------------------------------------------
269 vmci_qpair_get_produce_indexes(const struct vmci_qpair *qpair,
270 uint64_t *producer_tail, uint64_t *consumer_head)
272 struct vmci_queue_header *consume_q_header;
273 struct vmci_queue_header *produce_q_header;
276 return (VMCI_ERROR_INVALID_ARGS);
278 vmci_qpair_get_queue_headers(qpair, &produce_q_header,
280 vmci_queue_header_get_pointers(produce_q_header, consume_q_header,
281 producer_tail, consumer_head);
283 if ((producer_tail && *producer_tail >= qpair->produce_q_size) ||
284 (consumer_head && *consumer_head >= qpair->produce_q_size))
285 return (VMCI_ERROR_INVALID_SIZE);
287 return (VMCI_SUCCESS);
291 *------------------------------------------------------------------------------
293 * vmci_qpair_get_consume_indexes --
295 * This is the client interface for getting the current indexes of the
296 * QPair from the point of the view of the caller as the consumer.
305 *------------------------------------------------------------------------------
309 vmci_qpair_get_consume_indexes(const struct vmci_qpair *qpair,
310 uint64_t *consumer_tail, uint64_t *producer_head)
312 struct vmci_queue_header *consume_q_header;
313 struct vmci_queue_header *produce_q_header;
316 return (VMCI_ERROR_INVALID_ARGS);
318 vmci_qpair_get_queue_headers(qpair, &produce_q_header,
320 vmci_queue_header_get_pointers(consume_q_header, produce_q_header,
321 consumer_tail, producer_head);
323 if ((consumer_tail && *consumer_tail >= qpair->consume_q_size) ||
324 (producer_head && *producer_head >= qpair->consume_q_size))
325 return (VMCI_ERROR_INVALID_SIZE);
327 return (VMCI_SUCCESS);
331 *------------------------------------------------------------------------------
333 * vmci_qpair_produce_free_space --
335 * This is the client interface for getting the amount of free space in the
336 * QPair from the point of the view of the caller as the producer which is
342 * Number of available bytes into which data can be enqueued if > 0.
347 *------------------------------------------------------------------------------
351 vmci_qpair_produce_free_space(const struct vmci_qpair *qpair)
353 struct vmci_queue_header *consume_q_header;
354 struct vmci_queue_header *produce_q_header;
358 return (VMCI_ERROR_INVALID_ARGS);
360 vmci_qpair_get_queue_headers(qpair, &produce_q_header,
362 result = vmci_queue_header_free_space(produce_q_header, consume_q_header,
363 qpair->produce_q_size);
369 *------------------------------------------------------------------------------
371 * vmci_qpair_consume_free_space --
373 * This is the client interface for getting the amount of free space in the
374 * QPair from the point of the view of the caller as the consumer which is
375 * not the common case (see vmci_qpair_Produce_free_space(), above).
380 * Number of available bytes into which data can be enqueued if > 0.
385 *------------------------------------------------------------------------------
389 vmci_qpair_consume_free_space(const struct vmci_qpair *qpair)
391 struct vmci_queue_header *consume_q_header;
392 struct vmci_queue_header *produce_q_header;
396 return (VMCI_ERROR_INVALID_ARGS);
398 vmci_qpair_get_queue_headers(qpair, &produce_q_header,
400 result = vmci_queue_header_free_space(consume_q_header, produce_q_header,
401 qpair->consume_q_size);
407 *------------------------------------------------------------------------------
409 * vmci_qpair_produce_buf_ready --
411 * This is the client interface for getting the amount of enqueued data in
412 * the QPair from the point of the view of the caller as the producer which
413 * is not the common case (see vmci_qpair_Consume_buf_ready(), above).
417 * Empty queue if = 0.
418 * Number of bytes ready to be dequeued if > 0.
423 *------------------------------------------------------------------------------
427 vmci_qpair_produce_buf_ready(const struct vmci_qpair *qpair)
429 struct vmci_queue_header *consume_q_header;
430 struct vmci_queue_header *produce_q_header;
434 return (VMCI_ERROR_INVALID_ARGS);
436 vmci_qpair_get_queue_headers(qpair, &produce_q_header,
438 result = vmci_queue_header_buf_ready(produce_q_header, consume_q_header,
439 qpair->produce_q_size);
445 *------------------------------------------------------------------------------
447 * vmci_qpair_consume_buf_ready --
449 * This is the client interface for getting the amount of enqueued data in
450 * the QPair from the point of the view of the caller as the consumer which
451 * is the normal case.
455 * Empty queue if = 0.
456 * Number of bytes ready to be dequeued if > 0.
461 *------------------------------------------------------------------------------
465 vmci_qpair_consume_buf_ready(const struct vmci_qpair *qpair)
467 struct vmci_queue_header *consume_q_header;
468 struct vmci_queue_header *produce_q_header;
472 return (VMCI_ERROR_INVALID_ARGS);
474 vmci_qpair_get_queue_headers(qpair, &produce_q_header,
476 result = vmci_queue_header_buf_ready(consume_q_header, produce_q_header,
477 qpair->consume_q_size);
483 *------------------------------------------------------------------------------
487 * Enqueues a given buffer to the produce queue using the provided function.
488 * As many bytes as possible (space available in the queue) are enqueued.
491 * VMCI_ERROR_QUEUEPAIR_NOSPACE if no space was available to enqueue data.
492 * VMCI_ERROR_INVALID_SIZE, if any queue pointer is outside the queue
493 * (as defined by the queue size).
494 * VMCI_ERROR_INVALID_ARGS, if an error occured when accessing the buffer.
495 * VMCI_ERROR_QUEUEPAIR_NOTATTACHED, if the queue pair pages aren't
497 * Otherwise, the number of bytes written to the queue is returned.
500 * Updates the tail pointer of the produce queue.
502 *------------------------------------------------------------------------------
506 enqueue(struct vmci_queue *produce_q, struct vmci_queue *consume_q,
507 const uint64_t produce_q_size, const void *buf, size_t buf_size,
508 int buf_type, vmci_memcpy_to_queue_func memcpy_to_queue, bool can_block)
515 ASSERT((produce_q != NULL) && (consume_q != NULL));
517 free_space = vmci_queue_header_free_space(produce_q->q_header,
521 return (VMCI_ERROR_QUEUEPAIR_NOSPACE);
523 if (free_space < VMCI_SUCCESS)
524 return ((ssize_t)free_space);
526 written = (size_t)(free_space > buf_size ? buf_size : free_space);
527 tail = vmci_queue_header_producer_tail(produce_q->q_header);
528 if (LIKELY(tail + written < produce_q_size))
529 result = memcpy_to_queue(produce_q, tail, buf, 0, written,
530 buf_type, can_block);
532 /* Tail pointer wraps around. */
534 const size_t tmp = (size_t)(produce_q_size - tail);
536 result = memcpy_to_queue(produce_q, tail, buf, 0, tmp, buf_type,
538 if (result >= VMCI_SUCCESS)
539 result = memcpy_to_queue(produce_q, 0, buf, tmp,
540 written - tmp, buf_type, can_block);
543 if (result < VMCI_SUCCESS)
546 result = vmci_queue_add_producer_tail(produce_q, written,
548 if (result < VMCI_SUCCESS)
554 *------------------------------------------------------------------------------
558 * Dequeues data (if available) from the given consume queue. Writes data
559 * to the user provided buffer using the provided function.
562 * VMCI_ERROR_QUEUEPAIR_NODATA if no data was available to dequeue.
563 * VMCI_ERROR_INVALID_SIZE, if any queue pointer is outside the queue
564 * (as defined by the queue size).
565 * VMCI_ERROR_INVALID_ARGS, if an error occured when accessing the buffer.
566 * VMCI_ERROR_NOT_FOUND, if the vmm_world registered with the queue pair
568 * Otherwise the number of bytes dequeued is returned.
571 * Updates the head pointer of the consume queue.
573 *------------------------------------------------------------------------------
577 dequeue(struct vmci_queue *produce_q,
578 struct vmci_queue *consume_q, const uint64_t consume_q_size, void *buf,
579 size_t buf_size, int buf_type,
580 vmci_memcpy_from_queue_func memcpy_from_queue, bool update_consumer,
588 ASSERT((produce_q != NULL) && (consume_q != NULL));
590 buf_ready = vmci_queue_header_buf_ready(consume_q->q_header,
591 produce_q->q_header, consume_q_size);
593 return (VMCI_ERROR_QUEUEPAIR_NODATA);
594 if (buf_ready < VMCI_SUCCESS)
595 return ((ssize_t)buf_ready);
597 read = (size_t)(buf_ready > buf_size ? buf_size : buf_ready);
598 head = vmci_queue_header_consumer_head(produce_q->q_header);
599 if (LIKELY(head + read < consume_q_size))
600 result = memcpy_from_queue(buf, 0, consume_q, head, read,
601 buf_type, can_block);
603 /* Head pointer wraps around. */
605 const size_t tmp = (size_t)(consume_q_size - head);
607 result = memcpy_from_queue(buf, 0, consume_q, head, tmp,
608 buf_type, can_block);
609 if (result >= VMCI_SUCCESS)
610 result = memcpy_from_queue(buf, tmp, consume_q, 0,
611 read - tmp, buf_type, can_block);
614 if (result < VMCI_SUCCESS)
617 if (update_consumer) {
618 result = vmci_queue_add_consumer_head(produce_q, read,
620 if (result < VMCI_SUCCESS)
628 *------------------------------------------------------------------------------
630 * vmci_qpair_enqueue --
632 * This is the client interface for enqueueing data into the queue.
636 * Number of bytes enqueued if >= 0.
641 *------------------------------------------------------------------------------
645 vmci_qpair_enqueue(struct vmci_qpair *qpair, const void *buf, size_t buf_size,
651 return (VMCI_ERROR_INVALID_ARGS);
653 result = enqueue(qpair->produce_q, qpair->consume_q,
654 qpair->produce_q_size, buf, buf_size, buf_type,
655 qpair->flags & VMCI_QPFLAG_LOCAL?
656 vmci_memcpy_to_queue_local : vmci_memcpy_to_queue,
657 !(qpair->flags & VMCI_QPFLAG_NONBLOCK));
663 *------------------------------------------------------------------------------
665 * vmci_qpair_dequeue --
667 * This is the client interface for dequeueing data from the queue.
671 * Number of bytes dequeued if >= 0.
676 *------------------------------------------------------------------------------
680 vmci_qpair_dequeue(struct vmci_qpair *qpair, void *buf, size_t buf_size,
686 return (VMCI_ERROR_INVALID_ARGS);
688 result = dequeue(qpair->produce_q, qpair->consume_q,
689 qpair->consume_q_size, buf, buf_size, buf_type,
690 qpair->flags & VMCI_QPFLAG_LOCAL?
691 vmci_memcpy_from_queue_local : vmci_memcpy_from_queue, true,
692 !(qpair->flags & VMCI_QPFLAG_NONBLOCK));
698 *------------------------------------------------------------------------------
702 * This is the client interface for peeking into a queue. (I.e., copy
703 * data from the queue without updating the head pointer.)
707 * Number of bytes peeked, if >= 0.
712 *------------------------------------------------------------------------------
716 vmci_qpair_peek(struct vmci_qpair *qpair, void *buf, size_t buf_size,
722 return (VMCI_ERROR_INVALID_ARGS);
724 result = dequeue(qpair->produce_q, qpair->consume_q,
725 qpair->consume_q_size, buf, buf_size, buf_type,
726 qpair->flags & VMCI_QPFLAG_LOCAL?
727 vmci_memcpy_from_queue_local : vmci_memcpy_from_queue, false,
728 !(qpair->flags & VMCI_QPFLAG_NONBLOCK));
734 *------------------------------------------------------------------------------
736 * vmci_qpair_enquev --
738 * This is the client interface for enqueueing data into the queue.
742 * Number of bytes enqueued if >= 0.
747 *------------------------------------------------------------------------------
751 vmci_qpair_enquev(struct vmci_qpair *qpair, void *iov, size_t iov_size,
757 return (VMCI_ERROR_INVALID_ARGS);
759 result = enqueue(qpair->produce_q, qpair->consume_q,
760 qpair->produce_q_size, iov, iov_size, buf_type,
761 qpair->flags & VMCI_QPFLAG_LOCAL?
762 vmci_memcpy_to_queue_v_local : vmci_memcpy_to_queue_v,
763 !(qpair->flags & VMCI_QPFLAG_NONBLOCK));
769 *------------------------------------------------------------------------------
771 * vmci_qpair_dequev --
773 * This is the client interface for dequeueing data from the queue.
777 * Number of bytes dequeued if >= 0.
782 *------------------------------------------------------------------------------
786 vmci_qpair_dequev(struct vmci_qpair *qpair, void *iov, size_t iov_size,
792 return (VMCI_ERROR_INVALID_ARGS);
794 result = dequeue(qpair->produce_q, qpair->consume_q,
795 qpair->consume_q_size, iov, iov_size, buf_type,
796 qpair->flags & VMCI_QPFLAG_LOCAL?
797 vmci_memcpy_from_queue_v_local : vmci_memcpy_from_queue_v, true,
798 !(qpair->flags & VMCI_QPFLAG_NONBLOCK));
804 *------------------------------------------------------------------------------
806 * vmci_qpair_peekv --
808 * This is the client interface for peeking into a queue. (I.e., copy
809 * data from the queue without updating the head pointer.)
813 * Number of bytes peeked, if >= 0.
818 *------------------------------------------------------------------------------
822 vmci_qpair_peekv(struct vmci_qpair *qpair, void *iov, size_t iov_size,
828 return (VMCI_ERROR_INVALID_ARGS);
830 result = dequeue(qpair->produce_q, qpair->consume_q,
831 qpair->consume_q_size, iov, iov_size, buf_type,
832 qpair->flags & VMCI_QPFLAG_LOCAL?
833 vmci_memcpy_from_queue_v_local : vmci_memcpy_from_queue_v, false,
834 !(qpair->flags & VMCI_QPFLAG_NONBLOCK));