2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice unmodified, this list of conditions, and the following
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 * Implements the virtqueue interface as basically described
31 * in the original VirtIO paper.
34 #include <sys/cdefs.h>
35 __FBSDID("$FreeBSD$");
37 #include <sys/param.h>
38 #include <sys/systm.h>
39 #include <sys/kernel.h>
40 #include <sys/malloc.h>
41 #include <sys/sglist.h>
45 #include <machine/cpu.h>
46 #include <machine/bus.h>
47 #include <machine/atomic.h>
48 #include <machine/resource.h>
52 #include <dev/virtio/virtio.h>
53 #include <dev/virtio/virtqueue.h>
54 #include <dev/virtio/virtio_ring.h>
56 #include "virtio_bus_if.h"
60 char vq_name[VIRTQUEUE_MAX_NAME_SZ];
61 uint16_t vq_queue_index;
64 #define VIRTQUEUE_FLAG_INDIRECT 0x0001
65 #define VIRTQUEUE_FLAG_EVENT_IDX 0x0002
70 int vq_max_indirect_size;
71 int vq_indirect_mem_size;
72 virtqueue_intr_t *vq_intrhand;
73 void *vq_intrhand_arg;
77 uint16_t vq_queued_cnt;
79 * Head of the free chain in the descriptor table. If
80 * there are no free descriptors, this will be set to
81 * VQ_RING_DESC_CHAIN_END.
83 uint16_t vq_desc_head_idx;
85 * Last consumed descriptor in the used table,
86 * trails vq_ring.used->idx.
88 uint16_t vq_used_cons_idx;
90 struct vq_desc_extra {
92 struct vring_desc *indirect;
93 vm_paddr_t indirect_paddr;
99 * The maximum virtqueue size is 2^15. Use that value as the end of
100 * descriptor chain terminator since it will never be a valid index
101 * in the descriptor table. This is used to verify we are correctly
102 * handling vq_free_cnt.
104 #define VQ_RING_DESC_CHAIN_END 32768
106 #define VQASSERT(_vq, _exp, _msg, ...) \
107 KASSERT((_exp),("%s: %s - "_msg, __func__, (_vq)->vq_name, \
110 #define VQ_RING_ASSERT_VALID_IDX(_vq, _idx) \
111 VQASSERT((_vq), (_idx) < (_vq)->vq_nentries, \
112 "invalid ring index: %d, max: %d", (_idx), \
115 #define VQ_RING_ASSERT_CHAIN_TERM(_vq) \
116 VQASSERT((_vq), (_vq)->vq_desc_head_idx == \
117 VQ_RING_DESC_CHAIN_END, "full ring terminated " \
118 "incorrectly: head idx: %d", (_vq)->vq_desc_head_idx)
120 static int virtqueue_init_indirect(struct virtqueue *vq, int);
121 static void virtqueue_free_indirect(struct virtqueue *vq);
122 static void virtqueue_init_indirect_list(struct virtqueue *,
123 struct vring_desc *);
125 static void vq_ring_init(struct virtqueue *);
126 static void vq_ring_update_avail(struct virtqueue *, uint16_t);
127 static uint16_t vq_ring_enqueue_segments(struct virtqueue *,
128 struct vring_desc *, uint16_t, struct sglist *, int, int);
129 static int vq_ring_use_indirect(struct virtqueue *, int);
130 static void vq_ring_enqueue_indirect(struct virtqueue *, void *,
131 struct sglist *, int, int);
132 static int vq_ring_enable_interrupt(struct virtqueue *, uint16_t);
133 static int vq_ring_must_notify_host(struct virtqueue *);
134 static void vq_ring_notify_host(struct virtqueue *);
135 static void vq_ring_free_chain(struct virtqueue *, uint16_t);
138 virtqueue_filter_features(uint64_t features)
142 mask = (1 << VIRTIO_TRANSPORT_F_START) - 1;
143 mask |= VIRTIO_RING_F_INDIRECT_DESC;
144 mask |= VIRTIO_RING_F_EVENT_IDX;
146 return (features & mask);
150 virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size, int align,
151 vm_paddr_t highaddr, struct vq_alloc_info *info, struct virtqueue **vqp)
153 struct virtqueue *vq;
161 "virtqueue %d (%s) does not exist (size is zero)\n",
162 queue, info->vqai_name);
164 } else if (!powerof2(size)) {
166 "virtqueue %d (%s) size is not a power of 2: %d\n",
167 queue, info->vqai_name, size);
169 } else if (info->vqai_maxindirsz > VIRTIO_MAX_INDIRECT) {
170 device_printf(dev, "virtqueue %d (%s) requested too many "
171 "indirect descriptors: %d, max %d\n",
172 queue, info->vqai_name, info->vqai_maxindirsz,
173 VIRTIO_MAX_INDIRECT);
177 vq = malloc(sizeof(struct virtqueue) +
178 size * sizeof(struct vq_desc_extra), M_DEVBUF, M_NOWAIT | M_ZERO);
180 device_printf(dev, "cannot allocate virtqueue\n");
185 strlcpy(vq->vq_name, info->vqai_name, sizeof(vq->vq_name));
186 vq->vq_queue_index = queue;
187 vq->vq_alignment = align;
188 vq->vq_nentries = size;
189 vq->vq_free_cnt = size;
190 vq->vq_intrhand = info->vqai_intr;
191 vq->vq_intrhand_arg = info->vqai_intr_arg;
193 if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_EVENT_IDX) != 0)
194 vq->vq_flags |= VIRTQUEUE_FLAG_EVENT_IDX;
196 if (info->vqai_maxindirsz > 1) {
197 error = virtqueue_init_indirect(vq, info->vqai_maxindirsz);
202 vq->vq_ring_size = round_page(vring_size(size, align));
203 vq->vq_ring_mem = contigmalloc(vq->vq_ring_size, M_DEVBUF,
204 M_NOWAIT | M_ZERO, 0, highaddr, PAGE_SIZE, 0);
205 if (vq->vq_ring_mem == NULL) {
207 "cannot allocate memory for virtqueue ring\n");
213 virtqueue_disable_intr(vq);
225 virtqueue_init_indirect(struct virtqueue *vq, int indirect_size)
228 struct vq_desc_extra *dxp;
233 if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_INDIRECT_DESC) == 0) {
235 * Indirect descriptors requested by the driver but not
236 * negotiated. Return zero to keep the initialization
237 * going: we'll run fine without.
240 device_printf(dev, "virtqueue %d (%s) requested "
241 "indirect descriptors but not negotiated\n",
242 vq->vq_queue_index, vq->vq_name);
246 size = indirect_size * sizeof(struct vring_desc);
247 vq->vq_max_indirect_size = indirect_size;
248 vq->vq_indirect_mem_size = size;
249 vq->vq_flags |= VIRTQUEUE_FLAG_INDIRECT;
251 for (i = 0; i < vq->vq_nentries; i++) {
252 dxp = &vq->vq_descx[i];
254 dxp->indirect = malloc(size, M_DEVBUF, M_NOWAIT);
255 if (dxp->indirect == NULL) {
256 device_printf(dev, "cannot allocate indirect list\n");
260 dxp->indirect_paddr = vtophys(dxp->indirect);
261 virtqueue_init_indirect_list(vq, dxp->indirect);
268 virtqueue_free_indirect(struct virtqueue *vq)
270 struct vq_desc_extra *dxp;
273 for (i = 0; i < vq->vq_nentries; i++) {
274 dxp = &vq->vq_descx[i];
276 if (dxp->indirect == NULL)
279 free(dxp->indirect, M_DEVBUF);
280 dxp->indirect = NULL;
281 dxp->indirect_paddr = 0;
284 vq->vq_flags &= ~VIRTQUEUE_FLAG_INDIRECT;
285 vq->vq_indirect_mem_size = 0;
289 virtqueue_init_indirect_list(struct virtqueue *vq,
290 struct vring_desc *indirect)
294 bzero(indirect, vq->vq_indirect_mem_size);
296 for (i = 0; i < vq->vq_max_indirect_size - 1; i++)
297 indirect[i].next = i + 1;
298 indirect[i].next = VQ_RING_DESC_CHAIN_END;
302 virtqueue_reinit(struct virtqueue *vq, uint16_t size)
304 struct vq_desc_extra *dxp;
307 if (vq->vq_nentries != size) {
308 device_printf(vq->vq_dev,
309 "%s: '%s' changed size; old=%hu, new=%hu\n",
310 __func__, vq->vq_name, vq->vq_nentries, size);
314 /* Warn if the virtqueue was not properly cleaned up. */
315 if (vq->vq_free_cnt != vq->vq_nentries) {
316 device_printf(vq->vq_dev,
317 "%s: warning '%s' virtqueue not empty, "
318 "leaking %d entries\n", __func__, vq->vq_name,
319 vq->vq_nentries - vq->vq_free_cnt);
322 vq->vq_desc_head_idx = 0;
323 vq->vq_used_cons_idx = 0;
324 vq->vq_queued_cnt = 0;
325 vq->vq_free_cnt = vq->vq_nentries;
327 /* To be safe, reset all our allocated memory. */
328 bzero(vq->vq_ring_mem, vq->vq_ring_size);
329 for (i = 0; i < vq->vq_nentries; i++) {
330 dxp = &vq->vq_descx[i];
333 if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT)
334 virtqueue_init_indirect_list(vq, dxp->indirect);
338 virtqueue_disable_intr(vq);
344 virtqueue_free(struct virtqueue *vq)
347 if (vq->vq_free_cnt != vq->vq_nentries) {
348 device_printf(vq->vq_dev, "%s: freeing non-empty virtqueue, "
349 "leaking %d entries\n", vq->vq_name,
350 vq->vq_nentries - vq->vq_free_cnt);
353 if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT)
354 virtqueue_free_indirect(vq);
356 if (vq->vq_ring_mem != NULL) {
357 contigfree(vq->vq_ring_mem, vq->vq_ring_size, M_DEVBUF);
358 vq->vq_ring_size = 0;
359 vq->vq_ring_mem = NULL;
366 virtqueue_paddr(struct virtqueue *vq)
369 return (vtophys(vq->vq_ring_mem));
373 virtqueue_desc_paddr(struct virtqueue *vq)
376 return (vtophys(vq->vq_ring.desc));
380 virtqueue_avail_paddr(struct virtqueue *vq)
383 return (vtophys(vq->vq_ring.avail));
387 virtqueue_used_paddr(struct virtqueue *vq)
390 return (vtophys(vq->vq_ring.used));
394 virtqueue_index(struct virtqueue *vq)
396 return (vq->vq_queue_index);
400 virtqueue_size(struct virtqueue *vq)
403 return (vq->vq_nentries);
407 virtqueue_nfree(struct virtqueue *vq)
410 return (vq->vq_free_cnt);
414 virtqueue_empty(struct virtqueue *vq)
417 return (vq->vq_nentries == vq->vq_free_cnt);
421 virtqueue_full(struct virtqueue *vq)
424 return (vq->vq_free_cnt == 0);
428 virtqueue_notify(struct virtqueue *vq)
431 /* Ensure updated avail->idx is visible to host. */
434 if (vq_ring_must_notify_host(vq))
435 vq_ring_notify_host(vq);
436 vq->vq_queued_cnt = 0;
440 virtqueue_nused(struct virtqueue *vq)
442 uint16_t used_idx, nused;
444 used_idx = vq->vq_ring.used->idx;
446 nused = (uint16_t)(used_idx - vq->vq_used_cons_idx);
447 VQASSERT(vq, nused <= vq->vq_nentries, "used more than available");
453 virtqueue_intr_filter(struct virtqueue *vq)
456 if (vq->vq_used_cons_idx == vq->vq_ring.used->idx)
459 virtqueue_disable_intr(vq);
465 virtqueue_intr(struct virtqueue *vq)
468 vq->vq_intrhand(vq->vq_intrhand_arg);
472 virtqueue_enable_intr(struct virtqueue *vq)
475 return (vq_ring_enable_interrupt(vq, 0));
479 virtqueue_postpone_intr(struct virtqueue *vq, vq_postpone_t hint)
481 uint16_t ndesc, avail_idx;
483 avail_idx = vq->vq_ring.avail->idx;
484 ndesc = (uint16_t)(avail_idx - vq->vq_used_cons_idx);
487 case VQ_POSTPONE_SHORT:
490 case VQ_POSTPONE_LONG:
491 ndesc = (ndesc * 3) / 4;
493 case VQ_POSTPONE_EMPTIED:
497 return (vq_ring_enable_interrupt(vq, ndesc));
501 * Note this is only considered a hint to the host.
504 virtqueue_disable_intr(struct virtqueue *vq)
507 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
508 vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx -
511 vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
515 virtqueue_enqueue(struct virtqueue *vq, void *cookie, struct sglist *sg,
516 int readable, int writable)
518 struct vq_desc_extra *dxp;
520 uint16_t head_idx, idx;
522 needed = readable + writable;
524 VQASSERT(vq, cookie != NULL, "enqueuing with no cookie");
525 VQASSERT(vq, needed == sg->sg_nseg,
526 "segment count mismatch, %d, %d", needed, sg->sg_nseg);
528 needed <= vq->vq_nentries || needed <= vq->vq_max_indirect_size,
529 "too many segments to enqueue: %d, %d/%d", needed,
530 vq->vq_nentries, vq->vq_max_indirect_size);
534 if (vq->vq_free_cnt == 0)
537 if (vq_ring_use_indirect(vq, needed)) {
538 vq_ring_enqueue_indirect(vq, cookie, sg, readable, writable);
540 } else if (vq->vq_free_cnt < needed)
543 head_idx = vq->vq_desc_head_idx;
544 VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
545 dxp = &vq->vq_descx[head_idx];
547 VQASSERT(vq, dxp->cookie == NULL,
548 "cookie already exists for index %d", head_idx);
549 dxp->cookie = cookie;
550 dxp->ndescs = needed;
552 idx = vq_ring_enqueue_segments(vq, vq->vq_ring.desc, head_idx,
553 sg, readable, writable);
555 vq->vq_desc_head_idx = idx;
556 vq->vq_free_cnt -= needed;
557 if (vq->vq_free_cnt == 0)
558 VQ_RING_ASSERT_CHAIN_TERM(vq);
560 VQ_RING_ASSERT_VALID_IDX(vq, idx);
562 vq_ring_update_avail(vq, head_idx);
568 virtqueue_dequeue(struct virtqueue *vq, uint32_t *len)
570 struct vring_used_elem *uep;
572 uint16_t used_idx, desc_idx;
574 if (vq->vq_used_cons_idx == vq->vq_ring.used->idx)
577 used_idx = vq->vq_used_cons_idx++ & (vq->vq_nentries - 1);
578 uep = &vq->vq_ring.used->ring[used_idx];
581 desc_idx = (uint16_t) uep->id;
585 vq_ring_free_chain(vq, desc_idx);
587 cookie = vq->vq_descx[desc_idx].cookie;
588 VQASSERT(vq, cookie != NULL, "no cookie for index %d", desc_idx);
589 vq->vq_descx[desc_idx].cookie = NULL;
595 virtqueue_poll(struct virtqueue *vq, uint32_t *len)
599 VIRTIO_BUS_POLL(vq->vq_dev);
600 while ((cookie = virtqueue_dequeue(vq, len)) == NULL) {
602 VIRTIO_BUS_POLL(vq->vq_dev);
609 virtqueue_drain(struct virtqueue *vq, int *last)
617 while (idx < vq->vq_nentries && cookie == NULL) {
618 if ((cookie = vq->vq_descx[idx].cookie) != NULL) {
619 vq->vq_descx[idx].cookie = NULL;
620 /* Free chain to keep free count consistent. */
621 vq_ring_free_chain(vq, idx);
632 virtqueue_dump(struct virtqueue *vq)
638 printf("VQ: %s - size=%d; free=%d; used=%d; queued=%d; "
639 "desc_head_idx=%d; avail.idx=%d; used_cons_idx=%d; "
640 "used.idx=%d; used_event_idx=%d; avail.flags=0x%x; used.flags=0x%x\n",
641 vq->vq_name, vq->vq_nentries, vq->vq_free_cnt,
642 virtqueue_nused(vq), vq->vq_queued_cnt, vq->vq_desc_head_idx,
643 vq->vq_ring.avail->idx, vq->vq_used_cons_idx,
644 vq->vq_ring.used->idx,
645 vring_used_event(&vq->vq_ring),
646 vq->vq_ring.avail->flags,
647 vq->vq_ring.used->flags);
651 vq_ring_init(struct virtqueue *vq)
657 ring_mem = vq->vq_ring_mem;
658 size = vq->vq_nentries;
661 vring_init(vr, size, ring_mem, vq->vq_alignment);
663 for (i = 0; i < size - 1; i++)
664 vr->desc[i].next = i + 1;
665 vr->desc[i].next = VQ_RING_DESC_CHAIN_END;
669 vq_ring_update_avail(struct virtqueue *vq, uint16_t desc_idx)
674 * Place the head of the descriptor chain into the next slot and make
675 * it usable to the host. The chain is made available now rather than
676 * deferring to virtqueue_notify() in the hopes that if the host is
677 * currently running on another CPU, we can keep it processing the new
680 avail_idx = vq->vq_ring.avail->idx & (vq->vq_nentries - 1);
681 vq->vq_ring.avail->ring[avail_idx] = desc_idx;
684 vq->vq_ring.avail->idx++;
686 /* Keep pending count until virtqueue_notify(). */
691 vq_ring_enqueue_segments(struct virtqueue *vq, struct vring_desc *desc,
692 uint16_t head_idx, struct sglist *sg, int readable, int writable)
694 struct sglist_seg *seg;
695 struct vring_desc *dp;
699 needed = readable + writable;
701 for (i = 0, idx = head_idx, seg = sg->sg_segs;
703 i++, idx = dp->next, seg++) {
704 VQASSERT(vq, idx != VQ_RING_DESC_CHAIN_END,
705 "premature end of free desc chain");
708 dp->addr = seg->ss_paddr;
709 dp->len = seg->ss_len;
713 dp->flags |= VRING_DESC_F_NEXT;
715 dp->flags |= VRING_DESC_F_WRITE;
722 vq_ring_use_indirect(struct virtqueue *vq, int needed)
725 if ((vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) == 0)
728 if (vq->vq_max_indirect_size < needed)
738 vq_ring_enqueue_indirect(struct virtqueue *vq, void *cookie,
739 struct sglist *sg, int readable, int writable)
741 struct vring_desc *dp;
742 struct vq_desc_extra *dxp;
746 needed = readable + writable;
747 VQASSERT(vq, needed <= vq->vq_max_indirect_size,
748 "enqueuing too many indirect descriptors");
750 head_idx = vq->vq_desc_head_idx;
751 VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
752 dp = &vq->vq_ring.desc[head_idx];
753 dxp = &vq->vq_descx[head_idx];
755 VQASSERT(vq, dxp->cookie == NULL,
756 "cookie already exists for index %d", head_idx);
757 dxp->cookie = cookie;
760 dp->addr = dxp->indirect_paddr;
761 dp->len = needed * sizeof(struct vring_desc);
762 dp->flags = VRING_DESC_F_INDIRECT;
764 vq_ring_enqueue_segments(vq, dxp->indirect, 0,
765 sg, readable, writable);
767 vq->vq_desc_head_idx = dp->next;
769 if (vq->vq_free_cnt == 0)
770 VQ_RING_ASSERT_CHAIN_TERM(vq);
772 VQ_RING_ASSERT_VALID_IDX(vq, vq->vq_desc_head_idx);
774 vq_ring_update_avail(vq, head_idx);
778 vq_ring_enable_interrupt(struct virtqueue *vq, uint16_t ndesc)
782 * Enable interrupts, making sure we get the latest index of
783 * what's already been consumed.
785 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX)
786 vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx + ndesc;
788 vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
793 * Enough items may have already been consumed to meet our threshold
794 * since we last checked. Let our caller know so it processes the new
797 if (virtqueue_nused(vq) > ndesc)
804 vq_ring_must_notify_host(struct virtqueue *vq)
806 uint16_t new_idx, prev_idx, event_idx;
808 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
809 new_idx = vq->vq_ring.avail->idx;
810 prev_idx = new_idx - vq->vq_queued_cnt;
811 event_idx = vring_avail_event(&vq->vq_ring);
813 return (vring_need_event(event_idx, new_idx, prev_idx) != 0);
816 return ((vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY) == 0);
820 vq_ring_notify_host(struct virtqueue *vq)
823 VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index);
827 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
829 struct vring_desc *dp;
830 struct vq_desc_extra *dxp;
832 VQ_RING_ASSERT_VALID_IDX(vq, desc_idx);
833 dp = &vq->vq_ring.desc[desc_idx];
834 dxp = &vq->vq_descx[desc_idx];
836 if (vq->vq_free_cnt == 0)
837 VQ_RING_ASSERT_CHAIN_TERM(vq);
839 vq->vq_free_cnt += dxp->ndescs;
842 if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
843 while (dp->flags & VRING_DESC_F_NEXT) {
844 VQ_RING_ASSERT_VALID_IDX(vq, dp->next);
845 dp = &vq->vq_ring.desc[dp->next];
850 VQASSERT(vq, dxp->ndescs == 0,
851 "failed to free entire desc chain, remaining: %d", dxp->ndescs);
854 * We must append the existing free chain, if any, to the end of
855 * newly freed chain. If the virtqueue was completely used, then
856 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
858 dp->next = vq->vq_desc_head_idx;
859 vq->vq_desc_head_idx = desc_idx;