sys/dev/virtio/virtqueue.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice unmodified, this list of conditions, and the following
  12  *    disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  */
  28
  29 /*
  30  * Implements the virtqueue interface as basically described
  31  * in the original VirtIO paper.
  32  */
  33
  34 #include <sys/cdefs.h>
  35 __FBSDID("$FreeBSD$");
  36
  37 #include <sys/param.h>
  38 #include <sys/systm.h>
  39 #include <sys/kernel.h>
  40 #include <sys/malloc.h>
  41 #include <sys/sglist.h>
  42 #include <vm/vm.h>
  43 #include <vm/pmap.h>
  44
  45 #include <machine/cpu.h>
  46 #include <machine/bus.h>
  47 #include <machine/atomic.h>
  48 #include <machine/resource.h>
  49 #include <sys/bus.h>
  50 #include <sys/rman.h>
  51
  52 #include <dev/virtio/virtio.h>
  53 #include <dev/virtio/virtqueue.h>
  54 #include <dev/virtio/virtio_ring.h>
  55
  56 #include "virtio_bus_if.h"
  57
  58 struct virtqueue {
  59         device_t                 vq_dev;
  60         char                     vq_name[VIRTQUEUE_MAX_NAME_SZ];
  61         uint16_t                 vq_queue_index;
  62         uint16_t                 vq_nentries;
  63         uint32_t                 vq_flags;
  64 #define VIRTQUEUE_FLAG_INDIRECT  0x0001
  65 #define VIRTQUEUE_FLAG_EVENT_IDX 0x0002
  66
  67         int                      vq_alignment;
  68         int                      vq_ring_size;
  69         void                    *vq_ring_mem;
  70         int                      vq_max_indirect_size;
  71         int                      vq_indirect_mem_size;
  72         virtqueue_intr_t        *vq_intrhand;
  73         void                    *vq_intrhand_arg;
  74
  75         struct vring             vq_ring;
  76         uint16_t                 vq_free_cnt;
  77         uint16_t                 vq_queued_cnt;
  78         /*
  79          * Head of the free chain in the descriptor table. If
  80          * there are no free descriptors, this will be set to
  81          * VQ_RING_DESC_CHAIN_END.
  82          */
  83         uint16_t                 vq_desc_head_idx;
  84         /*
  85          * Last consumed descriptor in the used table,
  86          * trails vq_ring.used->idx.
  87          */
  88         uint16_t                 vq_used_cons_idx;
  89
  90         struct vq_desc_extra {
  91                 void              *cookie;
  92                 struct vring_desc *indirect;
  93                 vm_paddr_t         indirect_paddr;
  94                 uint16_t           ndescs;
  95         } vq_descx[0];
  96 };
  97
  98 /*
  99  * The maximum virtqueue size is 2^15. Use that value as the end of
 100  * descriptor chain terminator since it will never be a valid index
 101  * in the descriptor table. This is used to verify we are correctly
 102  * handling vq_free_cnt.
 103  */
 104 #define VQ_RING_DESC_CHAIN_END 32768
 105
 106 #define VQASSERT(_vq, _exp, _msg, ...)                          \
 107     KASSERT((_exp),("%s: %s - "_msg, __func__, (_vq)->vq_name,  \
 108         ##__VA_ARGS__))
 109
 110 #define VQ_RING_ASSERT_VALID_IDX(_vq, _idx)                     \
 111     VQASSERT((_vq), (_idx) < (_vq)->vq_nentries,                \
 112         "invalid ring index: %d, max: %d", (_idx),              \
 113         (_vq)->vq_nentries)
 114
 115 #define VQ_RING_ASSERT_CHAIN_TERM(_vq)                          \
 116     VQASSERT((_vq), (_vq)->vq_desc_head_idx ==                  \
 117         VQ_RING_DESC_CHAIN_END, "full ring terminated "         \
 118         "incorrectly: head idx: %d", (_vq)->vq_desc_head_idx)
 119
 120 static int      virtqueue_init_indirect(struct virtqueue *vq, int);
 121 static void     virtqueue_free_indirect(struct virtqueue *vq);
 122 static void     virtqueue_init_indirect_list(struct virtqueue *,
 123                     struct vring_desc *);
 124
 125 static void     vq_ring_init(struct virtqueue *);
 126 static void     vq_ring_update_avail(struct virtqueue *, uint16_t);
 127 static uint16_t vq_ring_enqueue_segments(struct virtqueue *,
 128                     struct vring_desc *, uint16_t, struct sglist *, int, int);
 129 static int      vq_ring_use_indirect(struct virtqueue *, int);
 130 static void     vq_ring_enqueue_indirect(struct virtqueue *, void *,
 131                     struct sglist *, int, int);
 132 static int      vq_ring_enable_interrupt(struct virtqueue *, uint16_t);
 133 static int      vq_ring_must_notify_host(struct virtqueue *);
 134 static void     vq_ring_notify_host(struct virtqueue *);
 135 static void     vq_ring_free_chain(struct virtqueue *, uint16_t);
 136
 137 uint64_t
 138 virtqueue_filter_features(uint64_t features)
 139 {
 140         uint64_t mask;
 141
 142         mask = (1 << VIRTIO_TRANSPORT_F_START) - 1;
 143         mask |= VIRTIO_RING_F_INDIRECT_DESC;
 144         mask |= VIRTIO_RING_F_EVENT_IDX;
 145         mask |= VIRTIO_F_VERSION_1;
 146
 147         return (features & mask);
 148 }
 149
 150 int
 151 virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size, int align,
 152     vm_paddr_t highaddr, struct vq_alloc_info *info, struct virtqueue **vqp)
 153 {
 154         struct virtqueue *vq;
 155         int error;
 156
 157         *vqp = NULL;
 158         error = 0;
 159
 160         if (size == 0) {
 161                 device_printf(dev,
 162                     "virtqueue %d (%s) does not exist (size is zero)\n",
 163                     queue, info->vqai_name);
 164                 return (ENODEV);
 165         } else if (!powerof2(size)) {
 166                 device_printf(dev,
 167                     "virtqueue %d (%s) size is not a power of 2: %d\n",
 168                     queue, info->vqai_name, size);
 169                 return (ENXIO);
 170         } else if (info->vqai_maxindirsz > VIRTIO_MAX_INDIRECT) {
 171                 device_printf(dev, "virtqueue %d (%s) requested too many "
 172                     "indirect descriptors: %d, max %d\n",
 173                     queue, info->vqai_name, info->vqai_maxindirsz,
 174                     VIRTIO_MAX_INDIRECT);
 175                 return (EINVAL);
 176         }
 177
 178         vq = malloc(sizeof(struct virtqueue) +
 179             size * sizeof(struct vq_desc_extra), M_DEVBUF, M_NOWAIT | M_ZERO);
 180         if (vq == NULL) {
 181                 device_printf(dev, "cannot allocate virtqueue\n");
 182                 return (ENOMEM);
 183         }
 184
 185         vq->vq_dev = dev;
 186         strlcpy(vq->vq_name, info->vqai_name, sizeof(vq->vq_name));
 187         vq->vq_queue_index = queue;
 188         vq->vq_alignment = align;
 189         vq->vq_nentries = size;
 190         vq->vq_free_cnt = size;
 191         vq->vq_intrhand = info->vqai_intr;
 192         vq->vq_intrhand_arg = info->vqai_intr_arg;
 193
 194         if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_EVENT_IDX) != 0)
 195                 vq->vq_flags |= VIRTQUEUE_FLAG_EVENT_IDX;
 196
 197         if (info->vqai_maxindirsz > 1) {
 198                 error = virtqueue_init_indirect(vq, info->vqai_maxindirsz);
 199                 if (error)
 200                         goto fail;
 201         }
 202
 203         vq->vq_ring_size = round_page(vring_size(size, align));
 204         vq->vq_ring_mem = contigmalloc(vq->vq_ring_size, M_DEVBUF,
 205             M_NOWAIT | M_ZERO, 0, highaddr, PAGE_SIZE, 0);
 206         if (vq->vq_ring_mem == NULL) {
 207                 device_printf(dev,
 208                     "cannot allocate memory for virtqueue ring\n");
 209                 error = ENOMEM;
 210                 goto fail;
 211         }
 212
 213         vq_ring_init(vq);
 214         virtqueue_disable_intr(vq);
 215
 216         *vqp = vq;
 217
 218 fail:
 219         if (error)
 220                 virtqueue_free(vq);
 221
 222         return (error);
 223 }
 224
 225 static int
 226 virtqueue_init_indirect(struct virtqueue *vq, int indirect_size)
 227 {
 228         device_t dev;
 229         struct vq_desc_extra *dxp;
 230         int i, size;
 231
 232         dev = vq->vq_dev;
 233
 234         if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_INDIRECT_DESC) == 0) {
 235                 /*
 236                  * Indirect descriptors requested by the driver but not
 237                  * negotiated. Return zero to keep the initialization
 238                  * going: we'll run fine without.
 239                  */
 240                 if (bootverbose)
 241                         device_printf(dev, "virtqueue %d (%s) requested "
 242                             "indirect descriptors but not negotiated\n",
 243                             vq->vq_queue_index, vq->vq_name);
 244                 return (0);
 245         }
 246
 247         size = indirect_size * sizeof(struct vring_desc);
 248         vq->vq_max_indirect_size = indirect_size;
 249         vq->vq_indirect_mem_size = size;
 250         vq->vq_flags |= VIRTQUEUE_FLAG_INDIRECT;
 251
 252         for (i = 0; i < vq->vq_nentries; i++) {
 253                 dxp = &vq->vq_descx[i];
 254
 255                 dxp->indirect = malloc(size, M_DEVBUF, M_NOWAIT);
 256                 if (dxp->indirect == NULL) {
 257                         device_printf(dev, "cannot allocate indirect list\n");
 258                         return (ENOMEM);
 259                 }
 260
 261                 dxp->indirect_paddr = vtophys(dxp->indirect);
 262                 virtqueue_init_indirect_list(vq, dxp->indirect);
 263         }
 264
 265         return (0);
 266 }
 267
 268 static void
 269 virtqueue_free_indirect(struct virtqueue *vq)
 270 {
 271         struct vq_desc_extra *dxp;
 272         int i;
 273
 274         for (i = 0; i < vq->vq_nentries; i++) {
 275                 dxp = &vq->vq_descx[i];
 276
 277                 if (dxp->indirect == NULL)
 278                         break;
 279
 280                 free(dxp->indirect, M_DEVBUF);
 281                 dxp->indirect = NULL;
 282                 dxp->indirect_paddr = 0;
 283         }
 284
 285         vq->vq_flags &= ~VIRTQUEUE_FLAG_INDIRECT;
 286         vq->vq_indirect_mem_size = 0;
 287 }
 288
 289 static void
 290 virtqueue_init_indirect_list(struct virtqueue *vq,
 291     struct vring_desc *indirect)
 292 {
 293         int i;
 294
 295         bzero(indirect, vq->vq_indirect_mem_size);
 296
 297         for (i = 0; i < vq->vq_max_indirect_size - 1; i++)
 298                 indirect[i].next = i + 1;
 299         indirect[i].next = VQ_RING_DESC_CHAIN_END;
 300 }
 301
 302 int
 303 virtqueue_reinit(struct virtqueue *vq, uint16_t size)
 304 {
 305         struct vq_desc_extra *dxp;
 306         int i;
 307
 308         if (vq->vq_nentries != size) {
 309                 device_printf(vq->vq_dev,
 310                     "%s: '%s' changed size; old=%hu, new=%hu\n",
 311                     __func__, vq->vq_name, vq->vq_nentries, size);
 312                 return (EINVAL);
 313         }
 314
 315         /* Warn if the virtqueue was not properly cleaned up. */
 316         if (vq->vq_free_cnt != vq->vq_nentries) {
 317                 device_printf(vq->vq_dev,
 318                     "%s: warning '%s' virtqueue not empty, "
 319                     "leaking %d entries\n", __func__, vq->vq_name,
 320                     vq->vq_nentries - vq->vq_free_cnt);
 321         }
 322
 323         vq->vq_desc_head_idx = 0;
 324         vq->vq_used_cons_idx = 0;
 325         vq->vq_queued_cnt = 0;
 326         vq->vq_free_cnt = vq->vq_nentries;
 327
 328         /* To be safe, reset all our allocated memory. */
 329         bzero(vq->vq_ring_mem, vq->vq_ring_size);
 330         for (i = 0; i < vq->vq_nentries; i++) {
 331                 dxp = &vq->vq_descx[i];
 332                 dxp->cookie = NULL;
 333                 dxp->ndescs = 0;
 334                 if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT)
 335                         virtqueue_init_indirect_list(vq, dxp->indirect);
 336         }
 337
 338         vq_ring_init(vq);
 339         virtqueue_disable_intr(vq);
 340
 341         return (0);
 342 }
 343
 344 void
 345 virtqueue_free(struct virtqueue *vq)
 346 {
 347
 348         if (vq->vq_free_cnt != vq->vq_nentries) {
 349                 device_printf(vq->vq_dev, "%s: freeing non-empty virtqueue, "
 350                     "leaking %d entries\n", vq->vq_name,
 351                     vq->vq_nentries - vq->vq_free_cnt);
 352         }
 353
 354         if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT)
 355                 virtqueue_free_indirect(vq);
 356
 357         if (vq->vq_ring_mem != NULL) {
 358                 contigfree(vq->vq_ring_mem, vq->vq_ring_size, M_DEVBUF);
 359                 vq->vq_ring_size = 0;
 360                 vq->vq_ring_mem = NULL;
 361         }
 362
 363         free(vq, M_DEVBUF);
 364 }
 365
 366 vm_paddr_t
 367 virtqueue_paddr(struct virtqueue *vq)
 368 {
 369
 370         return (vtophys(vq->vq_ring_mem));
 371 }
 372
 373 vm_paddr_t
 374 virtqueue_desc_paddr(struct virtqueue *vq)
 375 {
 376
 377         return (vtophys(vq->vq_ring.desc));
 378 }
 379
 380 vm_paddr_t
 381 virtqueue_avail_paddr(struct virtqueue *vq)
 382 {
 383
 384         return (vtophys(vq->vq_ring.avail));
 385 }
 386
 387 vm_paddr_t
 388 virtqueue_used_paddr(struct virtqueue *vq)
 389 {
 390
 391         return (vtophys(vq->vq_ring.used));
 392 }
 393
 394 uint16_t
 395 virtqueue_index(struct virtqueue *vq)
 396 {
 397         return (vq->vq_queue_index);
 398 }
 399
 400 int
 401 virtqueue_size(struct virtqueue *vq)
 402 {
 403
 404         return (vq->vq_nentries);
 405 }
 406
 407 int
 408 virtqueue_nfree(struct virtqueue *vq)
 409 {
 410
 411         return (vq->vq_free_cnt);
 412 }
 413
 414 int
 415 virtqueue_empty(struct virtqueue *vq)
 416 {
 417
 418         return (vq->vq_nentries == vq->vq_free_cnt);
 419 }
 420
 421 int
 422 virtqueue_full(struct virtqueue *vq)
 423 {
 424
 425         return (vq->vq_free_cnt == 0);
 426 }
 427
 428 void
 429 virtqueue_notify(struct virtqueue *vq)
 430 {
 431
 432         /* Ensure updated avail->idx is visible to host. */
 433         mb();
 434
 435         if (vq_ring_must_notify_host(vq))
 436                 vq_ring_notify_host(vq);
 437         vq->vq_queued_cnt = 0;
 438 }
 439
 440 int
 441 virtqueue_nused(struct virtqueue *vq)
 442 {
 443         uint16_t used_idx, nused;
 444
 445         used_idx = vq->vq_ring.used->idx;
 446
 447         nused = (uint16_t)(used_idx - vq->vq_used_cons_idx);
 448         VQASSERT(vq, nused <= vq->vq_nentries, "used more than available");
 449
 450         return (nused);
 451 }
 452
 453 int
 454 virtqueue_intr_filter(struct virtqueue *vq)
 455 {
 456
 457         if (vq->vq_used_cons_idx == vq->vq_ring.used->idx)
 458                 return (0);
 459
 460         virtqueue_disable_intr(vq);
 461
 462         return (1);
 463 }
 464
 465 void
 466 virtqueue_intr(struct virtqueue *vq)
 467 {
 468
 469         vq->vq_intrhand(vq->vq_intrhand_arg);
 470 }
 471
 472 int
 473 virtqueue_enable_intr(struct virtqueue *vq)
 474 {
 475
 476         return (vq_ring_enable_interrupt(vq, 0));
 477 }
 478
 479 int
 480 virtqueue_postpone_intr(struct virtqueue *vq, vq_postpone_t hint)
 481 {
 482         uint16_t ndesc, avail_idx;
 483
 484         avail_idx = vq->vq_ring.avail->idx;
 485         ndesc = (uint16_t)(avail_idx - vq->vq_used_cons_idx);
 486
 487         switch (hint) {
 488         case VQ_POSTPONE_SHORT:
 489                 ndesc = ndesc / 4;
 490                 break;
 491         case VQ_POSTPONE_LONG:
 492                 ndesc = (ndesc * 3) / 4;
 493                 break;
 494         case VQ_POSTPONE_EMPTIED:
 495                 break;
 496         }
 497
 498         return (vq_ring_enable_interrupt(vq, ndesc));
 499 }
 500
 501 /*
 502  * Note this is only considered a hint to the host.
 503  */
 504 void
 505 virtqueue_disable_intr(struct virtqueue *vq)
 506 {
 507
 508         if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
 509                 vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx -
 510                     vq->vq_nentries - 1;
 511         } else
 512                 vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
 513 }
 514
 515 int
 516 virtqueue_enqueue(struct virtqueue *vq, void *cookie, struct sglist *sg,
 517     int readable, int writable)
 518 {
 519         struct vq_desc_extra *dxp;
 520         int needed;
 521         uint16_t head_idx, idx;
 522
 523         needed = readable + writable;
 524
 525         VQASSERT(vq, cookie != NULL, "enqueuing with no cookie");
 526         VQASSERT(vq, needed == sg->sg_nseg,
 527             "segment count mismatch, %d, %d", needed, sg->sg_nseg);
 528         VQASSERT(vq,
 529             needed <= vq->vq_nentries || needed <= vq->vq_max_indirect_size,
 530             "too many segments to enqueue: %d, %d/%d", needed,
 531             vq->vq_nentries, vq->vq_max_indirect_size);
 532
 533         if (needed < 1)
 534                 return (EINVAL);
 535         if (vq->vq_free_cnt == 0)
 536                 return (ENOSPC);
 537
 538         if (vq_ring_use_indirect(vq, needed)) {
 539                 vq_ring_enqueue_indirect(vq, cookie, sg, readable, writable);
 540                 return (0);
 541         } else if (vq->vq_free_cnt < needed)
 542                 return (EMSGSIZE);
 543
 544         head_idx = vq->vq_desc_head_idx;
 545         VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
 546         dxp = &vq->vq_descx[head_idx];
 547
 548         VQASSERT(vq, dxp->cookie == NULL,
 549             "cookie already exists for index %d", head_idx);
 550         dxp->cookie = cookie;
 551         dxp->ndescs = needed;
 552
 553         idx = vq_ring_enqueue_segments(vq, vq->vq_ring.desc, head_idx,
 554             sg, readable, writable);
 555
 556         vq->vq_desc_head_idx = idx;
 557         vq->vq_free_cnt -= needed;
 558         if (vq->vq_free_cnt == 0)
 559                 VQ_RING_ASSERT_CHAIN_TERM(vq);
 560         else
 561                 VQ_RING_ASSERT_VALID_IDX(vq, idx);
 562
 563         vq_ring_update_avail(vq, head_idx);
 564
 565         return (0);
 566 }
 567
 568 void *
 569 virtqueue_dequeue(struct virtqueue *vq, uint32_t *len)
 570 {
 571         struct vring_used_elem *uep;
 572         void *cookie;
 573         uint16_t used_idx, desc_idx;
 574
 575         if (vq->vq_used_cons_idx == vq->vq_ring.used->idx)
 576                 return (NULL);
 577
 578         used_idx = vq->vq_used_cons_idx++ & (vq->vq_nentries - 1);
 579         uep = &vq->vq_ring.used->ring[used_idx];
 580
 581         rmb();
 582         desc_idx = (uint16_t) uep->id;
 583         if (len != NULL)
 584                 *len = uep->len;
 585
 586         vq_ring_free_chain(vq, desc_idx);
 587
 588         cookie = vq->vq_descx[desc_idx].cookie;
 589         VQASSERT(vq, cookie != NULL, "no cookie for index %d", desc_idx);
 590         vq->vq_descx[desc_idx].cookie = NULL;
 591
 592         return (cookie);
 593 }
 594
 595 void *
 596 virtqueue_poll(struct virtqueue *vq, uint32_t *len)
 597 {
 598         void *cookie;
 599
 600         VIRTIO_BUS_POLL(vq->vq_dev);
 601         while ((cookie = virtqueue_dequeue(vq, len)) == NULL) {
 602                 cpu_spinwait();
 603                 VIRTIO_BUS_POLL(vq->vq_dev);
 604         }
 605
 606         return (cookie);
 607 }
 608
 609 void *
 610 virtqueue_drain(struct virtqueue *vq, int *last)
 611 {
 612         void *cookie;
 613         int idx;
 614
 615         cookie = NULL;
 616         idx = *last;
 617
 618         while (idx < vq->vq_nentries && cookie == NULL) {
 619                 if ((cookie = vq->vq_descx[idx].cookie) != NULL) {
 620                         vq->vq_descx[idx].cookie = NULL;
 621                         /* Free chain to keep free count consistent. */
 622                         vq_ring_free_chain(vq, idx);
 623                 }
 624                 idx++;
 625         }
 626
 627         *last = idx;
 628
 629         return (cookie);
 630 }
 631
 632 void
 633 virtqueue_dump(struct virtqueue *vq)
 634 {
 635
 636         if (vq == NULL)
 637                 return;
 638
 639         printf("VQ: %s - size=%d; free=%d; used=%d; queued=%d; "
 640             "desc_head_idx=%d; avail.idx=%d; used_cons_idx=%d; "
 641             "used.idx=%d; used_event_idx=%d; avail.flags=0x%x; used.flags=0x%x\n",
 642             vq->vq_name, vq->vq_nentries, vq->vq_free_cnt,
 643             virtqueue_nused(vq), vq->vq_queued_cnt, vq->vq_desc_head_idx,
 644             vq->vq_ring.avail->idx, vq->vq_used_cons_idx,
 645             vq->vq_ring.used->idx,
 646                 vring_used_event(&vq->vq_ring),
 647             vq->vq_ring.avail->flags,
 648             vq->vq_ring.used->flags);
 649 }
 650
 651 static void
 652 vq_ring_init(struct virtqueue *vq)
 653 {
 654         struct vring *vr;
 655         char *ring_mem;
 656         int i, size;
 657
 658         ring_mem = vq->vq_ring_mem;
 659         size = vq->vq_nentries;
 660         vr = &vq->vq_ring;
 661
 662         vring_init(vr, size, ring_mem, vq->vq_alignment);
 663
 664         for (i = 0; i < size - 1; i++)
 665                 vr->desc[i].next = i + 1;
 666         vr->desc[i].next = VQ_RING_DESC_CHAIN_END;
 667 }
 668
 669 static void
 670 vq_ring_update_avail(struct virtqueue *vq, uint16_t desc_idx)
 671 {
 672         uint16_t avail_idx;
 673
 674         /*
 675          * Place the head of the descriptor chain into the next slot and make
 676          * it usable to the host. The chain is made available now rather than
 677          * deferring to virtqueue_notify() in the hopes that if the host is
 678          * currently running on another CPU, we can keep it processing the new
 679          * descriptor.
 680          */
 681         avail_idx = vq->vq_ring.avail->idx & (vq->vq_nentries - 1);
 682         vq->vq_ring.avail->ring[avail_idx] = desc_idx;
 683
 684         wmb();
 685         vq->vq_ring.avail->idx++;
 686
 687         /* Keep pending count until virtqueue_notify(). */
 688         vq->vq_queued_cnt++;
 689 }
 690
 691 static uint16_t
 692 vq_ring_enqueue_segments(struct virtqueue *vq, struct vring_desc *desc,
 693     uint16_t head_idx, struct sglist *sg, int readable, int writable)
 694 {
 695         struct sglist_seg *seg;
 696         struct vring_desc *dp;
 697         int i, needed;
 698         uint16_t idx;
 699
 700         needed = readable + writable;
 701
 702         for (i = 0, idx = head_idx, seg = sg->sg_segs;
 703              i < needed;
 704              i++, idx = dp->next, seg++) {
 705                 VQASSERT(vq, idx != VQ_RING_DESC_CHAIN_END,
 706                     "premature end of free desc chain");
 707
 708                 dp = &desc[idx];
 709                 dp->addr = seg->ss_paddr;
 710                 dp->len = seg->ss_len;
 711                 dp->flags = 0;
 712
 713                 if (i < needed - 1)
 714                         dp->flags |= VRING_DESC_F_NEXT;
 715                 if (i >= readable)
 716                         dp->flags |= VRING_DESC_F_WRITE;
 717         }
 718
 719         return (idx);
 720 }
 721
 722 static int
 723 vq_ring_use_indirect(struct virtqueue *vq, int needed)
 724 {
 725
 726         if ((vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) == 0)
 727                 return (0);
 728
 729         if (vq->vq_max_indirect_size < needed)
 730                 return (0);
 731
 732         if (needed < 2)
 733                 return (0);
 734
 735         return (1);
 736 }
 737
 738 static void
 739 vq_ring_enqueue_indirect(struct virtqueue *vq, void *cookie,
 740     struct sglist *sg, int readable, int writable)
 741 {
 742         struct vring_desc *dp;
 743         struct vq_desc_extra *dxp;
 744         int needed;
 745         uint16_t head_idx;
 746
 747         needed = readable + writable;
 748         VQASSERT(vq, needed <= vq->vq_max_indirect_size,
 749             "enqueuing too many indirect descriptors");
 750
 751         head_idx = vq->vq_desc_head_idx;
 752         VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
 753         dp = &vq->vq_ring.desc[head_idx];
 754         dxp = &vq->vq_descx[head_idx];
 755
 756         VQASSERT(vq, dxp->cookie == NULL,
 757             "cookie already exists for index %d", head_idx);
 758         dxp->cookie = cookie;
 759         dxp->ndescs = 1;
 760
 761         dp->addr = dxp->indirect_paddr;
 762         dp->len = needed * sizeof(struct vring_desc);
 763         dp->flags = VRING_DESC_F_INDIRECT;
 764
 765         vq_ring_enqueue_segments(vq, dxp->indirect, 0,
 766             sg, readable, writable);
 767
 768         vq->vq_desc_head_idx = dp->next;
 769         vq->vq_free_cnt--;
 770         if (vq->vq_free_cnt == 0)
 771                 VQ_RING_ASSERT_CHAIN_TERM(vq);
 772         else
 773                 VQ_RING_ASSERT_VALID_IDX(vq, vq->vq_desc_head_idx);
 774
 775         vq_ring_update_avail(vq, head_idx);
 776 }
 777
 778 static int
 779 vq_ring_enable_interrupt(struct virtqueue *vq, uint16_t ndesc)
 780 {
 781
 782         /*
 783          * Enable interrupts, making sure we get the latest index of
 784          * what's already been consumed.
 785          */
 786         if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX)
 787                 vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx + ndesc;
 788         else
 789                 vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
 790
 791         mb();
 792
 793         /*
 794          * Enough items may have already been consumed to meet our threshold
 795          * since we last checked. Let our caller know so it processes the new
 796          * entries.
 797          */
 798         if (virtqueue_nused(vq) > ndesc)
 799                 return (1);
 800
 801         return (0);
 802 }
 803
 804 static int
 805 vq_ring_must_notify_host(struct virtqueue *vq)
 806 {
 807         uint16_t new_idx, prev_idx, event_idx;
 808
 809         if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
 810                 new_idx = vq->vq_ring.avail->idx;
 811                 prev_idx = new_idx - vq->vq_queued_cnt;
 812                 event_idx = vring_avail_event(&vq->vq_ring);
 813
 814                 return (vring_need_event(event_idx, new_idx, prev_idx) != 0);
 815         }
 816
 817         return ((vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY) == 0);
 818 }
 819
 820 static void
 821 vq_ring_notify_host(struct virtqueue *vq)
 822 {
 823
 824         VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index);
 825 }
 826
 827 static void
 828 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
 829 {
 830         struct vring_desc *dp;
 831         struct vq_desc_extra *dxp;
 832
 833         VQ_RING_ASSERT_VALID_IDX(vq, desc_idx);
 834         dp = &vq->vq_ring.desc[desc_idx];
 835         dxp = &vq->vq_descx[desc_idx];
 836
 837         if (vq->vq_free_cnt == 0)
 838                 VQ_RING_ASSERT_CHAIN_TERM(vq);
 839
 840         vq->vq_free_cnt += dxp->ndescs;
 841         dxp->ndescs--;
 842
 843         if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
 844                 while (dp->flags & VRING_DESC_F_NEXT) {
 845                         VQ_RING_ASSERT_VALID_IDX(vq, dp->next);
 846                         dp = &vq->vq_ring.desc[dp->next];
 847                         dxp->ndescs--;
 848                 }
 849         }
 850
 851         VQASSERT(vq, dxp->ndescs == 0,
 852             "failed to free entire desc chain, remaining: %d", dxp->ndescs);
 853
 854         /*
 855          * We must append the existing free chain, if any, to the end of
 856          * newly freed chain. If the virtqueue was completely used, then
 857          * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
 858          */
 859         dp->next = vq->vq_desc_head_idx;
 860         vq->vq_desc_head_idx = desc_idx;
 861 }