sys/dev/virtio/virtqueue.c

   1 /*-
   2  * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice unmodified, this list of conditions, and the following
  10  *    disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 /*
  28  * Implements the virtqueue interface as basically described
  29  * in the original VirtIO paper.
  30  */
  31
  32 #include <sys/cdefs.h>
  33 __FBSDID("$FreeBSD$");
  34
  35 #include <sys/param.h>
  36 #include <sys/systm.h>
  37 #include <sys/kernel.h>
  38 #include <sys/malloc.h>
  39 #include <sys/sglist.h>
  40 #include <vm/vm.h>
  41 #include <vm/pmap.h>
  42
  43 #include <machine/cpu.h>
  44 #include <machine/bus.h>
  45 #include <machine/atomic.h>
  46 #include <machine/resource.h>
  47 #include <sys/bus.h>
  48 #include <sys/rman.h>
  49
  50 #include <dev/virtio/virtio.h>
  51 #include <dev/virtio/virtio_config.h>
  52 #include <dev/virtio/virtqueue.h>
  53 #include <dev/virtio/virtio_ring.h>
  54
  55 #include "virtio_bus_if.h"
  56
  57 struct virtqueue {
  58         device_t                 vq_dev;
  59         char                     vq_name[VIRTQUEUE_MAX_NAME_SZ];
  60         uint16_t                 vq_queue_index;
  61         uint16_t                 vq_nentries;
  62         uint32_t                 vq_flags;
  63 #define VIRTQUEUE_FLAG_INDIRECT  0x0001
  64 #define VIRTQUEUE_FLAG_EVENT_IDX 0x0002
  65
  66         int                      vq_alignment;
  67         int                      vq_ring_size;
  68         void                    *vq_ring_mem;
  69         int                      vq_max_indirect_size;
  70         int                      vq_indirect_mem_size;
  71         virtqueue_intr_t        *vq_intrhand;
  72         void                    *vq_intrhand_arg;
  73
  74         struct vring             vq_ring;
  75         uint16_t                 vq_free_cnt;
  76         uint16_t                 vq_queued_cnt;
  77         /*
  78          * Head of the free chain in the descriptor table. If
  79          * there are no free descriptors, this will be set to
  80          * VQ_RING_DESC_CHAIN_END.
  81          */
  82         uint16_t                 vq_desc_head_idx;
  83         /*
  84          * Last consumed descriptor in the used table,
  85          * trails vq_ring.used->idx.
  86          */
  87         uint16_t                 vq_used_cons_idx;
  88
  89         struct vq_desc_extra {
  90                 void              *cookie;
  91                 struct vring_desc *indirect;
  92                 vm_paddr_t         indirect_paddr;
  93                 uint16_t           ndescs;
  94         } vq_descx[0];
  95 };
  96
  97 /*
  98  * The maximum virtqueue size is 2^15. Use that value as the end of
  99  * descriptor chain terminator since it will never be a valid index
 100  * in the descriptor table. This is used to verify we are correctly
 101  * handling vq_free_cnt.
 102  */
 103 #define VQ_RING_DESC_CHAIN_END 32768
 104
 105 #define VQASSERT(_vq, _exp, _msg, ...)                          \
 106     KASSERT((_exp),("%s: %s - "_msg, __func__, (_vq)->vq_name,  \
 107         ##__VA_ARGS__))
 108
 109 #define VQ_RING_ASSERT_VALID_IDX(_vq, _idx)                     \
 110     VQASSERT((_vq), (_idx) < (_vq)->vq_nentries,                \
 111         "invalid ring index: %d, max: %d", (_idx),              \
 112         (_vq)->vq_nentries)
 113
 114 #define VQ_RING_ASSERT_CHAIN_TERM(_vq)                          \
 115     VQASSERT((_vq), (_vq)->vq_desc_head_idx ==                  \
 116         VQ_RING_DESC_CHAIN_END, "full ring terminated "         \
 117         "incorrectly: head idx: %d", (_vq)->vq_desc_head_idx)
 118
 119 static int      virtqueue_init_indirect(struct virtqueue *vq, int);
 120 static void     virtqueue_free_indirect(struct virtqueue *vq);
 121 static void     virtqueue_init_indirect_list(struct virtqueue *,
 122                     struct vring_desc *);
 123
 124 static void     vq_ring_init(struct virtqueue *);
 125 static void     vq_ring_update_avail(struct virtqueue *, uint16_t);
 126 static uint16_t vq_ring_enqueue_segments(struct virtqueue *,
 127                     struct vring_desc *, uint16_t, struct sglist *, int, int);
 128 static int      vq_ring_use_indirect(struct virtqueue *, int);
 129 static void     vq_ring_enqueue_indirect(struct virtqueue *, void *,
 130                     struct sglist *, int, int);
 131 static int      vq_ring_enable_interrupt(struct virtqueue *, uint16_t);
 132 static int      vq_ring_must_notify_host(struct virtqueue *);
 133 static void     vq_ring_notify_host(struct virtqueue *);
 134 static void     vq_ring_free_chain(struct virtqueue *, uint16_t);
 135
 136 uint64_t
 137 virtqueue_filter_features(uint64_t features)
 138 {
 139         uint64_t mask;
 140
 141         mask = (1 << VIRTIO_TRANSPORT_F_START) - 1;
 142         mask |= VIRTIO_RING_F_INDIRECT_DESC;
 143         mask |= VIRTIO_RING_F_EVENT_IDX;
 144
 145         return (features & mask);
 146 }
 147
 148 int
 149 virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size, int align,
 150     vm_paddr_t highaddr, struct vq_alloc_info *info, struct virtqueue **vqp)
 151 {
 152         struct virtqueue *vq;
 153         int error;
 154
 155         *vqp = NULL;
 156         error = 0;
 157
 158         if (size == 0) {
 159                 device_printf(dev,
 160                     "virtqueue %d (%s) does not exist (size is zero)\n",
 161                     queue, info->vqai_name);
 162                 return (ENODEV);
 163         } else if (!powerof2(size)) {
 164                 device_printf(dev,
 165                     "virtqueue %d (%s) size is not a power of 2: %d\n",
 166                     queue, info->vqai_name, size);
 167                 return (ENXIO);
 168         } else if (info->vqai_maxindirsz > VIRTIO_MAX_INDIRECT) {
 169                 device_printf(dev, "virtqueue %d (%s) requested too many "
 170                     "indirect descriptors: %d, max %d\n",
 171                     queue, info->vqai_name, info->vqai_maxindirsz,
 172                     VIRTIO_MAX_INDIRECT);
 173                 return (EINVAL);
 174         }
 175
 176         vq = malloc(sizeof(struct virtqueue) +
 177             size * sizeof(struct vq_desc_extra), M_DEVBUF, M_NOWAIT | M_ZERO);
 178         if (vq == NULL) {
 179                 device_printf(dev, "cannot allocate virtqueue\n");
 180                 return (ENOMEM);
 181         }
 182
 183         vq->vq_dev = dev;
 184         strlcpy(vq->vq_name, info->vqai_name, sizeof(vq->vq_name));
 185         vq->vq_queue_index = queue;
 186         vq->vq_alignment = align;
 187         vq->vq_nentries = size;
 188         vq->vq_free_cnt = size;
 189         vq->vq_intrhand = info->vqai_intr;
 190         vq->vq_intrhand_arg = info->vqai_intr_arg;
 191
 192         if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_EVENT_IDX) != 0)
 193                 vq->vq_flags |= VIRTQUEUE_FLAG_EVENT_IDX;
 194
 195         if (info->vqai_maxindirsz > 1) {
 196                 error = virtqueue_init_indirect(vq, info->vqai_maxindirsz);
 197                 if (error)
 198                         goto fail;
 199         }
 200
 201         vq->vq_ring_size = round_page(vring_size(size, align));
 202         vq->vq_ring_mem = contigmalloc(vq->vq_ring_size, M_DEVBUF,
 203             M_NOWAIT | M_ZERO, 0, highaddr, PAGE_SIZE, 0);
 204         if (vq->vq_ring_mem == NULL) {
 205                 device_printf(dev,
 206                     "cannot allocate memory for virtqueue ring\n");
 207                 error = ENOMEM;
 208                 goto fail;
 209         }
 210
 211         vq_ring_init(vq);
 212         virtqueue_disable_intr(vq);
 213
 214         *vqp = vq;
 215
 216 fail:
 217         if (error)
 218                 virtqueue_free(vq);
 219
 220         return (error);
 221 }
 222
 223 static int
 224 virtqueue_init_indirect(struct virtqueue *vq, int indirect_size)
 225 {
 226         device_t dev;
 227         struct vq_desc_extra *dxp;
 228         int i, size;
 229
 230         dev = vq->vq_dev;
 231
 232         if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_INDIRECT_DESC) == 0) {
 233                 /*
 234                  * Indirect descriptors requested by the driver but not
 235                  * negotiated. Return zero to keep the initialization
 236                  * going: we'll run fine without.
 237                  */
 238                 if (bootverbose)
 239                         device_printf(dev, "virtqueue %d (%s) requested "
 240                             "indirect descriptors but not negotiated\n",
 241                             vq->vq_queue_index, vq->vq_name);
 242                 return (0);
 243         }
 244
 245         size = indirect_size * sizeof(struct vring_desc);
 246         vq->vq_max_indirect_size = indirect_size;
 247         vq->vq_indirect_mem_size = size;
 248         vq->vq_flags |= VIRTQUEUE_FLAG_INDIRECT;
 249
 250         for (i = 0; i < vq->vq_nentries; i++) {
 251                 dxp = &vq->vq_descx[i];
 252
 253                 dxp->indirect = malloc(size, M_DEVBUF, M_NOWAIT);
 254                 if (dxp->indirect == NULL) {
 255                         device_printf(dev, "cannot allocate indirect list\n");
 256                         return (ENOMEM);
 257                 }
 258
 259                 dxp->indirect_paddr = vtophys(dxp->indirect);
 260                 virtqueue_init_indirect_list(vq, dxp->indirect);
 261         }
 262
 263         return (0);
 264 }
 265
 266 static void
 267 virtqueue_free_indirect(struct virtqueue *vq)
 268 {
 269         struct vq_desc_extra *dxp;
 270         int i;
 271
 272         for (i = 0; i < vq->vq_nentries; i++) {
 273                 dxp = &vq->vq_descx[i];
 274
 275                 if (dxp->indirect == NULL)
 276                         break;
 277
 278                 free(dxp->indirect, M_DEVBUF);
 279                 dxp->indirect = NULL;
 280                 dxp->indirect_paddr = 0;
 281         }
 282
 283         vq->vq_flags &= ~VIRTQUEUE_FLAG_INDIRECT;
 284         vq->vq_indirect_mem_size = 0;
 285 }
 286
 287 static void
 288 virtqueue_init_indirect_list(struct virtqueue *vq,
 289     struct vring_desc *indirect)
 290 {
 291         int i;
 292
 293         bzero(indirect, vq->vq_indirect_mem_size);
 294
 295         for (i = 0; i < vq->vq_max_indirect_size - 1; i++)
 296                 indirect[i].next = i + 1;
 297         indirect[i].next = VQ_RING_DESC_CHAIN_END;
 298 }
 299
 300 int
 301 virtqueue_reinit(struct virtqueue *vq, uint16_t size)
 302 {
 303         struct vq_desc_extra *dxp;
 304         int i;
 305
 306         if (vq->vq_nentries != size) {
 307                 device_printf(vq->vq_dev,
 308                     "%s: '%s' changed size; old=%hu, new=%hu\n",
 309                     __func__, vq->vq_name, vq->vq_nentries, size);
 310                 return (EINVAL);
 311         }
 312
 313         /* Warn if the virtqueue was not properly cleaned up. */
 314         if (vq->vq_free_cnt != vq->vq_nentries) {
 315                 device_printf(vq->vq_dev,
 316                     "%s: warning '%s' virtqueue not empty, "
 317                     "leaking %d entries\n", __func__, vq->vq_name,
 318                     vq->vq_nentries - vq->vq_free_cnt);
 319         }
 320
 321         vq->vq_desc_head_idx = 0;
 322         vq->vq_used_cons_idx = 0;
 323         vq->vq_queued_cnt = 0;
 324         vq->vq_free_cnt = vq->vq_nentries;
 325
 326         /* To be safe, reset all our allocated memory. */
 327         bzero(vq->vq_ring_mem, vq->vq_ring_size);
 328         for (i = 0; i < vq->vq_nentries; i++) {
 329                 dxp = &vq->vq_descx[i];
 330                 dxp->cookie = NULL;
 331                 dxp->ndescs = 0;
 332                 if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT)
 333                         virtqueue_init_indirect_list(vq, dxp->indirect);
 334         }
 335
 336         vq_ring_init(vq);
 337         virtqueue_disable_intr(vq);
 338
 339         return (0);
 340 }
 341
 342 void
 343 virtqueue_free(struct virtqueue *vq)
 344 {
 345
 346         if (vq->vq_free_cnt != vq->vq_nentries) {
 347                 device_printf(vq->vq_dev, "%s: freeing non-empty virtqueue, "
 348                     "leaking %d entries\n", vq->vq_name,
 349                     vq->vq_nentries - vq->vq_free_cnt);
 350         }
 351
 352         if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT)
 353                 virtqueue_free_indirect(vq);
 354
 355         if (vq->vq_ring_mem != NULL) {
 356                 contigfree(vq->vq_ring_mem, vq->vq_ring_size, M_DEVBUF);
 357                 vq->vq_ring_size = 0;
 358                 vq->vq_ring_mem = NULL;
 359         }
 360
 361         free(vq, M_DEVBUF);
 362 }
 363
 364 vm_paddr_t
 365 virtqueue_paddr(struct virtqueue *vq)
 366 {
 367
 368         return (vtophys(vq->vq_ring_mem));
 369 }
 370
 371 int
 372 virtqueue_size(struct virtqueue *vq)
 373 {
 374
 375         return (vq->vq_nentries);
 376 }
 377
 378 int
 379 virtqueue_empty(struct virtqueue *vq)
 380 {
 381
 382         return (vq->vq_nentries == vq->vq_free_cnt);
 383 }
 384
 385 int
 386 virtqueue_full(struct virtqueue *vq)
 387 {
 388
 389         return (vq->vq_free_cnt == 0);
 390 }
 391
 392 void
 393 virtqueue_notify(struct virtqueue *vq)
 394 {
 395
 396         /* Ensure updated avail->idx is visible to host. */
 397         mb();
 398
 399         if (vq_ring_must_notify_host(vq))
 400                 vq_ring_notify_host(vq);
 401         vq->vq_queued_cnt = 0;
 402 }
 403
 404 int
 405 virtqueue_nused(struct virtqueue *vq)
 406 {
 407         uint16_t used_idx, nused;
 408
 409         used_idx = vq->vq_ring.used->idx;
 410
 411         nused = (uint16_t)(used_idx - vq->vq_used_cons_idx);
 412         VQASSERT(vq, nused <= vq->vq_nentries, "used more than available");
 413
 414         return (nused);
 415 }
 416
 417 int
 418 virtqueue_intr_filter(struct virtqueue *vq)
 419 {
 420
 421         if (vq->vq_used_cons_idx == vq->vq_ring.used->idx)
 422                 return (0);
 423
 424         virtqueue_disable_intr(vq);
 425
 426         return (1);
 427 }
 428
 429 void
 430 virtqueue_intr(struct virtqueue *vq)
 431 {
 432
 433         vq->vq_intrhand(vq->vq_intrhand_arg);
 434 }
 435
 436 int
 437 virtqueue_enable_intr(struct virtqueue *vq)
 438 {
 439
 440         return (vq_ring_enable_interrupt(vq, 0));
 441 }
 442
 443 int
 444 virtqueue_postpone_intr(struct virtqueue *vq, vq_postpone_t hint)
 445 {
 446         uint16_t ndesc, avail_idx;
 447
 448         avail_idx = vq->vq_ring.avail->idx;
 449         ndesc = (uint16_t)(avail_idx - vq->vq_used_cons_idx);
 450
 451         switch (hint) {
 452         case VQ_POSTPONE_SHORT:
 453                 ndesc = ndesc / 4;
 454                 break;
 455         case VQ_POSTPONE_LONG:
 456                 ndesc = (ndesc * 3) / 4;
 457                 break;
 458         case VQ_POSTPONE_EMPTIED:
 459                 break;
 460         }
 461
 462         return (vq_ring_enable_interrupt(vq, ndesc));
 463 }
 464
 465 /*
 466  * Note this is only considered a hint to the host.
 467  */
 468 void
 469 virtqueue_disable_intr(struct virtqueue *vq)
 470 {
 471
 472         if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
 473                 vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx -
 474                     vq->vq_nentries - 1;
 475         } else
 476                 vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
 477 }
 478
 479 int
 480 virtqueue_enqueue(struct virtqueue *vq, void *cookie, struct sglist *sg,
 481     int readable, int writable)
 482 {
 483         struct vq_desc_extra *dxp;
 484         int needed;
 485         uint16_t head_idx, idx;
 486
 487         needed = readable + writable;
 488
 489         VQASSERT(vq, cookie != NULL, "enqueuing with no cookie");
 490         VQASSERT(vq, needed == sg->sg_nseg,
 491             "segment count mismatch, %d, %d", needed, sg->sg_nseg);
 492         VQASSERT(vq,
 493             needed <= vq->vq_nentries || needed <= vq->vq_max_indirect_size,
 494             "too many segments to enqueue: %d, %d/%d", needed,
 495             vq->vq_nentries, vq->vq_max_indirect_size);
 496
 497         if (needed < 1)
 498                 return (EINVAL);
 499         if (vq->vq_free_cnt == 0)
 500                 return (ENOSPC);
 501
 502         if (vq_ring_use_indirect(vq, needed)) {
 503                 vq_ring_enqueue_indirect(vq, cookie, sg, readable, writable);
 504                 return (0);
 505         } else if (vq->vq_free_cnt < needed)
 506                 return (EMSGSIZE);
 507
 508         head_idx = vq->vq_desc_head_idx;
 509         VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
 510         dxp = &vq->vq_descx[head_idx];
 511
 512         VQASSERT(vq, dxp->cookie == NULL,
 513             "cookie already exists for index %d", head_idx);
 514         dxp->cookie = cookie;
 515         dxp->ndescs = needed;
 516
 517         idx = vq_ring_enqueue_segments(vq, vq->vq_ring.desc, head_idx,
 518             sg, readable, writable);
 519
 520         vq->vq_desc_head_idx = idx;
 521         vq->vq_free_cnt -= needed;
 522         if (vq->vq_free_cnt == 0)
 523                 VQ_RING_ASSERT_CHAIN_TERM(vq);
 524         else
 525                 VQ_RING_ASSERT_VALID_IDX(vq, idx);
 526
 527         vq_ring_update_avail(vq, head_idx);
 528
 529         return (0);
 530 }
 531
 532 void *
 533 virtqueue_dequeue(struct virtqueue *vq, uint32_t *len)
 534 {
 535         struct vring_used_elem *uep;
 536         void *cookie;
 537         uint16_t used_idx, desc_idx;
 538
 539         if (vq->vq_used_cons_idx == vq->vq_ring.used->idx)
 540                 return (NULL);
 541
 542         used_idx = vq->vq_used_cons_idx++ & (vq->vq_nentries - 1);
 543         uep = &vq->vq_ring.used->ring[used_idx];
 544
 545         rmb();
 546         desc_idx = (uint16_t) uep->id;
 547         if (len != NULL)
 548                 *len = uep->len;
 549
 550         vq_ring_free_chain(vq, desc_idx);
 551
 552         cookie = vq->vq_descx[desc_idx].cookie;
 553         VQASSERT(vq, cookie != NULL, "no cookie for index %d", desc_idx);
 554         vq->vq_descx[desc_idx].cookie = NULL;
 555
 556         return (cookie);
 557 }
 558
 559 void *
 560 virtqueue_poll(struct virtqueue *vq, uint32_t *len)
 561 {
 562         void *cookie;
 563
 564         while ((cookie = virtqueue_dequeue(vq, len)) == NULL)
 565                 cpu_spinwait();
 566
 567         return (cookie);
 568 }
 569
 570 void *
 571 virtqueue_drain(struct virtqueue *vq, int *last)
 572 {
 573         void *cookie;
 574         int idx;
 575
 576         cookie = NULL;
 577         idx = *last;
 578
 579         while (idx < vq->vq_nentries && cookie == NULL) {
 580                 if ((cookie = vq->vq_descx[idx].cookie) != NULL) {
 581                         vq->vq_descx[idx].cookie = NULL;
 582                         /* Free chain to keep free count consistent. */
 583                         vq_ring_free_chain(vq, idx);
 584                 }
 585                 idx++;
 586         }
 587
 588         *last = idx;
 589
 590         return (cookie);
 591 }
 592
 593 void
 594 virtqueue_dump(struct virtqueue *vq)
 595 {
 596
 597         if (vq == NULL)
 598                 return;
 599
 600         printf("VQ: %s - size=%d; free=%d; used=%d; queued=%d; "
 601             "desc_head_idx=%d; avail.idx=%d; used_cons_idx=%d; "
 602             "used.idx=%d; avail.flags=0x%x; used.flags=0x%x\n",
 603             vq->vq_name, vq->vq_nentries, vq->vq_free_cnt,
 604             virtqueue_nused(vq), vq->vq_queued_cnt, vq->vq_desc_head_idx,
 605             vq->vq_ring.avail->idx, vq->vq_used_cons_idx,
 606             vq->vq_ring.used->idx, vq->vq_ring.avail->flags,
 607             vq->vq_ring.used->flags);
 608 }
 609
 610 static void
 611 vq_ring_init(struct virtqueue *vq)
 612 {
 613         struct vring *vr;
 614         char *ring_mem;
 615         int i, size;
 616
 617         ring_mem = vq->vq_ring_mem;
 618         size = vq->vq_nentries;
 619         vr = &vq->vq_ring;
 620
 621         vring_init(vr, size, ring_mem, vq->vq_alignment);
 622
 623         for (i = 0; i < size - 1; i++)
 624                 vr->desc[i].next = i + 1;
 625         vr->desc[i].next = VQ_RING_DESC_CHAIN_END;
 626 }
 627
 628 static void
 629 vq_ring_update_avail(struct virtqueue *vq, uint16_t desc_idx)
 630 {
 631         uint16_t avail_idx;
 632
 633         /*
 634          * Place the head of the descriptor chain into the next slot and make
 635          * it usable to the host. The chain is made available now rather than
 636          * deferring to virtqueue_notify() in the hopes that if the host is
 637          * currently running on another CPU, we can keep it processing the new
 638          * descriptor.
 639          */
 640         avail_idx = vq->vq_ring.avail->idx & (vq->vq_nentries - 1);
 641         vq->vq_ring.avail->ring[avail_idx] = desc_idx;
 642
 643         wmb();
 644         vq->vq_ring.avail->idx++;
 645
 646         /* Keep pending count until virtqueue_notify(). */
 647         vq->vq_queued_cnt++;
 648 }
 649
 650 static uint16_t
 651 vq_ring_enqueue_segments(struct virtqueue *vq, struct vring_desc *desc,
 652     uint16_t head_idx, struct sglist *sg, int readable, int writable)
 653 {
 654         struct sglist_seg *seg;
 655         struct vring_desc *dp;
 656         int i, needed;
 657         uint16_t idx;
 658
 659         needed = readable + writable;
 660
 661         for (i = 0, idx = head_idx, seg = sg->sg_segs;
 662              i < needed;
 663              i++, idx = dp->next, seg++) {
 664                 VQASSERT(vq, idx != VQ_RING_DESC_CHAIN_END,
 665                     "premature end of free desc chain");
 666
 667                 dp = &desc[idx];
 668                 dp->addr = seg->ss_paddr;
 669                 dp->len = seg->ss_len;
 670                 dp->flags = 0;
 671
 672                 if (i < needed - 1)
 673                         dp->flags |= VRING_DESC_F_NEXT;
 674                 if (i >= readable)
 675                         dp->flags |= VRING_DESC_F_WRITE;
 676         }
 677
 678         return (idx);
 679 }
 680
 681 static int
 682 vq_ring_use_indirect(struct virtqueue *vq, int needed)
 683 {
 684
 685         if ((vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) == 0)
 686                 return (0);
 687
 688         if (vq->vq_max_indirect_size < needed)
 689                 return (0);
 690
 691         if (needed < 2)
 692                 return (0);
 693
 694         return (1);
 695 }
 696
 697 static void
 698 vq_ring_enqueue_indirect(struct virtqueue *vq, void *cookie,
 699     struct sglist *sg, int readable, int writable)
 700 {
 701         struct vring_desc *dp;
 702         struct vq_desc_extra *dxp;
 703         int needed;
 704         uint16_t head_idx;
 705
 706         needed = readable + writable;
 707         VQASSERT(vq, needed <= vq->vq_max_indirect_size,
 708             "enqueuing too many indirect descriptors");
 709
 710         head_idx = vq->vq_desc_head_idx;
 711         VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
 712         dp = &vq->vq_ring.desc[head_idx];
 713         dxp = &vq->vq_descx[head_idx];
 714
 715         VQASSERT(vq, dxp->cookie == NULL,
 716             "cookie already exists for index %d", head_idx);
 717         dxp->cookie = cookie;
 718         dxp->ndescs = 1;
 719
 720         dp->addr = dxp->indirect_paddr;
 721         dp->len = needed * sizeof(struct vring_desc);
 722         dp->flags = VRING_DESC_F_INDIRECT;
 723
 724         vq_ring_enqueue_segments(vq, dxp->indirect, 0,
 725             sg, readable, writable);
 726
 727         vq->vq_desc_head_idx = dp->next;
 728         vq->vq_free_cnt--;
 729         if (vq->vq_free_cnt == 0)
 730                 VQ_RING_ASSERT_CHAIN_TERM(vq);
 731         else
 732                 VQ_RING_ASSERT_VALID_IDX(vq, vq->vq_desc_head_idx);
 733
 734         vq_ring_update_avail(vq, head_idx);
 735 }
 736
 737 static int
 738 vq_ring_enable_interrupt(struct virtqueue *vq, uint16_t ndesc)
 739 {
 740
 741         /*
 742          * Enable interrupts, making sure we get the latest index of
 743          * what's already been consumed.
 744          */
 745         if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX)
 746                 vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx + ndesc;
 747         else
 748                 vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
 749
 750         mb();
 751
 752         /*
 753          * Enough items may have already been consumed to meet our threshold
 754          * since we last checked. Let our caller know so it processes the new
 755          * entries.
 756          */
 757         if (virtqueue_nused(vq) > ndesc)
 758                 return (1);
 759
 760         return (0);
 761 }
 762
 763 static int
 764 vq_ring_must_notify_host(struct virtqueue *vq)
 765 {
 766         uint16_t new_idx, prev_idx, event_idx;
 767
 768         if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
 769                 new_idx = vq->vq_ring.avail->idx;
 770                 prev_idx = new_idx - vq->vq_queued_cnt;
 771                 event_idx = vring_avail_event(&vq->vq_ring);
 772
 773                 return (vring_need_event(event_idx, new_idx, prev_idx) != 0);
 774         }
 775
 776         return ((vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY) == 0);
 777 }
 778
 779 static void
 780 vq_ring_notify_host(struct virtqueue *vq)
 781 {
 782
 783         VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index);
 784 }
 785
 786 static void
 787 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
 788 {
 789         struct vring_desc *dp;
 790         struct vq_desc_extra *dxp;
 791
 792         VQ_RING_ASSERT_VALID_IDX(vq, desc_idx);
 793         dp = &vq->vq_ring.desc[desc_idx];
 794         dxp = &vq->vq_descx[desc_idx];
 795
 796         if (vq->vq_free_cnt == 0)
 797                 VQ_RING_ASSERT_CHAIN_TERM(vq);
 798
 799         vq->vq_free_cnt += dxp->ndescs;
 800         dxp->ndescs--;
 801
 802         if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
 803                 while (dp->flags & VRING_DESC_F_NEXT) {
 804                         VQ_RING_ASSERT_VALID_IDX(vq, dp->next);
 805                         dp = &vq->vq_ring.desc[dp->next];
 806                         dxp->ndescs--;
 807                 }
 808         }
 809
 810         VQASSERT(vq, dxp->ndescs == 0,
 811             "failed to free entire desc chain, remaining: %d", dxp->ndescs);
 812
 813         /*
 814          * We must append the existing free chain, if any, to the end of
 815          * newly freed chain. If the virtqueue was completely used, then
 816          * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
 817          */
 818         dp->next = vq->vq_desc_head_idx;
 819         vq->vq_desc_head_idx = desc_idx;
 820 }