sys/dev/virtio/balloon/virtio_balloon.c

   1 /*-
   2  * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice unmodified, this list of conditions, and the following
  10  *    disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 /* Driver for VirtIO memory balloon devices. */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include <sys/param.h>
  33 #include <sys/systm.h>
  34 #include <sys/kernel.h>
  35 #include <sys/endian.h>
  36 #include <sys/kthread.h>
  37 #include <sys/malloc.h>
  38 #include <sys/module.h>
  39 #include <sys/sglist.h>
  40 #include <sys/sysctl.h>
  41 #include <sys/lock.h>
  42 #include <sys/mutex.h>
  43 #include <sys/queue.h>
  44
  45 #include <vm/vm.h>
  46 #include <vm/vm_page.h>
  47
  48 #include <machine/bus.h>
  49 #include <machine/resource.h>
  50 #include <sys/bus.h>
  51 #include <sys/rman.h>
  52
  53 #include <dev/virtio/virtio.h>
  54 #include <dev/virtio/virtqueue.h>
  55 #include <dev/virtio/balloon/virtio_balloon.h>
  56
  57 #include "virtio_if.h"
  58
  59 struct vtballoon_softc {
  60         device_t                 vtballoon_dev;
  61         struct mtx               vtballoon_mtx;
  62         uint64_t                 vtballoon_features;
  63         uint32_t                 vtballoon_flags;
  64 #define VTBALLOON_FLAG_DETACH    0x01
  65
  66         struct virtqueue        *vtballoon_inflate_vq;
  67         struct virtqueue        *vtballoon_deflate_vq;
  68
  69         uint32_t                 vtballoon_desired_npages;
  70         uint32_t                 vtballoon_current_npages;
  71         TAILQ_HEAD(,vm_page)     vtballoon_pages;
  72
  73         struct thread           *vtballoon_td;
  74         uint32_t                *vtballoon_page_frames;
  75         int                      vtballoon_timeout;
  76 };
  77
  78 static struct virtio_feature_desc vtballoon_feature_desc[] = {
  79         { VIRTIO_BALLOON_F_MUST_TELL_HOST,      "MustTellHost"  },
  80         { VIRTIO_BALLOON_F_STATS_VQ,            "StatsVq"       },
  81
  82         { 0, NULL }
  83 };
  84
  85 static int      vtballoon_probe(device_t);
  86 static int      vtballoon_attach(device_t);
  87 static int      vtballoon_detach(device_t);
  88 static int      vtballoon_config_change(device_t);
  89
  90 static void     vtballoon_negotiate_features(struct vtballoon_softc *);
  91 static int      vtballoon_alloc_virtqueues(struct vtballoon_softc *);
  92
  93 static void     vtballoon_vq_intr(void *);
  94
  95 static void     vtballoon_inflate(struct vtballoon_softc *, int);
  96 static void     vtballoon_deflate(struct vtballoon_softc *, int);
  97
  98 static void     vtballoon_send_page_frames(struct vtballoon_softc *,
  99                     struct virtqueue *, int);
 100
 101 static void     vtballoon_pop(struct vtballoon_softc *);
 102 static void     vtballoon_stop(struct vtballoon_softc *);
 103
 104 static vm_page_t
 105                 vtballoon_alloc_page(struct vtballoon_softc *);
 106 static void     vtballoon_free_page(struct vtballoon_softc *, vm_page_t);
 107
 108 static int      vtballoon_sleep(struct vtballoon_softc *);
 109 static void     vtballoon_thread(void *);
 110 static void     vtballoon_add_sysctl(struct vtballoon_softc *);
 111
 112 /* Features desired/implemented by this driver. */
 113 #define VTBALLOON_FEATURES              0
 114
 115 /* Timeout between retries when the balloon needs inflating. */
 116 #define VTBALLOON_LOWMEM_TIMEOUT        hz
 117
 118 /*
 119  * Maximum number of pages we'll request to inflate or deflate
 120  * the balloon in one virtqueue request. Both Linux and NetBSD
 121  * have settled on 256, doing up to 1MB at a time.
 122  */
 123 #define VTBALLOON_PAGES_PER_REQUEST     256
 124
 125 /* Must be able to fix all pages frames in one page (segment). */
 126 CTASSERT(VTBALLOON_PAGES_PER_REQUEST * sizeof(uint32_t) <= PAGE_SIZE);
 127
 128 #define VTBALLOON_MTX(_sc)              &(_sc)->vtballoon_mtx
 129 #define VTBALLOON_LOCK_INIT(_sc, _name) mtx_init(VTBALLOON_MTX((_sc)), _name, \
 130                                             "VirtIO Balloon Lock", MTX_DEF)
 131 #define VTBALLOON_LOCK(_sc)             mtx_lock(VTBALLOON_MTX((_sc)))
 132 #define VTBALLOON_UNLOCK(_sc)           mtx_unlock(VTBALLOON_MTX((_sc)))
 133 #define VTBALLOON_LOCK_DESTROY(_sc)     mtx_destroy(VTBALLOON_MTX((_sc)))
 134
 135 static device_method_t vtballoon_methods[] = {
 136         /* Device methods. */
 137         DEVMETHOD(device_probe,         vtballoon_probe),
 138         DEVMETHOD(device_attach,        vtballoon_attach),
 139         DEVMETHOD(device_detach,        vtballoon_detach),
 140
 141         /* VirtIO methods. */
 142         DEVMETHOD(virtio_config_change, vtballoon_config_change),
 143
 144         DEVMETHOD_END
 145 };
 146
 147 static driver_t vtballoon_driver = {
 148         "vtballoon",
 149         vtballoon_methods,
 150         sizeof(struct vtballoon_softc)
 151 };
 152 static devclass_t vtballoon_devclass;
 153
 154 DRIVER_MODULE(virtio_balloon, virtio_pci, vtballoon_driver,
 155     vtballoon_devclass, 0, 0);
 156 MODULE_VERSION(virtio_balloon, 1);
 157 MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1);
 158
 159 static int
 160 vtballoon_probe(device_t dev)
 161 {
 162
 163         if (virtio_get_device_type(dev) != VIRTIO_ID_BALLOON)
 164                 return (ENXIO);
 165
 166         device_set_desc(dev, "VirtIO Balloon Adapter");
 167
 168         return (BUS_PROBE_DEFAULT);
 169 }
 170
 171 static int
 172 vtballoon_attach(device_t dev)
 173 {
 174         struct vtballoon_softc *sc;
 175         int error;
 176
 177         sc = device_get_softc(dev);
 178         sc->vtballoon_dev = dev;
 179
 180         VTBALLOON_LOCK_INIT(sc, device_get_nameunit(dev));
 181         TAILQ_INIT(&sc->vtballoon_pages);
 182
 183         vtballoon_add_sysctl(sc);
 184
 185         virtio_set_feature_desc(dev, vtballoon_feature_desc);
 186         vtballoon_negotiate_features(sc);
 187
 188         sc->vtballoon_page_frames = malloc(VTBALLOON_PAGES_PER_REQUEST *
 189             sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO);
 190         if (sc->vtballoon_page_frames == NULL) {
 191                 error = ENOMEM;
 192                 device_printf(dev,
 193                     "cannot allocate page frame request array\n");
 194                 goto fail;
 195         }
 196
 197         error = vtballoon_alloc_virtqueues(sc);
 198         if (error) {
 199                 device_printf(dev, "cannot allocate virtqueues\n");
 200                 goto fail;
 201         }
 202
 203         error = virtio_setup_intr(dev, INTR_TYPE_MISC);
 204         if (error) {
 205                 device_printf(dev, "cannot setup virtqueue interrupts\n");
 206                 goto fail;
 207         }
 208
 209         error = kthread_add(vtballoon_thread, sc, NULL, &sc->vtballoon_td,
 210             0, 0, "virtio_balloon");
 211         if (error) {
 212                 device_printf(dev, "cannot create balloon kthread\n");
 213                 goto fail;
 214         }
 215
 216         virtqueue_enable_intr(sc->vtballoon_inflate_vq);
 217         virtqueue_enable_intr(sc->vtballoon_deflate_vq);
 218
 219 fail:
 220         if (error)
 221                 vtballoon_detach(dev);
 222
 223         return (error);
 224 }
 225
 226 static int
 227 vtballoon_detach(device_t dev)
 228 {
 229         struct vtballoon_softc *sc;
 230
 231         sc = device_get_softc(dev);
 232
 233         if (sc->vtballoon_td != NULL) {
 234                 VTBALLOON_LOCK(sc);
 235                 sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH;
 236                 wakeup_one(sc);
 237                 msleep(sc->vtballoon_td, VTBALLOON_MTX(sc), 0, "vtbdth", 0);
 238                 VTBALLOON_UNLOCK(sc);
 239
 240                 sc->vtballoon_td = NULL;
 241         }
 242
 243         if (device_is_attached(dev)) {
 244                 vtballoon_pop(sc);
 245                 vtballoon_stop(sc);
 246         }
 247
 248         if (sc->vtballoon_page_frames != NULL) {
 249                 free(sc->vtballoon_page_frames, M_DEVBUF);
 250                 sc->vtballoon_page_frames = NULL;
 251         }
 252
 253         VTBALLOON_LOCK_DESTROY(sc);
 254
 255         return (0);
 256 }
 257
 258 static int
 259 vtballoon_config_change(device_t dev)
 260 {
 261         struct vtballoon_softc *sc;
 262
 263         sc = device_get_softc(dev);
 264
 265         VTBALLOON_LOCK(sc);
 266         wakeup_one(sc);
 267         VTBALLOON_UNLOCK(sc);
 268
 269         return (1);
 270 }
 271
 272 static void
 273 vtballoon_negotiate_features(struct vtballoon_softc *sc)
 274 {
 275         device_t dev;
 276         uint64_t features;
 277
 278         dev = sc->vtballoon_dev;
 279         features = virtio_negotiate_features(dev, VTBALLOON_FEATURES);
 280         sc->vtballoon_features = features;
 281 }
 282
 283 static int
 284 vtballoon_alloc_virtqueues(struct vtballoon_softc *sc)
 285 {
 286         device_t dev;
 287         struct vq_alloc_info vq_info[2];
 288         int nvqs;
 289
 290         dev = sc->vtballoon_dev;
 291         nvqs = 2;
 292
 293         VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtballoon_vq_intr, sc,
 294             &sc->vtballoon_inflate_vq, "%s inflate", device_get_nameunit(dev));
 295
 296         VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtballoon_vq_intr, sc,
 297             &sc->vtballoon_deflate_vq, "%s deflate", device_get_nameunit(dev));
 298
 299         return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info));
 300 }
 301
 302 static void
 303 vtballoon_vq_intr(void *xsc)
 304 {
 305         struct vtballoon_softc *sc;
 306
 307         sc = xsc;
 308
 309         VTBALLOON_LOCK(sc);
 310         wakeup_one(sc);
 311         VTBALLOON_UNLOCK(sc);
 312 }
 313
 314 static void
 315 vtballoon_inflate(struct vtballoon_softc *sc, int npages)
 316 {
 317         struct virtqueue *vq;
 318         vm_page_t m;
 319         int i;
 320
 321         vq = sc->vtballoon_inflate_vq;
 322
 323         if (npages > VTBALLOON_PAGES_PER_REQUEST)
 324                 npages = VTBALLOON_PAGES_PER_REQUEST;
 325
 326         for (i = 0; i < npages; i++) {
 327                 if ((m = vtballoon_alloc_page(sc)) == NULL) {
 328                         sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT;
 329                         break;
 330                 }
 331
 332                 sc->vtballoon_page_frames[i] =
 333                     VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
 334
 335                 KASSERT(m->queue == PQ_NONE,
 336                     ("%s: allocated page %p on queue", __func__, m));
 337                 TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, plinks.q);
 338         }
 339
 340         if (i > 0)
 341                 vtballoon_send_page_frames(sc, vq, i);
 342 }
 343
 344 static void
 345 vtballoon_deflate(struct vtballoon_softc *sc, int npages)
 346 {
 347         TAILQ_HEAD(, vm_page) free_pages;
 348         struct virtqueue *vq;
 349         vm_page_t m;
 350         int i;
 351
 352         vq = sc->vtballoon_deflate_vq;
 353         TAILQ_INIT(&free_pages);
 354
 355         if (npages > VTBALLOON_PAGES_PER_REQUEST)
 356                 npages = VTBALLOON_PAGES_PER_REQUEST;
 357
 358         for (i = 0; i < npages; i++) {
 359                 m = TAILQ_FIRST(&sc->vtballoon_pages);
 360                 KASSERT(m != NULL, ("%s: no more pages to deflate", __func__));
 361
 362                 sc->vtballoon_page_frames[i] =
 363                     VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
 364
 365                 TAILQ_REMOVE(&sc->vtballoon_pages, m, plinks.q);
 366                 TAILQ_INSERT_TAIL(&free_pages, m, plinks.q);
 367         }
 368
 369         if (i > 0) {
 370                 /* Always tell host first before freeing the pages. */
 371                 vtballoon_send_page_frames(sc, vq, i);
 372
 373                 while ((m = TAILQ_FIRST(&free_pages)) != NULL) {
 374                         TAILQ_REMOVE(&free_pages, m, plinks.q);
 375                         vtballoon_free_page(sc, m);
 376                 }
 377         }
 378
 379         KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) &&
 380             sc->vtballoon_current_npages == 0) ||
 381             (!TAILQ_EMPTY(&sc->vtballoon_pages) &&
 382             sc->vtballoon_current_npages != 0),
 383             ("%s: bogus page count %d", __func__,
 384             sc->vtballoon_current_npages));
 385 }
 386
 387 static void
 388 vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq,
 389     int npages)
 390 {
 391         struct sglist sg;
 392         struct sglist_seg segs[1];
 393         void *c;
 394         int error;
 395
 396         sglist_init(&sg, 1, segs);
 397
 398         error = sglist_append(&sg, sc->vtballoon_page_frames,
 399             npages * sizeof(uint32_t));
 400         KASSERT(error == 0, ("error adding page frames to sglist"));
 401
 402         error = virtqueue_enqueue(vq, vq, &sg, 1, 0);
 403         KASSERT(error == 0, ("error enqueuing page frames to virtqueue"));
 404         virtqueue_notify(vq);
 405
 406         /*
 407          * Inflate and deflate operations are done synchronously. The
 408          * interrupt handler will wake us up.
 409          */
 410         VTBALLOON_LOCK(sc);
 411         while ((c = virtqueue_dequeue(vq, NULL)) == NULL)
 412                 msleep(sc, VTBALLOON_MTX(sc), 0, "vtbspf", 0);
 413         VTBALLOON_UNLOCK(sc);
 414
 415         KASSERT(c == vq, ("unexpected balloon operation response"));
 416 }
 417
 418 static void
 419 vtballoon_pop(struct vtballoon_softc *sc)
 420 {
 421
 422         while (!TAILQ_EMPTY(&sc->vtballoon_pages))
 423                 vtballoon_deflate(sc, sc->vtballoon_current_npages);
 424 }
 425
 426 static void
 427 vtballoon_stop(struct vtballoon_softc *sc)
 428 {
 429
 430         virtqueue_disable_intr(sc->vtballoon_inflate_vq);
 431         virtqueue_disable_intr(sc->vtballoon_deflate_vq);
 432
 433         virtio_stop(sc->vtballoon_dev);
 434 }
 435
 436 static vm_page_t
 437 vtballoon_alloc_page(struct vtballoon_softc *sc)
 438 {
 439         vm_page_t m;
 440
 441         m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_WIRED |
 442             VM_ALLOC_NOOBJ);
 443         if (m != NULL)
 444                 sc->vtballoon_current_npages++;
 445
 446         return (m);
 447 }
 448
 449 static void
 450 vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m)
 451 {
 452
 453         vm_page_unwire(m, 0);
 454         vm_page_free(m);
 455         sc->vtballoon_current_npages--;
 456 }
 457
 458 static uint32_t
 459 vtballoon_desired_size(struct vtballoon_softc *sc)
 460 {
 461         uint32_t desired;
 462
 463         desired = virtio_read_dev_config_4(sc->vtballoon_dev,
 464             offsetof(struct virtio_balloon_config, num_pages));
 465
 466         return (le32toh(desired));
 467 }
 468
 469 static void
 470 vtballoon_update_size(struct vtballoon_softc *sc)
 471 {
 472
 473         virtio_write_dev_config_4(sc->vtballoon_dev,
 474             offsetof(struct virtio_balloon_config, actual),
 475             htole32(sc->vtballoon_current_npages));
 476 }
 477
 478 static int
 479 vtballoon_sleep(struct vtballoon_softc *sc)
 480 {
 481         int rc, timeout;
 482         uint32_t current, desired;
 483
 484         rc = 0;
 485         current = sc->vtballoon_current_npages;
 486
 487         VTBALLOON_LOCK(sc);
 488         for (;;) {
 489                 if (sc->vtballoon_flags & VTBALLOON_FLAG_DETACH) {
 490                         rc = 1;
 491                         break;
 492                 }
 493
 494                 desired = vtballoon_desired_size(sc);
 495                 sc->vtballoon_desired_npages = desired;
 496
 497                 /*
 498                  * If given, use non-zero timeout on the first time through
 499                  * the loop. On subsequent times, timeout will be zero so
 500                  * we will reevaluate the desired size of the balloon and
 501                  * break out to retry if needed.
 502                  */
 503                 timeout = sc->vtballoon_timeout;
 504                 sc->vtballoon_timeout = 0;
 505
 506                 if (current > desired)
 507                         break;
 508                 if (current < desired && timeout == 0)
 509                         break;
 510
 511                 msleep(sc, VTBALLOON_MTX(sc), 0, "vtbslp", timeout);
 512         }
 513         VTBALLOON_UNLOCK(sc);
 514
 515         return (rc);
 516 }
 517
 518 static void
 519 vtballoon_thread(void *xsc)
 520 {
 521         struct vtballoon_softc *sc;
 522         uint32_t current, desired;
 523
 524         sc = xsc;
 525
 526         for (;;) {
 527                 if (vtballoon_sleep(sc) != 0)
 528                         break;
 529
 530                 current = sc->vtballoon_current_npages;
 531                 desired = sc->vtballoon_desired_npages;
 532
 533                 if (desired != current) {
 534                         if (desired > current)
 535                                 vtballoon_inflate(sc, desired - current);
 536                         else
 537                                 vtballoon_deflate(sc, current - desired);
 538
 539                         vtballoon_update_size(sc);
 540                 }
 541         }
 542
 543         kthread_exit();
 544 }
 545
 546 static void
 547 vtballoon_add_sysctl(struct vtballoon_softc *sc)
 548 {
 549         device_t dev;
 550         struct sysctl_ctx_list *ctx;
 551         struct sysctl_oid *tree;
 552         struct sysctl_oid_list *child;
 553
 554         dev = sc->vtballoon_dev;
 555         ctx = device_get_sysctl_ctx(dev);
 556         tree = device_get_sysctl_tree(dev);
 557         child = SYSCTL_CHILDREN(tree);
 558
 559         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "desired",
 560             CTLFLAG_RD, &sc->vtballoon_desired_npages, sizeof(uint32_t),
 561             "Desired balloon size in pages");
 562
 563         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "current",
 564             CTLFLAG_RD, &sc->vtballoon_current_npages, sizeof(uint32_t),
 565             "Current balloon size in pages");
 566 }