sys/dev/virtio/balloon/virtio_balloon.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice unmodified, this list of conditions, and the following
  12  *    disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  */
  28
  29 /* Driver for VirtIO memory balloon devices. */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33
  34 #include <sys/param.h>
  35 #include <sys/systm.h>
  36 #include <sys/kernel.h>
  37 #include <sys/endian.h>
  38 #include <sys/kthread.h>
  39 #include <sys/malloc.h>
  40 #include <sys/module.h>
  41 #include <sys/sglist.h>
  42 #include <sys/sysctl.h>
  43 #include <sys/lock.h>
  44 #include <sys/mutex.h>
  45 #include <sys/queue.h>
  46
  47 #include <vm/vm.h>
  48 #include <vm/vm_page.h>
  49
  50 #include <machine/bus.h>
  51 #include <machine/resource.h>
  52 #include <sys/bus.h>
  53 #include <sys/rman.h>
  54
  55 #include <dev/virtio/virtio.h>
  56 #include <dev/virtio/virtqueue.h>
  57 #include <dev/virtio/balloon/virtio_balloon.h>
  58
  59 #include "virtio_if.h"
  60
  61 struct vtballoon_softc {
  62         device_t                 vtballoon_dev;
  63         struct mtx               vtballoon_mtx;
  64         uint64_t                 vtballoon_features;
  65         uint32_t                 vtballoon_flags;
  66 #define VTBALLOON_FLAG_DETACH    0x01
  67
  68         struct virtqueue        *vtballoon_inflate_vq;
  69         struct virtqueue        *vtballoon_deflate_vq;
  70
  71         uint32_t                 vtballoon_desired_npages;
  72         uint32_t                 vtballoon_current_npages;
  73         TAILQ_HEAD(,vm_page)     vtballoon_pages;
  74
  75         struct thread           *vtballoon_td;
  76         uint32_t                *vtballoon_page_frames;
  77         int                      vtballoon_timeout;
  78 };
  79
  80 static struct virtio_feature_desc vtballoon_feature_desc[] = {
  81         { VIRTIO_BALLOON_F_MUST_TELL_HOST,      "MustTellHost"  },
  82         { VIRTIO_BALLOON_F_STATS_VQ,            "StatsVq"       },
  83         { VIRTIO_BALLOON_F_DEFLATE_ON_OOM,      "DeflateOnOOM"  },
  84
  85         { 0, NULL }
  86 };
  87
  88 static int      vtballoon_probe(device_t);
  89 static int      vtballoon_attach(device_t);
  90 static int      vtballoon_detach(device_t);
  91 static int      vtballoon_config_change(device_t);
  92
  93 static int      vtballoon_negotiate_features(struct vtballoon_softc *);
  94 static int      vtballoon_setup_features(struct vtballoon_softc *);
  95 static int      vtballoon_alloc_virtqueues(struct vtballoon_softc *);
  96
  97 static void     vtballoon_vq_intr(void *);
  98
  99 static void     vtballoon_inflate(struct vtballoon_softc *, int);
 100 static void     vtballoon_deflate(struct vtballoon_softc *, int);
 101
 102 static void     vtballoon_send_page_frames(struct vtballoon_softc *,
 103                     struct virtqueue *, int);
 104
 105 static void     vtballoon_pop(struct vtballoon_softc *);
 106 static void     vtballoon_stop(struct vtballoon_softc *);
 107
 108 static vm_page_t
 109                 vtballoon_alloc_page(struct vtballoon_softc *);
 110 static void     vtballoon_free_page(struct vtballoon_softc *, vm_page_t);
 111
 112 static int      vtballoon_sleep(struct vtballoon_softc *);
 113 static void     vtballoon_thread(void *);
 114 static void     vtballoon_setup_sysctl(struct vtballoon_softc *);
 115
 116 #define vtballoon_modern(_sc) \
 117     (((_sc)->vtballoon_features & VIRTIO_F_VERSION_1) != 0)
 118
 119 /* Features desired/implemented by this driver. */
 120 #define VTBALLOON_FEATURES              VIRTIO_BALLOON_F_MUST_TELL_HOST
 121
 122 /* Timeout between retries when the balloon needs inflating. */
 123 #define VTBALLOON_LOWMEM_TIMEOUT        hz
 124
 125 /*
 126  * Maximum number of pages we'll request to inflate or deflate
 127  * the balloon in one virtqueue request. Both Linux and NetBSD
 128  * have settled on 256, doing up to 1MB at a time.
 129  */
 130 #define VTBALLOON_PAGES_PER_REQUEST     256
 131
 132 /* Must be able to fix all pages frames in one page (segment). */
 133 CTASSERT(VTBALLOON_PAGES_PER_REQUEST * sizeof(uint32_t) <= PAGE_SIZE);
 134
 135 #define VTBALLOON_MTX(_sc)              &(_sc)->vtballoon_mtx
 136 #define VTBALLOON_LOCK_INIT(_sc, _name) mtx_init(VTBALLOON_MTX((_sc)), _name, \
 137                                             "VirtIO Balloon Lock", MTX_DEF)
 138 #define VTBALLOON_LOCK(_sc)             mtx_lock(VTBALLOON_MTX((_sc)))
 139 #define VTBALLOON_UNLOCK(_sc)           mtx_unlock(VTBALLOON_MTX((_sc)))
 140 #define VTBALLOON_LOCK_DESTROY(_sc)     mtx_destroy(VTBALLOON_MTX((_sc)))
 141
 142 static device_method_t vtballoon_methods[] = {
 143         /* Device methods. */
 144         DEVMETHOD(device_probe,         vtballoon_probe),
 145         DEVMETHOD(device_attach,        vtballoon_attach),
 146         DEVMETHOD(device_detach,        vtballoon_detach),
 147
 148         /* VirtIO methods. */
 149         DEVMETHOD(virtio_config_change, vtballoon_config_change),
 150
 151         DEVMETHOD_END
 152 };
 153
 154 static driver_t vtballoon_driver = {
 155         "vtballoon",
 156         vtballoon_methods,
 157         sizeof(struct vtballoon_softc)
 158 };
 159 static devclass_t vtballoon_devclass;
 160
 161 VIRTIO_DRIVER_MODULE(virtio_balloon, vtballoon_driver,
 162     vtballoon_devclass, 0, 0);
 163 MODULE_VERSION(virtio_balloon, 1);
 164 MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1);
 165
 166 VIRTIO_SIMPLE_PNPINFO(virtio_balloon, VIRTIO_ID_BALLOON,
 167     "VirtIO Balloon Adapter");
 168
 169 static int
 170 vtballoon_probe(device_t dev)
 171 {
 172         return (VIRTIO_SIMPLE_PROBE(dev, virtio_balloon));
 173 }
 174
 175 static int
 176 vtballoon_attach(device_t dev)
 177 {
 178         struct vtballoon_softc *sc;
 179         int error;
 180
 181         sc = device_get_softc(dev);
 182         sc->vtballoon_dev = dev;
 183         virtio_set_feature_desc(dev, vtballoon_feature_desc);
 184
 185         VTBALLOON_LOCK_INIT(sc, device_get_nameunit(dev));
 186         TAILQ_INIT(&sc->vtballoon_pages);
 187
 188         vtballoon_setup_sysctl(sc);
 189
 190         error = vtballoon_setup_features(sc);
 191         if (error) {
 192                 device_printf(dev, "cannot setup features\n");
 193                 goto fail;
 194         }
 195
 196         sc->vtballoon_page_frames = malloc(VTBALLOON_PAGES_PER_REQUEST *
 197             sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO);
 198         if (sc->vtballoon_page_frames == NULL) {
 199                 error = ENOMEM;
 200                 device_printf(dev,
 201                     "cannot allocate page frame request array\n");
 202                 goto fail;
 203         }
 204
 205         error = vtballoon_alloc_virtqueues(sc);
 206         if (error) {
 207                 device_printf(dev, "cannot allocate virtqueues\n");
 208                 goto fail;
 209         }
 210
 211         error = virtio_setup_intr(dev, INTR_TYPE_MISC);
 212         if (error) {
 213                 device_printf(dev, "cannot setup virtqueue interrupts\n");
 214                 goto fail;
 215         }
 216
 217         error = kthread_add(vtballoon_thread, sc, NULL, &sc->vtballoon_td,
 218             0, 0, "virtio_balloon");
 219         if (error) {
 220                 device_printf(dev, "cannot create balloon kthread\n");
 221                 goto fail;
 222         }
 223
 224         virtqueue_enable_intr(sc->vtballoon_inflate_vq);
 225         virtqueue_enable_intr(sc->vtballoon_deflate_vq);
 226
 227 fail:
 228         if (error)
 229                 vtballoon_detach(dev);
 230
 231         return (error);
 232 }
 233
 234 static int
 235 vtballoon_detach(device_t dev)
 236 {
 237         struct vtballoon_softc *sc;
 238
 239         sc = device_get_softc(dev);
 240
 241         if (sc->vtballoon_td != NULL) {
 242                 VTBALLOON_LOCK(sc);
 243                 sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH;
 244                 wakeup_one(sc);
 245                 msleep(sc->vtballoon_td, VTBALLOON_MTX(sc), 0, "vtbdth", 0);
 246                 VTBALLOON_UNLOCK(sc);
 247
 248                 sc->vtballoon_td = NULL;
 249         }
 250
 251         if (device_is_attached(dev)) {
 252                 vtballoon_pop(sc);
 253                 vtballoon_stop(sc);
 254         }
 255
 256         if (sc->vtballoon_page_frames != NULL) {
 257                 free(sc->vtballoon_page_frames, M_DEVBUF);
 258                 sc->vtballoon_page_frames = NULL;
 259         }
 260
 261         VTBALLOON_LOCK_DESTROY(sc);
 262
 263         return (0);
 264 }
 265
 266 static int
 267 vtballoon_config_change(device_t dev)
 268 {
 269         struct vtballoon_softc *sc;
 270
 271         sc = device_get_softc(dev);
 272
 273         VTBALLOON_LOCK(sc);
 274         wakeup_one(sc);
 275         VTBALLOON_UNLOCK(sc);
 276
 277         return (1);
 278 }
 279
 280 static int
 281 vtballoon_negotiate_features(struct vtballoon_softc *sc)
 282 {
 283         device_t dev;
 284         uint64_t features;
 285
 286         dev = sc->vtballoon_dev;
 287         features = VTBALLOON_FEATURES;
 288
 289         sc->vtballoon_features = virtio_negotiate_features(dev, features);
 290         return (virtio_finalize_features(dev));
 291 }
 292
 293 static int
 294 vtballoon_setup_features(struct vtballoon_softc *sc)
 295 {
 296         int error;
 297
 298         error = vtballoon_negotiate_features(sc);
 299         if (error)
 300                 return (error);
 301
 302         return (0);
 303 }
 304
 305 static int
 306 vtballoon_alloc_virtqueues(struct vtballoon_softc *sc)
 307 {
 308         device_t dev;
 309         struct vq_alloc_info vq_info[2];
 310         int nvqs;
 311
 312         dev = sc->vtballoon_dev;
 313         nvqs = 2;
 314
 315         VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtballoon_vq_intr, sc,
 316             &sc->vtballoon_inflate_vq, "%s inflate", device_get_nameunit(dev));
 317
 318         VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtballoon_vq_intr, sc,
 319             &sc->vtballoon_deflate_vq, "%s deflate", device_get_nameunit(dev));
 320
 321         return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info));
 322 }
 323
 324 static void
 325 vtballoon_vq_intr(void *xsc)
 326 {
 327         struct vtballoon_softc *sc;
 328
 329         sc = xsc;
 330
 331         VTBALLOON_LOCK(sc);
 332         wakeup_one(sc);
 333         VTBALLOON_UNLOCK(sc);
 334 }
 335
 336 static void
 337 vtballoon_inflate(struct vtballoon_softc *sc, int npages)
 338 {
 339         struct virtqueue *vq;
 340         vm_page_t m;
 341         int i;
 342
 343         vq = sc->vtballoon_inflate_vq;
 344
 345         if (npages > VTBALLOON_PAGES_PER_REQUEST)
 346                 npages = VTBALLOON_PAGES_PER_REQUEST;
 347
 348         for (i = 0; i < npages; i++) {
 349                 if ((m = vtballoon_alloc_page(sc)) == NULL) {
 350                         sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT;
 351                         break;
 352                 }
 353
 354                 sc->vtballoon_page_frames[i] =
 355                     VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
 356
 357                 KASSERT(m->a.queue == PQ_NONE,
 358                     ("%s: allocated page %p on queue", __func__, m));
 359                 TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, plinks.q);
 360         }
 361
 362         if (i > 0)
 363                 vtballoon_send_page_frames(sc, vq, i);
 364 }
 365
 366 static void
 367 vtballoon_deflate(struct vtballoon_softc *sc, int npages)
 368 {
 369         TAILQ_HEAD(, vm_page) free_pages;
 370         struct virtqueue *vq;
 371         vm_page_t m;
 372         int i;
 373
 374         vq = sc->vtballoon_deflate_vq;
 375         TAILQ_INIT(&free_pages);
 376
 377         if (npages > VTBALLOON_PAGES_PER_REQUEST)
 378                 npages = VTBALLOON_PAGES_PER_REQUEST;
 379
 380         for (i = 0; i < npages; i++) {
 381                 m = TAILQ_FIRST(&sc->vtballoon_pages);
 382                 KASSERT(m != NULL, ("%s: no more pages to deflate", __func__));
 383
 384                 sc->vtballoon_page_frames[i] =
 385                     VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
 386
 387                 TAILQ_REMOVE(&sc->vtballoon_pages, m, plinks.q);
 388                 TAILQ_INSERT_TAIL(&free_pages, m, plinks.q);
 389         }
 390
 391         if (i > 0) {
 392                 /* Always tell host first before freeing the pages. */
 393                 vtballoon_send_page_frames(sc, vq, i);
 394
 395                 while ((m = TAILQ_FIRST(&free_pages)) != NULL) {
 396                         TAILQ_REMOVE(&free_pages, m, plinks.q);
 397                         vtballoon_free_page(sc, m);
 398                 }
 399         }
 400
 401         KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) &&
 402             sc->vtballoon_current_npages == 0) ||
 403             (!TAILQ_EMPTY(&sc->vtballoon_pages) &&
 404             sc->vtballoon_current_npages != 0),
 405             ("%s: bogus page count %d", __func__,
 406             sc->vtballoon_current_npages));
 407 }
 408
 409 static void
 410 vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq,
 411     int npages)
 412 {
 413         struct sglist sg;
 414         struct sglist_seg segs[1];
 415         void *c;
 416         int error;
 417
 418         sglist_init(&sg, 1, segs);
 419
 420         error = sglist_append(&sg, sc->vtballoon_page_frames,
 421             npages * sizeof(uint32_t));
 422         KASSERT(error == 0, ("error adding page frames to sglist"));
 423
 424         error = virtqueue_enqueue(vq, vq, &sg, 1, 0);
 425         KASSERT(error == 0, ("error enqueuing page frames to virtqueue"));
 426         virtqueue_notify(vq);
 427
 428         /*
 429          * Inflate and deflate operations are done synchronously. The
 430          * interrupt handler will wake us up.
 431          */
 432         VTBALLOON_LOCK(sc);
 433         while ((c = virtqueue_dequeue(vq, NULL)) == NULL)
 434                 msleep(sc, VTBALLOON_MTX(sc), 0, "vtbspf", 0);
 435         VTBALLOON_UNLOCK(sc);
 436
 437         KASSERT(c == vq, ("unexpected balloon operation response"));
 438 }
 439
 440 static void
 441 vtballoon_pop(struct vtballoon_softc *sc)
 442 {
 443
 444         while (!TAILQ_EMPTY(&sc->vtballoon_pages))
 445                 vtballoon_deflate(sc, sc->vtballoon_current_npages);
 446 }
 447
 448 static void
 449 vtballoon_stop(struct vtballoon_softc *sc)
 450 {
 451
 452         virtqueue_disable_intr(sc->vtballoon_inflate_vq);
 453         virtqueue_disable_intr(sc->vtballoon_deflate_vq);
 454
 455         virtio_stop(sc->vtballoon_dev);
 456 }
 457
 458 static vm_page_t
 459 vtballoon_alloc_page(struct vtballoon_softc *sc)
 460 {
 461         vm_page_t m;
 462
 463         m = vm_page_alloc_noobj(VM_ALLOC_NODUMP);
 464         if (m != NULL)
 465                 sc->vtballoon_current_npages++;
 466
 467         return (m);
 468 }
 469
 470 static void
 471 vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m)
 472 {
 473
 474         vm_page_free(m);
 475         sc->vtballoon_current_npages--;
 476 }
 477
 478 static uint32_t
 479 vtballoon_desired_size(struct vtballoon_softc *sc)
 480 {
 481         uint32_t desired;
 482
 483         desired = virtio_read_dev_config_4(sc->vtballoon_dev,
 484             offsetof(struct virtio_balloon_config, num_pages));
 485
 486         if (vtballoon_modern(sc))
 487                 return (desired);
 488         else
 489                 return (le32toh(desired));
 490 }
 491
 492 static void
 493 vtballoon_update_size(struct vtballoon_softc *sc)
 494 {
 495         uint32_t npages;
 496
 497         npages = sc->vtballoon_current_npages;
 498         if (!vtballoon_modern(sc))
 499                 npages = htole32(npages);
 500
 501         virtio_write_dev_config_4(sc->vtballoon_dev,
 502             offsetof(struct virtio_balloon_config, actual), npages);
 503 }
 504
 505 static int
 506 vtballoon_sleep(struct vtballoon_softc *sc)
 507 {
 508         int rc, timeout;
 509         uint32_t current, desired;
 510
 511         rc = 0;
 512         current = sc->vtballoon_current_npages;
 513
 514         VTBALLOON_LOCK(sc);
 515         for (;;) {
 516                 if (sc->vtballoon_flags & VTBALLOON_FLAG_DETACH) {
 517                         rc = 1;
 518                         break;
 519                 }
 520
 521                 desired = vtballoon_desired_size(sc);
 522                 sc->vtballoon_desired_npages = desired;
 523
 524                 /*
 525                  * If given, use non-zero timeout on the first time through
 526                  * the loop. On subsequent times, timeout will be zero so
 527                  * we will reevaluate the desired size of the balloon and
 528                  * break out to retry if needed.
 529                  */
 530                 timeout = sc->vtballoon_timeout;
 531                 sc->vtballoon_timeout = 0;
 532
 533                 if (current > desired)
 534                         break;
 535                 if (current < desired && timeout == 0)
 536                         break;
 537
 538                 msleep(sc, VTBALLOON_MTX(sc), 0, "vtbslp", timeout);
 539         }
 540         VTBALLOON_UNLOCK(sc);
 541
 542         return (rc);
 543 }
 544
 545 static void
 546 vtballoon_thread(void *xsc)
 547 {
 548         struct vtballoon_softc *sc;
 549         uint32_t current, desired;
 550
 551         sc = xsc;
 552
 553         for (;;) {
 554                 if (vtballoon_sleep(sc) != 0)
 555                         break;
 556
 557                 current = sc->vtballoon_current_npages;
 558                 desired = sc->vtballoon_desired_npages;
 559
 560                 if (desired != current) {
 561                         if (desired > current)
 562                                 vtballoon_inflate(sc, desired - current);
 563                         else
 564                                 vtballoon_deflate(sc, current - desired);
 565
 566                         vtballoon_update_size(sc);
 567                 }
 568         }
 569
 570         kthread_exit();
 571 }
 572
 573 static void
 574 vtballoon_setup_sysctl(struct vtballoon_softc *sc)
 575 {
 576         device_t dev;
 577         struct sysctl_ctx_list *ctx;
 578         struct sysctl_oid *tree;
 579         struct sysctl_oid_list *child;
 580
 581         dev = sc->vtballoon_dev;
 582         ctx = device_get_sysctl_ctx(dev);
 583         tree = device_get_sysctl_tree(dev);
 584         child = SYSCTL_CHILDREN(tree);
 585
 586         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "desired",
 587             CTLFLAG_RD, &sc->vtballoon_desired_npages, sizeof(uint32_t),
 588             "Desired balloon size in pages");
 589
 590         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "current",
 591             CTLFLAG_RD, &sc->vtballoon_current_npages, sizeof(uint32_t),
 592             "Current balloon size in pages");
 593 }