sys/dev/virtio/balloon/virtio_balloon.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice unmodified, this list of conditions, and the following
  12  *    disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  */
  28
  29 /* Driver for VirtIO memory balloon devices. */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33
  34 #include <sys/param.h>
  35 #include <sys/systm.h>
  36 #include <sys/kernel.h>
  37 #include <sys/endian.h>
  38 #include <sys/kthread.h>
  39 #include <sys/malloc.h>
  40 #include <sys/module.h>
  41 #include <sys/sglist.h>
  42 #include <sys/sysctl.h>
  43 #include <sys/lock.h>
  44 #include <sys/mutex.h>
  45 #include <sys/queue.h>
  46
  47 #include <vm/vm.h>
  48 #include <vm/vm_page.h>
  49
  50 #include <machine/bus.h>
  51 #include <machine/resource.h>
  52 #include <sys/bus.h>
  53 #include <sys/rman.h>
  54
  55 #include <dev/virtio/virtio.h>
  56 #include <dev/virtio/virtqueue.h>
  57 #include <dev/virtio/balloon/virtio_balloon.h>
  58
  59 #include "virtio_if.h"
  60
  61 struct vtballoon_softc {
  62         device_t                 vtballoon_dev;
  63         struct mtx               vtballoon_mtx;
  64         uint64_t                 vtballoon_features;
  65         uint32_t                 vtballoon_flags;
  66 #define VTBALLOON_FLAG_DETACH    0x01
  67
  68         struct virtqueue        *vtballoon_inflate_vq;
  69         struct virtqueue        *vtballoon_deflate_vq;
  70
  71         uint32_t                 vtballoon_desired_npages;
  72         uint32_t                 vtballoon_current_npages;
  73         TAILQ_HEAD(,vm_page)     vtballoon_pages;
  74
  75         struct thread           *vtballoon_td;
  76         uint32_t                *vtballoon_page_frames;
  77         int                      vtballoon_timeout;
  78 };
  79
  80 static struct virtio_feature_desc vtballoon_feature_desc[] = {
  81         { VIRTIO_BALLOON_F_MUST_TELL_HOST,      "MustTellHost"  },
  82         { VIRTIO_BALLOON_F_STATS_VQ,            "StatsVq"       },
  83
  84         { 0, NULL }
  85 };
  86
  87 static int      vtballoon_probe(device_t);
  88 static int      vtballoon_attach(device_t);
  89 static int      vtballoon_detach(device_t);
  90 static int      vtballoon_config_change(device_t);
  91
  92 static void     vtballoon_negotiate_features(struct vtballoon_softc *);
  93 static int      vtballoon_alloc_virtqueues(struct vtballoon_softc *);
  94
  95 static void     vtballoon_vq_intr(void *);
  96
  97 static void     vtballoon_inflate(struct vtballoon_softc *, int);
  98 static void     vtballoon_deflate(struct vtballoon_softc *, int);
  99
 100 static void     vtballoon_send_page_frames(struct vtballoon_softc *,
 101                     struct virtqueue *, int);
 102
 103 static void     vtballoon_pop(struct vtballoon_softc *);
 104 static void     vtballoon_stop(struct vtballoon_softc *);
 105
 106 static vm_page_t
 107                 vtballoon_alloc_page(struct vtballoon_softc *);
 108 static void     vtballoon_free_page(struct vtballoon_softc *, vm_page_t);
 109
 110 static int      vtballoon_sleep(struct vtballoon_softc *);
 111 static void     vtballoon_thread(void *);
 112 static void     vtballoon_add_sysctl(struct vtballoon_softc *);
 113
 114 /* Features desired/implemented by this driver. */
 115 #define VTBALLOON_FEATURES              0
 116
 117 /* Timeout between retries when the balloon needs inflating. */
 118 #define VTBALLOON_LOWMEM_TIMEOUT        hz
 119
 120 /*
 121  * Maximum number of pages we'll request to inflate or deflate
 122  * the balloon in one virtqueue request. Both Linux and NetBSD
 123  * have settled on 256, doing up to 1MB at a time.
 124  */
 125 #define VTBALLOON_PAGES_PER_REQUEST     256
 126
 127 /* Must be able to fix all pages frames in one page (segment). */
 128 CTASSERT(VTBALLOON_PAGES_PER_REQUEST * sizeof(uint32_t) <= PAGE_SIZE);
 129
 130 #define VTBALLOON_MTX(_sc)              &(_sc)->vtballoon_mtx
 131 #define VTBALLOON_LOCK_INIT(_sc, _name) mtx_init(VTBALLOON_MTX((_sc)), _name, \
 132                                             "VirtIO Balloon Lock", MTX_DEF)
 133 #define VTBALLOON_LOCK(_sc)             mtx_lock(VTBALLOON_MTX((_sc)))
 134 #define VTBALLOON_UNLOCK(_sc)           mtx_unlock(VTBALLOON_MTX((_sc)))
 135 #define VTBALLOON_LOCK_DESTROY(_sc)     mtx_destroy(VTBALLOON_MTX((_sc)))
 136
 137 static device_method_t vtballoon_methods[] = {
 138         /* Device methods. */
 139         DEVMETHOD(device_probe,         vtballoon_probe),
 140         DEVMETHOD(device_attach,        vtballoon_attach),
 141         DEVMETHOD(device_detach,        vtballoon_detach),
 142
 143         /* VirtIO methods. */
 144         DEVMETHOD(virtio_config_change, vtballoon_config_change),
 145
 146         DEVMETHOD_END
 147 };
 148
 149 static driver_t vtballoon_driver = {
 150         "vtballoon",
 151         vtballoon_methods,
 152         sizeof(struct vtballoon_softc)
 153 };
 154 static devclass_t vtballoon_devclass;
 155
 156 DRIVER_MODULE(virtio_balloon, virtio_mmio, vtballoon_driver,
 157     vtballoon_devclass, 0, 0);
 158 DRIVER_MODULE(virtio_balloon, virtio_pci, vtballoon_driver,
 159     vtballoon_devclass, 0, 0);
 160 MODULE_VERSION(virtio_balloon, 1);
 161 MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1);
 162
 163 VIRTIO_SIMPLE_PNPTABLE(virtio_balloon, VIRTIO_ID_BALLOON,
 164     "VirtIO Balloon Adapter");
 165 VIRTIO_SIMPLE_PNPINFO(virtio_mmio, virtio_balloon);
 166 VIRTIO_SIMPLE_PNPINFO(virtio_pci, virtio_balloon);
 167
 168 static int
 169 vtballoon_probe(device_t dev)
 170 {
 171         return (VIRTIO_SIMPLE_PROBE(dev, virtio_balloon));
 172 }
 173
 174 static int
 175 vtballoon_attach(device_t dev)
 176 {
 177         struct vtballoon_softc *sc;
 178         int error;
 179
 180         sc = device_get_softc(dev);
 181         sc->vtballoon_dev = dev;
 182
 183         VTBALLOON_LOCK_INIT(sc, device_get_nameunit(dev));
 184         TAILQ_INIT(&sc->vtballoon_pages);
 185
 186         vtballoon_add_sysctl(sc);
 187
 188         virtio_set_feature_desc(dev, vtballoon_feature_desc);
 189         vtballoon_negotiate_features(sc);
 190
 191         sc->vtballoon_page_frames = malloc(VTBALLOON_PAGES_PER_REQUEST *
 192             sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO);
 193         if (sc->vtballoon_page_frames == NULL) {
 194                 error = ENOMEM;
 195                 device_printf(dev,
 196                     "cannot allocate page frame request array\n");
 197                 goto fail;
 198         }
 199
 200         error = vtballoon_alloc_virtqueues(sc);
 201         if (error) {
 202                 device_printf(dev, "cannot allocate virtqueues\n");
 203                 goto fail;
 204         }
 205
 206         error = virtio_setup_intr(dev, INTR_TYPE_MISC);
 207         if (error) {
 208                 device_printf(dev, "cannot setup virtqueue interrupts\n");
 209                 goto fail;
 210         }
 211
 212         error = kthread_add(vtballoon_thread, sc, NULL, &sc->vtballoon_td,
 213             0, 0, "virtio_balloon");
 214         if (error) {
 215                 device_printf(dev, "cannot create balloon kthread\n");
 216                 goto fail;
 217         }
 218
 219         virtqueue_enable_intr(sc->vtballoon_inflate_vq);
 220         virtqueue_enable_intr(sc->vtballoon_deflate_vq);
 221
 222 fail:
 223         if (error)
 224                 vtballoon_detach(dev);
 225
 226         return (error);
 227 }
 228
 229 static int
 230 vtballoon_detach(device_t dev)
 231 {
 232         struct vtballoon_softc *sc;
 233
 234         sc = device_get_softc(dev);
 235
 236         if (sc->vtballoon_td != NULL) {
 237                 VTBALLOON_LOCK(sc);
 238                 sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH;
 239                 wakeup_one(sc);
 240                 msleep(sc->vtballoon_td, VTBALLOON_MTX(sc), 0, "vtbdth", 0);
 241                 VTBALLOON_UNLOCK(sc);
 242
 243                 sc->vtballoon_td = NULL;
 244         }
 245
 246         if (device_is_attached(dev)) {
 247                 vtballoon_pop(sc);
 248                 vtballoon_stop(sc);
 249         }
 250
 251         if (sc->vtballoon_page_frames != NULL) {
 252                 free(sc->vtballoon_page_frames, M_DEVBUF);
 253                 sc->vtballoon_page_frames = NULL;
 254         }
 255
 256         VTBALLOON_LOCK_DESTROY(sc);
 257
 258         return (0);
 259 }
 260
 261 static int
 262 vtballoon_config_change(device_t dev)
 263 {
 264         struct vtballoon_softc *sc;
 265
 266         sc = device_get_softc(dev);
 267
 268         VTBALLOON_LOCK(sc);
 269         wakeup_one(sc);
 270         VTBALLOON_UNLOCK(sc);
 271
 272         return (1);
 273 }
 274
 275 static void
 276 vtballoon_negotiate_features(struct vtballoon_softc *sc)
 277 {
 278         device_t dev;
 279         uint64_t features;
 280
 281         dev = sc->vtballoon_dev;
 282         features = virtio_negotiate_features(dev, VTBALLOON_FEATURES);
 283         sc->vtballoon_features = features;
 284 }
 285
 286 static int
 287 vtballoon_alloc_virtqueues(struct vtballoon_softc *sc)
 288 {
 289         device_t dev;
 290         struct vq_alloc_info vq_info[2];
 291         int nvqs;
 292
 293         dev = sc->vtballoon_dev;
 294         nvqs = 2;
 295
 296         VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtballoon_vq_intr, sc,
 297             &sc->vtballoon_inflate_vq, "%s inflate", device_get_nameunit(dev));
 298
 299         VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtballoon_vq_intr, sc,
 300             &sc->vtballoon_deflate_vq, "%s deflate", device_get_nameunit(dev));
 301
 302         return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info));
 303 }
 304
 305 static void
 306 vtballoon_vq_intr(void *xsc)
 307 {
 308         struct vtballoon_softc *sc;
 309
 310         sc = xsc;
 311
 312         VTBALLOON_LOCK(sc);
 313         wakeup_one(sc);
 314         VTBALLOON_UNLOCK(sc);
 315 }
 316
 317 static void
 318 vtballoon_inflate(struct vtballoon_softc *sc, int npages)
 319 {
 320         struct virtqueue *vq;
 321         vm_page_t m;
 322         int i;
 323
 324         vq = sc->vtballoon_inflate_vq;
 325
 326         if (npages > VTBALLOON_PAGES_PER_REQUEST)
 327                 npages = VTBALLOON_PAGES_PER_REQUEST;
 328
 329         for (i = 0; i < npages; i++) {
 330                 if ((m = vtballoon_alloc_page(sc)) == NULL) {
 331                         sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT;
 332                         break;
 333                 }
 334
 335                 sc->vtballoon_page_frames[i] =
 336                     VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
 337
 338                 KASSERT(m->a.queue == PQ_NONE,
 339                     ("%s: allocated page %p on queue", __func__, m));
 340                 TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, plinks.q);
 341         }
 342
 343         if (i > 0)
 344                 vtballoon_send_page_frames(sc, vq, i);
 345 }
 346
 347 static void
 348 vtballoon_deflate(struct vtballoon_softc *sc, int npages)
 349 {
 350         TAILQ_HEAD(, vm_page) free_pages;
 351         struct virtqueue *vq;
 352         vm_page_t m;
 353         int i;
 354
 355         vq = sc->vtballoon_deflate_vq;
 356         TAILQ_INIT(&free_pages);
 357
 358         if (npages > VTBALLOON_PAGES_PER_REQUEST)
 359                 npages = VTBALLOON_PAGES_PER_REQUEST;
 360
 361         for (i = 0; i < npages; i++) {
 362                 m = TAILQ_FIRST(&sc->vtballoon_pages);
 363                 KASSERT(m != NULL, ("%s: no more pages to deflate", __func__));
 364
 365                 sc->vtballoon_page_frames[i] =
 366                     VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
 367
 368                 TAILQ_REMOVE(&sc->vtballoon_pages, m, plinks.q);
 369                 TAILQ_INSERT_TAIL(&free_pages, m, plinks.q);
 370         }
 371
 372         if (i > 0) {
 373                 /* Always tell host first before freeing the pages. */
 374                 vtballoon_send_page_frames(sc, vq, i);
 375
 376                 while ((m = TAILQ_FIRST(&free_pages)) != NULL) {
 377                         TAILQ_REMOVE(&free_pages, m, plinks.q);
 378                         vtballoon_free_page(sc, m);
 379                 }
 380         }
 381
 382         KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) &&
 383             sc->vtballoon_current_npages == 0) ||
 384             (!TAILQ_EMPTY(&sc->vtballoon_pages) &&
 385             sc->vtballoon_current_npages != 0),
 386             ("%s: bogus page count %d", __func__,
 387             sc->vtballoon_current_npages));
 388 }
 389
 390 static void
 391 vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq,
 392     int npages)
 393 {
 394         struct sglist sg;
 395         struct sglist_seg segs[1];
 396         void *c;
 397         int error;
 398
 399         sglist_init(&sg, 1, segs);
 400
 401         error = sglist_append(&sg, sc->vtballoon_page_frames,
 402             npages * sizeof(uint32_t));
 403         KASSERT(error == 0, ("error adding page frames to sglist"));
 404
 405         error = virtqueue_enqueue(vq, vq, &sg, 1, 0);
 406         KASSERT(error == 0, ("error enqueuing page frames to virtqueue"));
 407         virtqueue_notify(vq);
 408
 409         /*
 410          * Inflate and deflate operations are done synchronously. The
 411          * interrupt handler will wake us up.
 412          */
 413         VTBALLOON_LOCK(sc);
 414         while ((c = virtqueue_dequeue(vq, NULL)) == NULL)
 415                 msleep(sc, VTBALLOON_MTX(sc), 0, "vtbspf", 0);
 416         VTBALLOON_UNLOCK(sc);
 417
 418         KASSERT(c == vq, ("unexpected balloon operation response"));
 419 }
 420
 421 static void
 422 vtballoon_pop(struct vtballoon_softc *sc)
 423 {
 424
 425         while (!TAILQ_EMPTY(&sc->vtballoon_pages))
 426                 vtballoon_deflate(sc, sc->vtballoon_current_npages);
 427 }
 428
 429 static void
 430 vtballoon_stop(struct vtballoon_softc *sc)
 431 {
 432
 433         virtqueue_disable_intr(sc->vtballoon_inflate_vq);
 434         virtqueue_disable_intr(sc->vtballoon_deflate_vq);
 435
 436         virtio_stop(sc->vtballoon_dev);
 437 }
 438
 439 static vm_page_t
 440 vtballoon_alloc_page(struct vtballoon_softc *sc)
 441 {
 442         vm_page_t m;
 443
 444         m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ);
 445         if (m != NULL)
 446                 sc->vtballoon_current_npages++;
 447
 448         return (m);
 449 }
 450
 451 static void
 452 vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m)
 453 {
 454
 455         vm_page_free(m);
 456         sc->vtballoon_current_npages--;
 457 }
 458
 459 static uint32_t
 460 vtballoon_desired_size(struct vtballoon_softc *sc)
 461 {
 462         uint32_t desired;
 463
 464         desired = virtio_read_dev_config_4(sc->vtballoon_dev,
 465             offsetof(struct virtio_balloon_config, num_pages));
 466
 467         return (le32toh(desired));
 468 }
 469
 470 static void
 471 vtballoon_update_size(struct vtballoon_softc *sc)
 472 {
 473
 474         virtio_write_dev_config_4(sc->vtballoon_dev,
 475             offsetof(struct virtio_balloon_config, actual),
 476             htole32(sc->vtballoon_current_npages));
 477 }
 478
 479 static int
 480 vtballoon_sleep(struct vtballoon_softc *sc)
 481 {
 482         int rc, timeout;
 483         uint32_t current, desired;
 484
 485         rc = 0;
 486         current = sc->vtballoon_current_npages;
 487
 488         VTBALLOON_LOCK(sc);
 489         for (;;) {
 490                 if (sc->vtballoon_flags & VTBALLOON_FLAG_DETACH) {
 491                         rc = 1;
 492                         break;
 493                 }
 494
 495                 desired = vtballoon_desired_size(sc);
 496                 sc->vtballoon_desired_npages = desired;
 497
 498                 /*
 499                  * If given, use non-zero timeout on the first time through
 500                  * the loop. On subsequent times, timeout will be zero so
 501                  * we will reevaluate the desired size of the balloon and
 502                  * break out to retry if needed.
 503                  */
 504                 timeout = sc->vtballoon_timeout;
 505                 sc->vtballoon_timeout = 0;
 506
 507                 if (current > desired)
 508                         break;
 509                 if (current < desired && timeout == 0)
 510                         break;
 511
 512                 msleep(sc, VTBALLOON_MTX(sc), 0, "vtbslp", timeout);
 513         }
 514         VTBALLOON_UNLOCK(sc);
 515
 516         return (rc);
 517 }
 518
 519 static void
 520 vtballoon_thread(void *xsc)
 521 {
 522         struct vtballoon_softc *sc;
 523         uint32_t current, desired;
 524
 525         sc = xsc;
 526
 527         for (;;) {
 528                 if (vtballoon_sleep(sc) != 0)
 529                         break;
 530
 531                 current = sc->vtballoon_current_npages;
 532                 desired = sc->vtballoon_desired_npages;
 533
 534                 if (desired != current) {
 535                         if (desired > current)
 536                                 vtballoon_inflate(sc, desired - current);
 537                         else
 538                                 vtballoon_deflate(sc, current - desired);
 539
 540                         vtballoon_update_size(sc);
 541                 }
 542         }
 543
 544         kthread_exit();
 545 }
 546
 547 static void
 548 vtballoon_add_sysctl(struct vtballoon_softc *sc)
 549 {
 550         device_t dev;
 551         struct sysctl_ctx_list *ctx;
 552         struct sysctl_oid *tree;
 553         struct sysctl_oid_list *child;
 554
 555         dev = sc->vtballoon_dev;
 556         ctx = device_get_sysctl_ctx(dev);
 557         tree = device_get_sysctl_tree(dev);
 558         child = SYSCTL_CHILDREN(tree);
 559
 560         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "desired",
 561             CTLFLAG_RD, &sc->vtballoon_desired_npages, sizeof(uint32_t),
 562             "Desired balloon size in pages");
 563
 564         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "current",
 565             CTLFLAG_RD, &sc->vtballoon_current_npages, sizeof(uint32_t),
 566             "Current balloon size in pages");
 567 }