2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice unmodified, this list of conditions, and the following
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 /* Driver for VirtIO memory balloon devices. */
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/kernel.h>
37 #include <sys/endian.h>
38 #include <sys/kthread.h>
39 #include <sys/malloc.h>
40 #include <sys/module.h>
41 #include <sys/sglist.h>
42 #include <sys/sysctl.h>
44 #include <sys/mutex.h>
45 #include <sys/queue.h>
48 #include <vm/vm_page.h>
50 #include <machine/bus.h>
51 #include <machine/resource.h>
55 #include <dev/virtio/virtio.h>
56 #include <dev/virtio/virtqueue.h>
57 #include <dev/virtio/balloon/virtio_balloon.h>
59 #include "virtio_if.h"
61 struct vtballoon_softc {
62 device_t vtballoon_dev;
63 struct mtx vtballoon_mtx;
64 uint64_t vtballoon_features;
65 uint32_t vtballoon_flags;
66 #define VTBALLOON_FLAG_DETACH 0x01
68 struct virtqueue *vtballoon_inflate_vq;
69 struct virtqueue *vtballoon_deflate_vq;
71 uint32_t vtballoon_desired_npages;
72 uint32_t vtballoon_current_npages;
73 TAILQ_HEAD(,vm_page) vtballoon_pages;
75 struct thread *vtballoon_td;
76 uint32_t *vtballoon_page_frames;
77 int vtballoon_timeout;
80 static struct virtio_feature_desc vtballoon_feature_desc[] = {
81 { VIRTIO_BALLOON_F_MUST_TELL_HOST, "MustTellHost" },
82 { VIRTIO_BALLOON_F_STATS_VQ, "StatsVq" },
87 static int vtballoon_probe(device_t);
88 static int vtballoon_attach(device_t);
89 static int vtballoon_detach(device_t);
90 static int vtballoon_config_change(device_t);
92 static void vtballoon_negotiate_features(struct vtballoon_softc *);
93 static int vtballoon_alloc_virtqueues(struct vtballoon_softc *);
95 static void vtballoon_vq_intr(void *);
97 static void vtballoon_inflate(struct vtballoon_softc *, int);
98 static void vtballoon_deflate(struct vtballoon_softc *, int);
100 static void vtballoon_send_page_frames(struct vtballoon_softc *,
101 struct virtqueue *, int);
103 static void vtballoon_pop(struct vtballoon_softc *);
104 static void vtballoon_stop(struct vtballoon_softc *);
107 vtballoon_alloc_page(struct vtballoon_softc *);
108 static void vtballoon_free_page(struct vtballoon_softc *, vm_page_t);
110 static int vtballoon_sleep(struct vtballoon_softc *);
111 static void vtballoon_thread(void *);
112 static void vtballoon_add_sysctl(struct vtballoon_softc *);
114 /* Features desired/implemented by this driver. */
115 #define VTBALLOON_FEATURES 0
117 /* Timeout between retries when the balloon needs inflating. */
118 #define VTBALLOON_LOWMEM_TIMEOUT hz
121 * Maximum number of pages we'll request to inflate or deflate
122 * the balloon in one virtqueue request. Both Linux and NetBSD
123 * have settled on 256, doing up to 1MB at a time.
125 #define VTBALLOON_PAGES_PER_REQUEST 256
127 /* Must be able to fix all pages frames in one page (segment). */
128 CTASSERT(VTBALLOON_PAGES_PER_REQUEST * sizeof(uint32_t) <= PAGE_SIZE);
130 #define VTBALLOON_MTX(_sc) &(_sc)->vtballoon_mtx
131 #define VTBALLOON_LOCK_INIT(_sc, _name) mtx_init(VTBALLOON_MTX((_sc)), _name, \
132 "VirtIO Balloon Lock", MTX_DEF)
133 #define VTBALLOON_LOCK(_sc) mtx_lock(VTBALLOON_MTX((_sc)))
134 #define VTBALLOON_UNLOCK(_sc) mtx_unlock(VTBALLOON_MTX((_sc)))
135 #define VTBALLOON_LOCK_DESTROY(_sc) mtx_destroy(VTBALLOON_MTX((_sc)))
137 static device_method_t vtballoon_methods[] = {
138 /* Device methods. */
139 DEVMETHOD(device_probe, vtballoon_probe),
140 DEVMETHOD(device_attach, vtballoon_attach),
141 DEVMETHOD(device_detach, vtballoon_detach),
143 /* VirtIO methods. */
144 DEVMETHOD(virtio_config_change, vtballoon_config_change),
149 static driver_t vtballoon_driver = {
152 sizeof(struct vtballoon_softc)
154 static devclass_t vtballoon_devclass;
156 DRIVER_MODULE(virtio_balloon, virtio_mmio, vtballoon_driver,
157 vtballoon_devclass, 0, 0);
158 DRIVER_MODULE(virtio_balloon, virtio_pci, vtballoon_driver,
159 vtballoon_devclass, 0, 0);
160 MODULE_VERSION(virtio_balloon, 1);
161 MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1);
163 VIRTIO_SIMPLE_PNPTABLE(virtio_balloon, VIRTIO_ID_BALLOON,
164 "VirtIO Balloon Adapter");
165 VIRTIO_SIMPLE_PNPINFO(virtio_mmio, virtio_balloon);
166 VIRTIO_SIMPLE_PNPINFO(virtio_pci, virtio_balloon);
169 vtballoon_probe(device_t dev)
171 return (VIRTIO_SIMPLE_PROBE(dev, virtio_balloon));
175 vtballoon_attach(device_t dev)
177 struct vtballoon_softc *sc;
180 sc = device_get_softc(dev);
181 sc->vtballoon_dev = dev;
183 VTBALLOON_LOCK_INIT(sc, device_get_nameunit(dev));
184 TAILQ_INIT(&sc->vtballoon_pages);
186 vtballoon_add_sysctl(sc);
188 virtio_set_feature_desc(dev, vtballoon_feature_desc);
189 vtballoon_negotiate_features(sc);
191 sc->vtballoon_page_frames = malloc(VTBALLOON_PAGES_PER_REQUEST *
192 sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO);
193 if (sc->vtballoon_page_frames == NULL) {
196 "cannot allocate page frame request array\n");
200 error = vtballoon_alloc_virtqueues(sc);
202 device_printf(dev, "cannot allocate virtqueues\n");
206 error = virtio_setup_intr(dev, INTR_TYPE_MISC);
208 device_printf(dev, "cannot setup virtqueue interrupts\n");
212 error = kthread_add(vtballoon_thread, sc, NULL, &sc->vtballoon_td,
213 0, 0, "virtio_balloon");
215 device_printf(dev, "cannot create balloon kthread\n");
219 virtqueue_enable_intr(sc->vtballoon_inflate_vq);
220 virtqueue_enable_intr(sc->vtballoon_deflate_vq);
224 vtballoon_detach(dev);
230 vtballoon_detach(device_t dev)
232 struct vtballoon_softc *sc;
234 sc = device_get_softc(dev);
236 if (sc->vtballoon_td != NULL) {
238 sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH;
240 msleep(sc->vtballoon_td, VTBALLOON_MTX(sc), 0, "vtbdth", 0);
241 VTBALLOON_UNLOCK(sc);
243 sc->vtballoon_td = NULL;
246 if (device_is_attached(dev)) {
251 if (sc->vtballoon_page_frames != NULL) {
252 free(sc->vtballoon_page_frames, M_DEVBUF);
253 sc->vtballoon_page_frames = NULL;
256 VTBALLOON_LOCK_DESTROY(sc);
262 vtballoon_config_change(device_t dev)
264 struct vtballoon_softc *sc;
266 sc = device_get_softc(dev);
270 VTBALLOON_UNLOCK(sc);
276 vtballoon_negotiate_features(struct vtballoon_softc *sc)
281 dev = sc->vtballoon_dev;
282 features = virtio_negotiate_features(dev, VTBALLOON_FEATURES);
283 sc->vtballoon_features = features;
287 vtballoon_alloc_virtqueues(struct vtballoon_softc *sc)
290 struct vq_alloc_info vq_info[2];
293 dev = sc->vtballoon_dev;
296 VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtballoon_vq_intr, sc,
297 &sc->vtballoon_inflate_vq, "%s inflate", device_get_nameunit(dev));
299 VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtballoon_vq_intr, sc,
300 &sc->vtballoon_deflate_vq, "%s deflate", device_get_nameunit(dev));
302 return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info));
306 vtballoon_vq_intr(void *xsc)
308 struct vtballoon_softc *sc;
314 VTBALLOON_UNLOCK(sc);
318 vtballoon_inflate(struct vtballoon_softc *sc, int npages)
320 struct virtqueue *vq;
324 vq = sc->vtballoon_inflate_vq;
326 if (npages > VTBALLOON_PAGES_PER_REQUEST)
327 npages = VTBALLOON_PAGES_PER_REQUEST;
329 for (i = 0; i < npages; i++) {
330 if ((m = vtballoon_alloc_page(sc)) == NULL) {
331 sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT;
335 sc->vtballoon_page_frames[i] =
336 VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
338 KASSERT(m->a.queue == PQ_NONE,
339 ("%s: allocated page %p on queue", __func__, m));
340 TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, plinks.q);
344 vtballoon_send_page_frames(sc, vq, i);
348 vtballoon_deflate(struct vtballoon_softc *sc, int npages)
350 TAILQ_HEAD(, vm_page) free_pages;
351 struct virtqueue *vq;
355 vq = sc->vtballoon_deflate_vq;
356 TAILQ_INIT(&free_pages);
358 if (npages > VTBALLOON_PAGES_PER_REQUEST)
359 npages = VTBALLOON_PAGES_PER_REQUEST;
361 for (i = 0; i < npages; i++) {
362 m = TAILQ_FIRST(&sc->vtballoon_pages);
363 KASSERT(m != NULL, ("%s: no more pages to deflate", __func__));
365 sc->vtballoon_page_frames[i] =
366 VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
368 TAILQ_REMOVE(&sc->vtballoon_pages, m, plinks.q);
369 TAILQ_INSERT_TAIL(&free_pages, m, plinks.q);
373 /* Always tell host first before freeing the pages. */
374 vtballoon_send_page_frames(sc, vq, i);
376 while ((m = TAILQ_FIRST(&free_pages)) != NULL) {
377 TAILQ_REMOVE(&free_pages, m, plinks.q);
378 vtballoon_free_page(sc, m);
382 KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) &&
383 sc->vtballoon_current_npages == 0) ||
384 (!TAILQ_EMPTY(&sc->vtballoon_pages) &&
385 sc->vtballoon_current_npages != 0),
386 ("%s: bogus page count %d", __func__,
387 sc->vtballoon_current_npages));
391 vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq,
395 struct sglist_seg segs[1];
399 sglist_init(&sg, 1, segs);
401 error = sglist_append(&sg, sc->vtballoon_page_frames,
402 npages * sizeof(uint32_t));
403 KASSERT(error == 0, ("error adding page frames to sglist"));
405 error = virtqueue_enqueue(vq, vq, &sg, 1, 0);
406 KASSERT(error == 0, ("error enqueuing page frames to virtqueue"));
407 virtqueue_notify(vq);
410 * Inflate and deflate operations are done synchronously. The
411 * interrupt handler will wake us up.
414 while ((c = virtqueue_dequeue(vq, NULL)) == NULL)
415 msleep(sc, VTBALLOON_MTX(sc), 0, "vtbspf", 0);
416 VTBALLOON_UNLOCK(sc);
418 KASSERT(c == vq, ("unexpected balloon operation response"));
422 vtballoon_pop(struct vtballoon_softc *sc)
425 while (!TAILQ_EMPTY(&sc->vtballoon_pages))
426 vtballoon_deflate(sc, sc->vtballoon_current_npages);
430 vtballoon_stop(struct vtballoon_softc *sc)
433 virtqueue_disable_intr(sc->vtballoon_inflate_vq);
434 virtqueue_disable_intr(sc->vtballoon_deflate_vq);
436 virtio_stop(sc->vtballoon_dev);
440 vtballoon_alloc_page(struct vtballoon_softc *sc)
444 m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ);
446 sc->vtballoon_current_npages++;
452 vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m)
456 sc->vtballoon_current_npages--;
460 vtballoon_desired_size(struct vtballoon_softc *sc)
464 desired = virtio_read_dev_config_4(sc->vtballoon_dev,
465 offsetof(struct virtio_balloon_config, num_pages));
467 return (le32toh(desired));
471 vtballoon_update_size(struct vtballoon_softc *sc)
474 virtio_write_dev_config_4(sc->vtballoon_dev,
475 offsetof(struct virtio_balloon_config, actual),
476 htole32(sc->vtballoon_current_npages));
480 vtballoon_sleep(struct vtballoon_softc *sc)
483 uint32_t current, desired;
486 current = sc->vtballoon_current_npages;
490 if (sc->vtballoon_flags & VTBALLOON_FLAG_DETACH) {
495 desired = vtballoon_desired_size(sc);
496 sc->vtballoon_desired_npages = desired;
499 * If given, use non-zero timeout on the first time through
500 * the loop. On subsequent times, timeout will be zero so
501 * we will reevaluate the desired size of the balloon and
502 * break out to retry if needed.
504 timeout = sc->vtballoon_timeout;
505 sc->vtballoon_timeout = 0;
507 if (current > desired)
509 if (current < desired && timeout == 0)
512 msleep(sc, VTBALLOON_MTX(sc), 0, "vtbslp", timeout);
514 VTBALLOON_UNLOCK(sc);
520 vtballoon_thread(void *xsc)
522 struct vtballoon_softc *sc;
523 uint32_t current, desired;
528 if (vtballoon_sleep(sc) != 0)
531 current = sc->vtballoon_current_npages;
532 desired = sc->vtballoon_desired_npages;
534 if (desired != current) {
535 if (desired > current)
536 vtballoon_inflate(sc, desired - current);
538 vtballoon_deflate(sc, current - desired);
540 vtballoon_update_size(sc);
548 vtballoon_add_sysctl(struct vtballoon_softc *sc)
551 struct sysctl_ctx_list *ctx;
552 struct sysctl_oid *tree;
553 struct sysctl_oid_list *child;
555 dev = sc->vtballoon_dev;
556 ctx = device_get_sysctl_ctx(dev);
557 tree = device_get_sysctl_tree(dev);
558 child = SYSCTL_CHILDREN(tree);
560 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "desired",
561 CTLFLAG_RD, &sc->vtballoon_desired_npages, sizeof(uint32_t),
562 "Desired balloon size in pages");
564 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "current",
565 CTLFLAG_RD, &sc->vtballoon_current_npages, sizeof(uint32_t),
566 "Current balloon size in pages");