sys/dev/xen/balloon/balloon.c

   1 /******************************************************************************
   2  * balloon.c
   3  *
   4  * Xen balloon driver - enables returning/claiming memory to/from Xen.
   5  *
   6  * Copyright (c) 2003, B Dragovic
   7  * Copyright (c) 2003-2004, M Williamson, K Fraser
   8  * Copyright (c) 2005 Dan M. Smith, IBM Corporation
   9  *
  10  * This file may be distributed separately from the Linux kernel, or
  11  * incorporated into other software packages, subject to the following license:
  12  *
  13  * Permission is hereby granted, free of charge, to any person obtaining a copy
  14  * of this source file (the "Software"), to deal in the Software without
  15  * restriction, including without limitation the rights to use, copy, modify,
  16  * merge, publish, distribute, sublicense, and/or sell copies of the Software,
  17  * and to permit persons to whom the Software is furnished to do so, subject to
  18  * the following conditions:
  19  *
  20  * The above copyright notice and this permission notice shall be included in
  21  * all copies or substantial portions of the Software.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  24  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  25  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  26  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  27  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  28  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  29  * IN THE SOFTWARE.
  30  */
  31
  32 #include <sys/cdefs.h>
  33 __FBSDID("$FreeBSD$");
  34
  35 #include <sys/param.h>
  36 #include <sys/lock.h>
  37 #include <sys/kernel.h>
  38 #include <sys/kthread.h>
  39 #include <sys/malloc.h>
  40 #include <sys/mutex.h>
  41 #include <sys/sysctl.h>
  42
  43 #include <machine/xen/xen-os.h>
  44 #include <machine/xen/xenfunc.h>
  45 #include <machine/xen/xenvar.h>
  46 #include <xen/hypervisor.h>
  47 #include <xen/xenbus/xenbusvar.h>
  48
  49 #include <vm/vm.h>
  50 #include <vm/vm_page.h>
  51
  52 MALLOC_DEFINE(M_BALLOON, "Balloon", "Xen Balloon Driver");
  53
  54 struct mtx balloon_mutex;
  55
  56 /*
  57  * Protects atomic reservation decrease/increase against concurrent increases.
  58  * Also protects non-atomic updates of current_pages and driver_pages, and
  59  * balloon lists.
  60  */
  61 struct mtx balloon_lock;
  62
  63 /* We increase/decrease in batches which fit in a page */
  64 static unsigned long frame_list[PAGE_SIZE / sizeof(unsigned long)];
  65 #define ARRAY_SIZE(A)   (sizeof(A) / sizeof(A[0]))
  66
  67 struct balloon_stats {
  68         /* We aim for 'current allocation' == 'target allocation'. */
  69         unsigned long current_pages;
  70         unsigned long target_pages;
  71         /* We may hit the hard limit in Xen. If we do then we remember it. */
  72         unsigned long hard_limit;
  73         /*
  74          * Drivers may alter the memory reservation independently, but they
  75          * must inform the balloon driver so we avoid hitting the hard limit.
  76          */
  77         unsigned long driver_pages;
  78         /* Number of pages in high- and low-memory balloons. */
  79         unsigned long balloon_low;
  80         unsigned long balloon_high;
  81 };
  82
  83 static struct balloon_stats balloon_stats;
  84 #define bs balloon_stats
  85
  86 SYSCTL_DECL(_dev_xen);
  87 SYSCTL_NODE(_dev_xen, OID_AUTO, balloon, CTLFLAG_RD, NULL, "Balloon");
  88 SYSCTL_ULONG(_dev_xen_balloon, OID_AUTO, current, CTLFLAG_RD,
  89     &bs.current_pages, 0, "Current allocation");
  90 SYSCTL_ULONG(_dev_xen_balloon, OID_AUTO, target, CTLFLAG_RD,
  91     &bs.target_pages, 0, "Target allocation");
  92 SYSCTL_ULONG(_dev_xen_balloon, OID_AUTO, driver_pages, CTLFLAG_RD,
  93     &bs.driver_pages, 0, "Driver pages");
  94 SYSCTL_ULONG(_dev_xen_balloon, OID_AUTO, hard_limit, CTLFLAG_RD,
  95     &bs.hard_limit, 0, "Xen hard limit");
  96 SYSCTL_ULONG(_dev_xen_balloon, OID_AUTO, low_mem, CTLFLAG_RD,
  97     &bs.balloon_low, 0, "Low-mem balloon");
  98 SYSCTL_ULONG(_dev_xen_balloon, OID_AUTO, high_mem, CTLFLAG_RD,
  99     &bs.balloon_high, 0, "High-mem balloon");
 100
 101 struct balloon_entry {
 102         vm_page_t page;
 103         STAILQ_ENTRY(balloon_entry) list;
 104 };
 105
 106 /* List of ballooned pages, threaded through the mem_map array. */
 107 static STAILQ_HEAD(,balloon_entry) ballooned_pages;
 108
 109 /* Main work function, always executed in process context. */
 110 static void balloon_process(void *unused);
 111
 112 #define IPRINTK(fmt, args...) \
 113         printk(KERN_INFO "xen_mem: " fmt, ##args)
 114 #define WPRINTK(fmt, args...) \
 115         printk(KERN_WARNING "xen_mem: " fmt, ##args)
 116
 117 /* balloon_append: add the given page to the balloon. */
 118 static void
 119 balloon_append(vm_page_t page)
 120 {
 121         struct balloon_entry *entry;
 122
 123         entry = malloc(sizeof(struct balloon_entry), M_BALLOON, M_WAITOK);
 124         entry->page = page;
 125         STAILQ_INSERT_HEAD(&ballooned_pages, entry, list);
 126         bs.balloon_low++;
 127 }
 128
 129 /* balloon_retrieve: rescue a page from the balloon, if it is not empty. */
 130 static vm_page_t
 131 balloon_retrieve(void)
 132 {
 133         vm_page_t page;
 134         struct balloon_entry *entry;
 135
 136         if (STAILQ_EMPTY(&ballooned_pages))
 137                 return NULL;
 138
 139         entry = STAILQ_FIRST(&ballooned_pages);
 140         STAILQ_REMOVE_HEAD(&ballooned_pages, list);
 141
 142         page = entry->page;
 143         free(entry, M_DEVBUF);
 144
 145         bs.balloon_low--;
 146
 147         return page;
 148 }
 149
 150 static void
 151 balloon_alarm(void *unused)
 152 {
 153         wakeup(balloon_process);
 154 }
 155
 156 static unsigned long
 157 current_target(void)
 158 {
 159         unsigned long target = min(bs.target_pages, bs.hard_limit);
 160         if (target > (bs.current_pages + bs.balloon_low + bs.balloon_high))
 161                 target = bs.current_pages + bs.balloon_low + bs.balloon_high;
 162         return target;
 163 }
 164
 165 static unsigned long
 166 minimum_target(void)
 167 {
 168 #ifdef XENHVM
 169 #define max_pfn physmem
 170 #else
 171 #define max_pfn HYPERVISOR_shared_info->arch.max_pfn
 172 #endif
 173         unsigned long min_pages, curr_pages = current_target();
 174
 175 #define MB2PAGES(mb) ((mb) << (20 - PAGE_SHIFT))
 176         /* Simple continuous piecewiese linear function:
 177          *  max MiB -> min MiB  gradient
 178          *       0         0
 179          *      16        16
 180          *      32        24
 181          *     128        72    (1/2)
 182          *     512       168    (1/4)
 183          *    2048       360    (1/8)
 184          *    8192       552    (1/32)
 185          *   32768      1320
 186          *  131072      4392
 187          */
 188         if (max_pfn < MB2PAGES(128))
 189                 min_pages = MB2PAGES(8) + (max_pfn >> 1);
 190         else if (max_pfn < MB2PAGES(512))
 191                 min_pages = MB2PAGES(40) + (max_pfn >> 2);
 192         else if (max_pfn < MB2PAGES(2048))
 193                 min_pages = MB2PAGES(104) + (max_pfn >> 3);
 194         else
 195                 min_pages = MB2PAGES(296) + (max_pfn >> 5);
 196 #undef MB2PAGES
 197
 198         /* Don't enforce growth */
 199         return min(min_pages, curr_pages);
 200 #ifndef CONFIG_XEN
 201 #undef max_pfn
 202 #endif
 203 }
 204
 205 static int
 206 increase_reservation(unsigned long nr_pages)
 207 {
 208         unsigned long  pfn, i;
 209         struct balloon_entry *entry;
 210         vm_page_t      page;
 211         long           rc;
 212         struct xen_memory_reservation reservation = {
 213                 .address_bits = 0,
 214                 .extent_order = 0,
 215                 .domid        = DOMID_SELF
 216         };
 217
 218         if (nr_pages > ARRAY_SIZE(frame_list))
 219                 nr_pages = ARRAY_SIZE(frame_list);
 220
 221         mtx_lock(&balloon_lock);
 222
 223         for (entry = STAILQ_FIRST(&ballooned_pages), i = 0;
 224              i < nr_pages; i++, entry = STAILQ_NEXT(entry, list)) {
 225                 KASSERT(entry, ("ballooned_pages list corrupt"));
 226                 page = entry->page;
 227                 frame_list[i] = (VM_PAGE_TO_PHYS(page) >> PAGE_SHIFT);
 228         }
 229
 230         set_xen_guest_handle(reservation.extent_start, frame_list);
 231         reservation.nr_extents   = nr_pages;
 232         rc = HYPERVISOR_memory_op(
 233                 XENMEM_populate_physmap, &reservation);
 234         if (rc < nr_pages) {
 235                 if (rc > 0) {
 236                         int ret;
 237
 238                         /* We hit the Xen hard limit: reprobe. */
 239                         reservation.nr_extents = rc;
 240                         ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation,
 241                                         &reservation);
 242                         KASSERT(ret == rc, ("HYPERVISOR_memory_op failed"));
 243                 }
 244                 if (rc >= 0)
 245                         bs.hard_limit = (bs.current_pages + rc -
 246                                          bs.driver_pages);
 247                 goto out;
 248         }
 249
 250         for (i = 0; i < nr_pages; i++) {
 251                 page = balloon_retrieve();
 252                 KASSERT(page, ("balloon_retrieve failed"));
 253
 254                 pfn = (VM_PAGE_TO_PHYS(page) >> PAGE_SHIFT);
 255                 KASSERT((xen_feature(XENFEAT_auto_translated_physmap) ||
 256                         !phys_to_machine_mapping_valid(pfn)),
 257                     ("auto translated physmap but mapping is valid"));
 258
 259                 set_phys_to_machine(pfn, frame_list[i]);
 260
 261 #if 0
 262 #ifndef XENHVM
 263                 /* Link back into the page tables if not highmem. */
 264                 if (pfn < max_low_pfn) {
 265                         int ret;
 266                         ret = HYPERVISOR_update_va_mapping(
 267                                 (unsigned long)__va(pfn << PAGE_SHIFT),
 268                                 pfn_pte_ma(frame_list[i], PAGE_KERNEL),
 269                                 0);
 270                         PASSING(ret == 0,
 271                             ("HYPERVISOR_update_va_mapping failed"));
 272                 }
 273 #endif
 274 #endif
 275
 276                 /* Relinquish the page back to the allocator. */
 277                 vm_page_unwire(page, 0);
 278                 vm_page_free(page);
 279         }
 280
 281         bs.current_pages += nr_pages;
 282         //totalram_pages = bs.current_pages;
 283
 284  out:
 285         mtx_unlock(&balloon_lock);
 286
 287         return 0;
 288 }
 289
 290 static int
 291 decrease_reservation(unsigned long nr_pages)
 292 {
 293         unsigned long  pfn, i;
 294         vm_page_t      page;
 295         int            need_sleep = 0;
 296         int ret;
 297         struct xen_memory_reservation reservation = {
 298                 .address_bits = 0,
 299                 .extent_order = 0,
 300                 .domid        = DOMID_SELF
 301         };
 302
 303         if (nr_pages > ARRAY_SIZE(frame_list))
 304                 nr_pages = ARRAY_SIZE(frame_list);
 305
 306         for (i = 0; i < nr_pages; i++) {
 307                 int color = 0;
 308                 if ((page = vm_page_alloc(NULL, color++,
 309                             VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ |
 310                             VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
 311                         nr_pages = i;
 312                         need_sleep = 1;
 313                         break;
 314                 }
 315
 316                 pfn = (VM_PAGE_TO_PHYS(page) >> PAGE_SHIFT);
 317                 frame_list[i] = PFNTOMFN(pfn);
 318
 319 #if 0
 320                 if (!PageHighMem(page)) {
 321                         v = phys_to_virt(pfn << PAGE_SHIFT);
 322                         scrub_pages(v, 1);
 323 #ifdef CONFIG_XEN
 324                         ret = HYPERVISOR_update_va_mapping(
 325                                 (unsigned long)v, __pte_ma(0), 0);
 326                         BUG_ON(ret);
 327 #endif
 328                 }
 329 #endif
 330 #ifdef CONFIG_XEN_SCRUB_PAGES
 331                 else {
 332                         v = kmap(page);
 333                         scrub_pages(v, 1);
 334                         kunmap(page);
 335                 }
 336 #endif
 337         }
 338
 339 #ifdef CONFIG_XEN
 340         /* Ensure that ballooned highmem pages don't have kmaps. */
 341         kmap_flush_unused();
 342         flush_tlb_all();
 343 #endif
 344
 345         mtx_lock(&balloon_lock);
 346
 347         /* No more mappings: invalidate P2M and add to balloon. */
 348         for (i = 0; i < nr_pages; i++) {
 349                 pfn = MFNTOPFN(frame_list[i]);
 350                 set_phys_to_machine(pfn, INVALID_P2M_ENTRY);
 351                 balloon_append(PHYS_TO_VM_PAGE(pfn << PAGE_SHIFT));
 352         }
 353
 354         set_xen_guest_handle(reservation.extent_start, frame_list);
 355         reservation.nr_extents   = nr_pages;
 356         ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation, &reservation);
 357         KASSERT(ret == nr_pages, ("HYPERVISOR_memory_op failed"));
 358
 359         bs.current_pages -= nr_pages;
 360         //totalram_pages = bs.current_pages;
 361
 362         mtx_unlock(&balloon_lock);
 363
 364         return (need_sleep);
 365 }
 366
 367 /*
 368  * We avoid multiple worker processes conflicting via the balloon mutex.
 369  * We may of course race updates of the target counts (which are protected
 370  * by the balloon lock), or with changes to the Xen hard limit, but we will
 371  * recover from these in time.
 372  */
 373 static void
 374 balloon_process(void *unused)
 375 {
 376         int need_sleep = 0;
 377         long credit;
 378
 379         mtx_lock(&balloon_mutex);
 380         for (;;) {
 381                 do {
 382                         credit = current_target() - bs.current_pages;
 383                         if (credit > 0)
 384                                 need_sleep = (increase_reservation(credit) != 0);
 385                         if (credit < 0)
 386                                 need_sleep = (decrease_reservation(-credit) != 0);
 387
 388                 } while ((credit != 0) && !need_sleep);
 389
 390                 /* Schedule more work if there is some still to be done. */
 391                 if (current_target() != bs.current_pages)
 392                         timeout(balloon_alarm, NULL, ticks + hz);
 393
 394                 msleep(balloon_process, &balloon_mutex, 0, "balloon", -1);
 395         }
 396         mtx_unlock(&balloon_mutex);
 397 }
 398
 399 /* Resets the Xen limit, sets new target, and kicks off processing. */
 400 static void
 401 set_new_target(unsigned long target)
 402 {
 403         /* No need for lock. Not read-modify-write updates. */
 404         bs.hard_limit   = ~0UL;
 405         bs.target_pages = max(target, minimum_target());
 406         wakeup(balloon_process);
 407 }
 408
 409 static struct xenbus_watch target_watch =
 410 {
 411         .node = "memory/target"
 412 };
 413
 414 /* React to a change in the target key */
 415 static void
 416 watch_target(struct xenbus_watch *watch,
 417              const char **vec, unsigned int len)
 418 {
 419         unsigned long long new_target;
 420         int err;
 421
 422         err = xenbus_scanf(XBT_NIL, "memory", "target", NULL,
 423             "%llu", &new_target);
 424         if (err) {
 425                 /* This is ok (for domain0 at least) - so just return */
 426                 return;
 427         }
 428
 429         /* The given memory/target value is in KiB, so it needs converting to
 430            pages.  PAGE_SHIFT converts bytes to pages, hence PAGE_SHIFT - 10.
 431         */
 432         set_new_target(new_target >> (PAGE_SHIFT - 10));
 433
 434 }
 435
 436 static void
 437 balloon_init_watcher(void *arg)
 438 {
 439         int err;
 440
 441         err = register_xenbus_watch(&target_watch);
 442         if (err)
 443                 printf("Failed to set balloon watcher\n");
 444
 445 }
 446 SYSINIT(balloon_init_watcher, SI_SUB_PSEUDO, SI_ORDER_ANY,
 447     balloon_init_watcher, NULL);
 448
 449 static void
 450 balloon_init(void *arg)
 451 {
 452 #ifndef XENHVM
 453         vm_page_t page;
 454         unsigned long pfn;
 455
 456 #define max_pfn HYPERVISOR_shared_info->arch.max_pfn
 457 #endif
 458
 459         if (!is_running_on_xen())
 460                 return;
 461
 462         mtx_init(&balloon_lock, "balloon_lock", NULL, MTX_DEF);
 463         mtx_init(&balloon_mutex, "balloon_mutex", NULL, MTX_DEF);
 464
 465 #ifndef XENHVM
 466         bs.current_pages = min(xen_start_info->nr_pages, max_pfn);
 467 #else
 468         bs.current_pages = physmem;
 469 #endif
 470         bs.target_pages  = bs.current_pages;
 471         bs.balloon_low   = 0;
 472         bs.balloon_high  = 0;
 473         bs.driver_pages  = 0UL;
 474         bs.hard_limit    = ~0UL;
 475
 476         kproc_create(balloon_process, NULL, NULL, 0, 0, "balloon");
 477 //      init_timer(&balloon_timer);
 478 //      balloon_timer.data = 0;
 479 //      balloon_timer.function = balloon_alarm;
 480
 481 #ifndef XENHVM
 482         /* Initialise the balloon with excess memory space. */
 483         for (pfn = xen_start_info->nr_pages; pfn < max_pfn; pfn++) {
 484                 page = PHYS_TO_VM_PAGE(pfn << PAGE_SHIFT);
 485                 balloon_append(page);
 486         }
 487 #undef max_pfn
 488 #endif
 489
 490         target_watch.callback = watch_target;
 491
 492         return;
 493 }
 494 SYSINIT(balloon_init, SI_SUB_PSEUDO, SI_ORDER_ANY, balloon_init, NULL);
 495
 496 void balloon_update_driver_allowance(long delta);
 497
 498 void
 499 balloon_update_driver_allowance(long delta)
 500 {
 501         mtx_lock(&balloon_lock);
 502         bs.driver_pages += delta;
 503         mtx_unlock(&balloon_lock);
 504 }
 505
 506 #if 0
 507 static int dealloc_pte_fn(
 508         pte_t *pte, struct page *pte_page, unsigned long addr, void *data)
 509 {
 510         unsigned long mfn = pte_mfn(*pte);
 511         int ret;
 512         struct xen_memory_reservation reservation = {
 513                 .extent_start = &mfn,
 514                 .nr_extents   = 1,
 515                 .extent_order = 0,
 516                 .domid        = DOMID_SELF
 517         };
 518         set_pte_at(&init_mm, addr, pte, __pte_ma(0));
 519         set_phys_to_machine(__pa(addr) >> PAGE_SHIFT, INVALID_P2M_ENTRY);
 520         ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation, &reservation);
 521         KASSERT(ret == 1, ("HYPERVISOR_memory_op failed"));
 522         return 0;
 523 }
 524
 525 #endif
 526
 527 #if 0
 528 vm_page_t
 529 balloon_alloc_empty_page_range(unsigned long nr_pages)
 530 {
 531         vm_page_t pages;
 532         int i, rc;
 533         unsigned long *mfn_list;
 534         struct xen_memory_reservation reservation = {
 535                 .address_bits = 0,
 536                 .extent_order = 0,
 537                 .domid        = DOMID_SELF
 538         };
 539
 540         pages = vm_page_alloc_contig(nr_pages, 0, -1, 4, 4)
 541         if (pages == NULL)
 542                 return NULL;
 543
 544         mfn_list = malloc(nr_pages*sizeof(unsigned long), M_DEVBUF, M_WAITOK);
 545
 546         for (i = 0; i < nr_pages; i++) {
 547                 mfn_list[i] = PFNTOMFN(VM_PAGE_TO_PHYS(pages[i]) >> PAGE_SHIFT);
 548                 PFNTOMFN(i) = INVALID_P2M_ENTRY;
 549                 reservation.extent_start = mfn_list;
 550                 reservation.nr_extents = nr_pages;
 551                 rc = HYPERVISOR_memory_op(XENMEM_decrease_reservation,
 552                     &reservation);
 553                 KASSERT(rc == nr_pages, ("HYPERVISOR_memory_op failed"));
 554         }
 555
 556         current_pages -= nr_pages;
 557
 558         wakeup(balloon_process);
 559
 560         return pages;
 561 }
 562
 563 void
 564 balloon_dealloc_empty_page_range(vm_page_t page, unsigned long nr_pages)
 565 {
 566         unsigned long i;
 567
 568         for (i = 0; i < nr_pages; i++)
 569                 balloon_append(page + i);
 570
 571         wakeup(balloon_process);
 572 }
 573 #endif