sys/dev/xen/balloon/balloon.c

   1 /******************************************************************************
   2  * balloon.c
   3  *
   4  * Xen balloon driver - enables returning/claiming memory to/from Xen.
   5  *
   6  * Copyright (c) 2003, B Dragovic
   7  * Copyright (c) 2003-2004, M Williamson, K Fraser
   8  * Copyright (c) 2005 Dan M. Smith, IBM Corporation
   9  *
  10  * This file may be distributed separately from the Linux kernel, or
  11  * incorporated into other software packages, subject to the following license:
  12  *
  13  * Permission is hereby granted, free of charge, to any person obtaining a copy
  14  * of this source file (the "Software"), to deal in the Software without
  15  * restriction, including without limitation the rights to use, copy, modify,
  16  * merge, publish, distribute, sublicense, and/or sell copies of the Software,
  17  * and to permit persons to whom the Software is furnished to do so, subject to
  18  * the following conditions:
  19  *
  20  * The above copyright notice and this permission notice shall be included in
  21  * all copies or substantial portions of the Software.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  24  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  25  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  26  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  27  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  28  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  29  * IN THE SOFTWARE.
  30  */
  31
  32 #include <sys/cdefs.h>
  33 __FBSDID("$FreeBSD$");
  34
  35 #include <sys/param.h>
  36 #include <sys/lock.h>
  37 #include <sys/kernel.h>
  38 #include <sys/kthread.h>
  39 #include <sys/malloc.h>
  40 #include <sys/mutex.h>
  41 #include <sys/sysctl.h>
  42
  43 #include <machine/xen/xen-os.h>
  44 #include <machine/xen/xenvar.h>
  45 #include <machine/xen/xenfunc.h>
  46 #include <xen/hypervisor.h>
  47 #include <xen/xenstore/xenstorevar.h>
  48
  49 #include <vm/vm.h>
  50 #include <vm/vm_page.h>
  51
  52 static MALLOC_DEFINE(M_BALLOON, "Balloon", "Xen Balloon Driver");
  53
  54 struct mtx balloon_mutex;
  55
  56 /*
  57  * Protects atomic reservation decrease/increase against concurrent increases.
  58  * Also protects non-atomic updates of current_pages and driver_pages, and
  59  * balloon lists.
  60  */
  61 struct mtx balloon_lock;
  62
  63 /* We increase/decrease in batches which fit in a page */
  64 static unsigned long frame_list[PAGE_SIZE / sizeof(unsigned long)];
  65 #define ARRAY_SIZE(A)   (sizeof(A) / sizeof(A[0]))
  66
  67 struct balloon_stats {
  68         /* We aim for 'current allocation' == 'target allocation'. */
  69         unsigned long current_pages;
  70         unsigned long target_pages;
  71         /* We may hit the hard limit in Xen. If we do then we remember it. */
  72         unsigned long hard_limit;
  73         /*
  74          * Drivers may alter the memory reservation independently, but they
  75          * must inform the balloon driver so we avoid hitting the hard limit.
  76          */
  77         unsigned long driver_pages;
  78         /* Number of pages in high- and low-memory balloons. */
  79         unsigned long balloon_low;
  80         unsigned long balloon_high;
  81 };
  82
  83 static struct balloon_stats balloon_stats;
  84 #define bs balloon_stats
  85
  86 SYSCTL_DECL(_dev_xen);
  87 static SYSCTL_NODE(_dev_xen, OID_AUTO, balloon, CTLFLAG_RD, NULL, "Balloon");
  88 SYSCTL_ULONG(_dev_xen_balloon, OID_AUTO, current, CTLFLAG_RD,
  89     &bs.current_pages, 0, "Current allocation");
  90 SYSCTL_ULONG(_dev_xen_balloon, OID_AUTO, target, CTLFLAG_RD,
  91     &bs.target_pages, 0, "Target allocation");
  92 SYSCTL_ULONG(_dev_xen_balloon, OID_AUTO, driver_pages, CTLFLAG_RD,
  93     &bs.driver_pages, 0, "Driver pages");
  94 SYSCTL_ULONG(_dev_xen_balloon, OID_AUTO, hard_limit, CTLFLAG_RD,
  95     &bs.hard_limit, 0, "Xen hard limit");
  96 SYSCTL_ULONG(_dev_xen_balloon, OID_AUTO, low_mem, CTLFLAG_RD,
  97     &bs.balloon_low, 0, "Low-mem balloon");
  98 SYSCTL_ULONG(_dev_xen_balloon, OID_AUTO, high_mem, CTLFLAG_RD,
  99     &bs.balloon_high, 0, "High-mem balloon");
 100
 101 struct balloon_entry {
 102         vm_page_t page;
 103         STAILQ_ENTRY(balloon_entry) list;
 104 };
 105
 106 /* List of ballooned pages, threaded through the mem_map array. */
 107 static STAILQ_HEAD(,balloon_entry) ballooned_pages;
 108
 109 /* Main work function, always executed in process context. */
 110 static void balloon_process(void *unused);
 111
 112 #define IPRINTK(fmt, args...) \
 113         printk(KERN_INFO "xen_mem: " fmt, ##args)
 114 #define WPRINTK(fmt, args...) \
 115         printk(KERN_WARNING "xen_mem: " fmt, ##args)
 116
 117 /* balloon_append: add the given page to the balloon. */
 118 static void
 119 balloon_append(vm_page_t page)
 120 {
 121         struct balloon_entry *entry;
 122
 123         entry = malloc(sizeof(struct balloon_entry), M_BALLOON, M_WAITOK);
 124         entry->page = page;
 125         STAILQ_INSERT_HEAD(&ballooned_pages, entry, list);
 126         bs.balloon_low++;
 127 }
 128
 129 /* balloon_retrieve: rescue a page from the balloon, if it is not empty. */
 130 static vm_page_t
 131 balloon_retrieve(void)
 132 {
 133         vm_page_t page;
 134         struct balloon_entry *entry;
 135
 136         if (STAILQ_EMPTY(&ballooned_pages))
 137                 return NULL;
 138
 139         entry = STAILQ_FIRST(&ballooned_pages);
 140         STAILQ_REMOVE_HEAD(&ballooned_pages, list);
 141
 142         page = entry->page;
 143         free(entry, M_BALLOON);
 144
 145         bs.balloon_low--;
 146
 147         return page;
 148 }
 149
 150 static unsigned long
 151 current_target(void)
 152 {
 153         unsigned long target = min(bs.target_pages, bs.hard_limit);
 154         if (target > (bs.current_pages + bs.balloon_low + bs.balloon_high))
 155                 target = bs.current_pages + bs.balloon_low + bs.balloon_high;
 156         return target;
 157 }
 158
 159 static unsigned long
 160 minimum_target(void)
 161 {
 162 #ifdef XENHVM
 163 #define max_pfn physmem
 164 #else
 165 #define max_pfn HYPERVISOR_shared_info->arch.max_pfn
 166 #endif
 167         unsigned long min_pages, curr_pages = current_target();
 168
 169 #define MB2PAGES(mb) ((mb) << (20 - PAGE_SHIFT))
 170         /* Simple continuous piecewiese linear function:
 171          *  max MiB -> min MiB  gradient
 172          *       0         0
 173          *      16        16
 174          *      32        24
 175          *     128        72    (1/2)
 176          *     512       168    (1/4)
 177          *    2048       360    (1/8)
 178          *    8192       552    (1/32)
 179          *   32768      1320
 180          *  131072      4392
 181          */
 182         if (max_pfn < MB2PAGES(128))
 183                 min_pages = MB2PAGES(8) + (max_pfn >> 1);
 184         else if (max_pfn < MB2PAGES(512))
 185                 min_pages = MB2PAGES(40) + (max_pfn >> 2);
 186         else if (max_pfn < MB2PAGES(2048))
 187                 min_pages = MB2PAGES(104) + (max_pfn >> 3);
 188         else
 189                 min_pages = MB2PAGES(296) + (max_pfn >> 5);
 190 #undef MB2PAGES
 191
 192         /* Don't enforce growth */
 193         return min(min_pages, curr_pages);
 194 #ifndef CONFIG_XEN
 195 #undef max_pfn
 196 #endif
 197 }
 198
 199 static int
 200 increase_reservation(unsigned long nr_pages)
 201 {
 202         unsigned long  pfn, i;
 203         struct balloon_entry *entry;
 204         vm_page_t      page;
 205         long           rc;
 206         struct xen_memory_reservation reservation = {
 207                 .address_bits = 0,
 208                 .extent_order = 0,
 209                 .domid        = DOMID_SELF
 210         };
 211
 212         if (nr_pages > ARRAY_SIZE(frame_list))
 213                 nr_pages = ARRAY_SIZE(frame_list);
 214
 215         mtx_lock(&balloon_lock);
 216
 217         for (entry = STAILQ_FIRST(&ballooned_pages), i = 0;
 218              i < nr_pages; i++, entry = STAILQ_NEXT(entry, list)) {
 219                 KASSERT(entry, ("ballooned_pages list corrupt"));
 220                 page = entry->page;
 221                 frame_list[i] = (VM_PAGE_TO_PHYS(page) >> PAGE_SHIFT);
 222         }
 223
 224         set_xen_guest_handle(reservation.extent_start, frame_list);
 225         reservation.nr_extents   = nr_pages;
 226         rc = HYPERVISOR_memory_op(
 227                 XENMEM_populate_physmap, &reservation);
 228         if (rc < nr_pages) {
 229                 if (rc > 0) {
 230                         int ret;
 231
 232                         /* We hit the Xen hard limit: reprobe. */
 233                         reservation.nr_extents = rc;
 234                         ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation,
 235                                         &reservation);
 236                         KASSERT(ret == rc, ("HYPERVISOR_memory_op failed"));
 237                 }
 238                 if (rc >= 0)
 239                         bs.hard_limit = (bs.current_pages + rc -
 240                                          bs.driver_pages);
 241                 goto out;
 242         }
 243
 244         for (i = 0; i < nr_pages; i++) {
 245                 page = balloon_retrieve();
 246                 KASSERT(page, ("balloon_retrieve failed"));
 247
 248                 pfn = (VM_PAGE_TO_PHYS(page) >> PAGE_SHIFT);
 249                 KASSERT((xen_feature(XENFEAT_auto_translated_physmap) ||
 250                         !phys_to_machine_mapping_valid(pfn)),
 251                     ("auto translated physmap but mapping is valid"));
 252
 253                 set_phys_to_machine(pfn, frame_list[i]);
 254
 255 #if 0
 256 #ifndef XENHVM
 257                 /* Link back into the page tables if not highmem. */
 258                 if (pfn < max_low_pfn) {
 259                         int ret;
 260                         ret = HYPERVISOR_update_va_mapping(
 261                                 (unsigned long)__va(pfn << PAGE_SHIFT),
 262                                 pfn_pte_ma(frame_list[i], PAGE_KERNEL),
 263                                 0);
 264                         PASSING(ret == 0,
 265                             ("HYPERVISOR_update_va_mapping failed"));
 266                 }
 267 #endif
 268 #endif
 269
 270                 /* Relinquish the page back to the allocator. */
 271                 vm_page_unwire(page, 0);
 272                 vm_page_free(page);
 273         }
 274
 275         bs.current_pages += nr_pages;
 276         //totalram_pages = bs.current_pages;
 277
 278  out:
 279         mtx_unlock(&balloon_lock);
 280
 281         return 0;
 282 }
 283
 284 static int
 285 decrease_reservation(unsigned long nr_pages)
 286 {
 287         unsigned long  pfn, i;
 288         vm_page_t      page;
 289         int            need_sleep = 0;
 290         int ret;
 291         struct xen_memory_reservation reservation = {
 292                 .address_bits = 0,
 293                 .extent_order = 0,
 294                 .domid        = DOMID_SELF
 295         };
 296
 297         if (nr_pages > ARRAY_SIZE(frame_list))
 298                 nr_pages = ARRAY_SIZE(frame_list);
 299
 300         for (i = 0; i < nr_pages; i++) {
 301                 if ((page = vm_page_alloc(NULL, 0,
 302                             VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ |
 303                             VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
 304                         nr_pages = i;
 305                         need_sleep = 1;
 306                         break;
 307                 }
 308
 309                 pfn = (VM_PAGE_TO_PHYS(page) >> PAGE_SHIFT);
 310                 frame_list[i] = PFNTOMFN(pfn);
 311
 312 #if 0
 313                 if (!PageHighMem(page)) {
 314                         v = phys_to_virt(pfn << PAGE_SHIFT);
 315                         scrub_pages(v, 1);
 316 #ifdef CONFIG_XEN
 317                         ret = HYPERVISOR_update_va_mapping(
 318                                 (unsigned long)v, __pte_ma(0), 0);
 319                         BUG_ON(ret);
 320 #endif
 321                 }
 322 #endif
 323 #ifdef CONFIG_XEN_SCRUB_PAGES
 324                 else {
 325                         v = kmap(page);
 326                         scrub_pages(v, 1);
 327                         kunmap(page);
 328                 }
 329 #endif
 330         }
 331
 332 #ifdef CONFIG_XEN
 333         /* Ensure that ballooned highmem pages don't have kmaps. */
 334         kmap_flush_unused();
 335         flush_tlb_all();
 336 #endif
 337
 338         mtx_lock(&balloon_lock);
 339
 340         /* No more mappings: invalidate P2M and add to balloon. */
 341         for (i = 0; i < nr_pages; i++) {
 342                 pfn = MFNTOPFN(frame_list[i]);
 343                 set_phys_to_machine(pfn, INVALID_P2M_ENTRY);
 344                 balloon_append(PHYS_TO_VM_PAGE(pfn << PAGE_SHIFT));
 345         }
 346
 347         set_xen_guest_handle(reservation.extent_start, frame_list);
 348         reservation.nr_extents   = nr_pages;
 349         ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation, &reservation);
 350         KASSERT(ret == nr_pages, ("HYPERVISOR_memory_op failed"));
 351
 352         bs.current_pages -= nr_pages;
 353         //totalram_pages = bs.current_pages;
 354
 355         mtx_unlock(&balloon_lock);
 356
 357         return (need_sleep);
 358 }
 359
 360 /*
 361  * We avoid multiple worker processes conflicting via the balloon mutex.
 362  * We may of course race updates of the target counts (which are protected
 363  * by the balloon lock), or with changes to the Xen hard limit, but we will
 364  * recover from these in time.
 365  */
 366 static void
 367 balloon_process(void *unused)
 368 {
 369         int need_sleep = 0;
 370         long credit;
 371
 372         mtx_lock(&balloon_mutex);
 373         for (;;) {
 374                 int sleep_time;
 375
 376                 do {
 377                         credit = current_target() - bs.current_pages;
 378                         if (credit > 0)
 379                                 need_sleep = (increase_reservation(credit) != 0);
 380                         if (credit < 0)
 381                                 need_sleep = (decrease_reservation(-credit) != 0);
 382
 383                 } while ((credit != 0) && !need_sleep);
 384
 385                 /* Schedule more work if there is some still to be done. */
 386                 if (current_target() != bs.current_pages)
 387                         sleep_time = hz;
 388                 else
 389                         sleep_time = 0;
 390
 391                 msleep(balloon_process, &balloon_mutex, 0, "balloon",
 392                        sleep_time);
 393         }
 394         mtx_unlock(&balloon_mutex);
 395 }
 396
 397 /* Resets the Xen limit, sets new target, and kicks off processing. */
 398 static void
 399 set_new_target(unsigned long target)
 400 {
 401         /* No need for lock. Not read-modify-write updates. */
 402         bs.hard_limit   = ~0UL;
 403         bs.target_pages = max(target, minimum_target());
 404         wakeup(balloon_process);
 405 }
 406
 407 static struct xs_watch target_watch =
 408 {
 409         .node = "memory/target"
 410 };
 411
 412 /* React to a change in the target key */
 413 static void
 414 watch_target(struct xs_watch *watch,
 415              const char **vec, unsigned int len)
 416 {
 417         unsigned long long new_target;
 418         int err;
 419
 420         err = xs_scanf(XST_NIL, "memory", "target", NULL,
 421             "%llu", &new_target);
 422         if (err) {
 423                 /* This is ok (for domain0 at least) - so just return */
 424                 return;
 425         }
 426
 427         /* The given memory/target value is in KiB, so it needs converting to
 428            pages.  PAGE_SHIFT converts bytes to pages, hence PAGE_SHIFT - 10.
 429         */
 430         set_new_target(new_target >> (PAGE_SHIFT - 10));
 431
 432 }
 433
 434 static void
 435 balloon_init_watcher(void *arg)
 436 {
 437         int err;
 438
 439         if (!is_running_on_xen())
 440                 return;
 441
 442         err = xs_register_watch(&target_watch);
 443         if (err)
 444                 printf("Failed to set balloon watcher\n");
 445
 446 }
 447 SYSINIT(balloon_init_watcher, SI_SUB_PSEUDO, SI_ORDER_ANY,
 448     balloon_init_watcher, NULL);
 449
 450 static void
 451 balloon_init(void *arg)
 452 {
 453 #ifndef XENHVM
 454         vm_page_t page;
 455         unsigned long pfn;
 456
 457 #define max_pfn HYPERVISOR_shared_info->arch.max_pfn
 458 #endif
 459
 460         if (!is_running_on_xen())
 461                 return;
 462
 463         mtx_init(&balloon_lock, "balloon_lock", NULL, MTX_DEF);
 464         mtx_init(&balloon_mutex, "balloon_mutex", NULL, MTX_DEF);
 465
 466 #ifndef XENHVM
 467         bs.current_pages = min(xen_start_info->nr_pages, max_pfn);
 468 #else
 469         bs.current_pages = physmem;
 470 #endif
 471         bs.target_pages  = bs.current_pages;
 472         bs.balloon_low   = 0;
 473         bs.balloon_high  = 0;
 474         bs.driver_pages  = 0UL;
 475         bs.hard_limit    = ~0UL;
 476
 477         kproc_create(balloon_process, NULL, NULL, 0, 0, "balloon");
 478
 479 #ifndef XENHVM
 480         /* Initialise the balloon with excess memory space. */
 481         for (pfn = xen_start_info->nr_pages; pfn < max_pfn; pfn++) {
 482                 page = PHYS_TO_VM_PAGE(pfn << PAGE_SHIFT);
 483                 balloon_append(page);
 484         }
 485 #undef max_pfn
 486 #endif
 487
 488         target_watch.callback = watch_target;
 489
 490         return;
 491 }
 492 SYSINIT(balloon_init, SI_SUB_PSEUDO, SI_ORDER_ANY, balloon_init, NULL);
 493
 494 void balloon_update_driver_allowance(long delta);
 495
 496 void
 497 balloon_update_driver_allowance(long delta)
 498 {
 499         mtx_lock(&balloon_lock);
 500         bs.driver_pages += delta;
 501         mtx_unlock(&balloon_lock);
 502 }
 503
 504 #if 0
 505 static int dealloc_pte_fn(
 506         pte_t *pte, struct page *pte_page, unsigned long addr, void *data)
 507 {
 508         unsigned long mfn = pte_mfn(*pte);
 509         int ret;
 510         struct xen_memory_reservation reservation = {
 511                 .extent_start = &mfn,
 512                 .nr_extents   = 1,
 513                 .extent_order = 0,
 514                 .domid        = DOMID_SELF
 515         };
 516         set_pte_at(&init_mm, addr, pte, __pte_ma(0));
 517         set_phys_to_machine(__pa(addr) >> PAGE_SHIFT, INVALID_P2M_ENTRY);
 518         ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation, &reservation);
 519         KASSERT(ret == 1, ("HYPERVISOR_memory_op failed"));
 520         return 0;
 521 }
 522
 523 #endif
 524
 525 #if 0
 526 vm_page_t
 527 balloon_alloc_empty_page_range(unsigned long nr_pages)
 528 {
 529         vm_page_t pages;
 530         int i, rc;
 531         unsigned long *mfn_list;
 532         struct xen_memory_reservation reservation = {
 533                 .address_bits = 0,
 534                 .extent_order = 0,
 535                 .domid        = DOMID_SELF
 536         };
 537
 538         pages = vm_page_alloc_contig(nr_pages, 0, -1, 4, 4)
 539         if (pages == NULL)
 540                 return NULL;
 541
 542         mfn_list = malloc(nr_pages*sizeof(unsigned long), M_DEVBUF, M_WAITOK);
 543
 544         for (i = 0; i < nr_pages; i++) {
 545                 mfn_list[i] = PFNTOMFN(VM_PAGE_TO_PHYS(pages[i]) >> PAGE_SHIFT);
 546                 PFNTOMFN(i) = INVALID_P2M_ENTRY;
 547                 reservation.extent_start = mfn_list;
 548                 reservation.nr_extents = nr_pages;
 549                 rc = HYPERVISOR_memory_op(XENMEM_decrease_reservation,
 550                     &reservation);
 551                 KASSERT(rc == nr_pages, ("HYPERVISOR_memory_op failed"));
 552         }
 553
 554         current_pages -= nr_pages;
 555
 556         wakeup(balloon_process);
 557
 558         return pages;
 559 }
 560
 561 void
 562 balloon_dealloc_empty_page_range(vm_page_t page, unsigned long nr_pages)
 563 {
 564         unsigned long i;
 565
 566         for (i = 0; i < nr_pages; i++)
 567                 balloon_append(page + i);
 568
 569         wakeup(balloon_process);
 570 }
 571 #endif