sys/dev/drm2/ttm/ttm_page_alloc.c

   1 /*
   2  * Copyright (c) Red Hat Inc.
   3
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sub license,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the
  12  * next paragraph) shall be included in all copies or substantial portions
  13  * of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors: Dave Airlie <airlied@redhat.com>
  24  *          Jerome Glisse <jglisse@redhat.com>
  25  *          Pauli Nieminen <suokkos@gmail.com>
  26  */
  27 /*
  28  * Copyright (c) 2013 The FreeBSD Foundation
  29  * All rights reserved.
  30  *
  31  * Portions of this software were developed by Konstantin Belousov
  32  * <kib@FreeBSD.org> under sponsorship from the FreeBSD Foundation.
  33  */
  34
  35 /* simple list based uncached page pool
  36  * - Pool collects resently freed pages for reuse
  37  * - Use page->lru to keep a free list
  38  * - doesn't track currently in use pages
  39  */
  40
  41 #include <sys/cdefs.h>
  42 __FBSDID("$FreeBSD$");
  43
  44 #include <dev/drm2/drmP.h>
  45 #include <dev/drm2/ttm/ttm_bo_driver.h>
  46 #include <dev/drm2/ttm/ttm_page_alloc.h>
  47 #include <vm/vm_pageout.h>
  48
  49 #define NUM_PAGES_TO_ALLOC              (PAGE_SIZE/sizeof(vm_page_t))
  50 #define SMALL_ALLOCATION                16
  51 #define FREE_ALL_PAGES                  (~0U)
  52 /* times are in msecs */
  53 #define PAGE_FREE_INTERVAL              1000
  54
  55 /**
  56  * struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages.
  57  *
  58  * @lock: Protects the shared pool from concurrnet access. Must be used with
  59  * irqsave/irqrestore variants because pool allocator maybe called from
  60  * delayed work.
  61  * @fill_lock: Prevent concurrent calls to fill.
  62  * @list: Pool of free uc/wc pages for fast reuse.
  63  * @gfp_flags: Flags to pass for alloc_page.
  64  * @npages: Number of pages in pool.
  65  */
  66 struct ttm_page_pool {
  67         struct mtx              lock;
  68         bool                    fill_lock;
  69         bool                    dma32;
  70         struct pglist           list;
  71         int                     ttm_page_alloc_flags;
  72         unsigned                npages;
  73         char                    *name;
  74         unsigned long           nfrees;
  75         unsigned long           nrefills;
  76 };
  77
  78 /**
  79  * Limits for the pool. They are handled without locks because only place where
  80  * they may change is in sysfs store. They won't have immediate effect anyway
  81  * so forcing serialization to access them is pointless.
  82  */
  83
  84 struct ttm_pool_opts {
  85         unsigned        alloc_size;
  86         unsigned        max_size;
  87         unsigned        small;
  88 };
  89
  90 #define NUM_POOLS 4
  91
  92 /**
  93  * struct ttm_pool_manager - Holds memory pools for fst allocation
  94  *
  95  * Manager is read only object for pool code so it doesn't need locking.
  96  *
  97  * @free_interval: minimum number of jiffies between freeing pages from pool.
  98  * @page_alloc_inited: reference counting for pool allocation.
  99  * @work: Work that is used to shrink the pool. Work is only run when there is
 100  * some pages to free.
 101  * @small_allocation: Limit in number of pages what is small allocation.
 102  *
 103  * @pools: All pool objects in use.
 104  **/
 105 struct ttm_pool_manager {
 106         unsigned int kobj_ref;
 107         eventhandler_tag lowmem_handler;
 108         struct ttm_pool_opts    options;
 109
 110         union {
 111                 struct ttm_page_pool    u_pools[NUM_POOLS];
 112                 struct _utag {
 113                         struct ttm_page_pool    u_wc_pool;
 114                         struct ttm_page_pool    u_uc_pool;
 115                         struct ttm_page_pool    u_wc_pool_dma32;
 116                         struct ttm_page_pool    u_uc_pool_dma32;
 117                 } _ut;
 118         } _u;
 119 };
 120
 121 #define pools _u.u_pools
 122 #define wc_pool _u._ut.u_wc_pool
 123 #define uc_pool _u._ut.u_uc_pool
 124 #define wc_pool_dma32 _u._ut.u_wc_pool_dma32
 125 #define uc_pool_dma32 _u._ut.u_uc_pool_dma32
 126
 127 MALLOC_DEFINE(M_TTM_POOLMGR, "ttm_poolmgr", "TTM Pool Manager");
 128
 129 static void
 130 ttm_vm_page_free(vm_page_t m)
 131 {
 132
 133         KASSERT(m->object == NULL, ("ttm page %p is owned", m));
 134         KASSERT(m->wire_count == 1, ("ttm lost wire %p", m));
 135         KASSERT((m->flags & PG_FICTITIOUS) != 0, ("ttm lost fictitious %p", m));
 136         KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("ttm got unmanaged %p", m));
 137         m->flags &= ~PG_FICTITIOUS;
 138         m->oflags |= VPO_UNMANAGED;
 139         vm_page_unwire(m, 0);
 140         vm_page_free(m);
 141 }
 142
 143 static vm_memattr_t
 144 ttm_caching_state_to_vm(enum ttm_caching_state cstate)
 145 {
 146
 147         switch (cstate) {
 148         case tt_uncached:
 149                 return (VM_MEMATTR_UNCACHEABLE);
 150         case tt_wc:
 151                 return (VM_MEMATTR_WRITE_COMBINING);
 152         case tt_cached:
 153                 return (VM_MEMATTR_WRITE_BACK);
 154         }
 155         panic("caching state %d\n", cstate);
 156 }
 157
 158 static vm_page_t
 159 ttm_vm_page_alloc_dma32(int req, vm_memattr_t memattr)
 160 {
 161         vm_page_t p;
 162         int tries;
 163
 164         for (tries = 0; ; tries++) {
 165                 p = vm_page_alloc_contig(NULL, 0, req, 1, 0, 0xffffffff,
 166                     PAGE_SIZE, 0, memattr);
 167                 if (p != NULL || tries > 2)
 168                         return (p);
 169
 170                 /*
 171                  * Before growing the cache see if this is just a normal
 172                  * memory shortage.
 173                  */
 174                 VM_WAIT;
 175                 vm_pageout_grow_cache(tries, 0, 0xffffffff);
 176         }
 177 }
 178
 179 static vm_page_t
 180 ttm_vm_page_alloc_any(int req, vm_memattr_t memattr)
 181 {
 182         vm_page_t p;
 183
 184         while (1) {
 185                 p = vm_page_alloc(NULL, 0, req);
 186                 if (p != NULL)
 187                         break;
 188                 VM_WAIT;
 189         }
 190         pmap_page_set_memattr(p, memattr);
 191         return (p);
 192 }
 193
 194 static vm_page_t
 195 ttm_vm_page_alloc(int flags, enum ttm_caching_state cstate)
 196 {
 197         vm_page_t p;
 198         vm_memattr_t memattr;
 199         int req;
 200
 201         memattr = ttm_caching_state_to_vm(cstate);
 202         req = VM_ALLOC_NORMAL | VM_ALLOC_WIRED | VM_ALLOC_NOOBJ;
 203         if ((flags & TTM_PAGE_FLAG_ZERO_ALLOC) != 0)
 204                 req |= VM_ALLOC_ZERO;
 205
 206         if ((flags & TTM_PAGE_FLAG_DMA32) != 0)
 207                 p = ttm_vm_page_alloc_dma32(req, memattr);
 208         else
 209                 p = ttm_vm_page_alloc_any(req, memattr);
 210
 211         if (p != NULL) {
 212                 p->oflags &= ~VPO_UNMANAGED;
 213                 p->flags |= PG_FICTITIOUS;
 214         }
 215         return (p);
 216 }
 217
 218 static void ttm_pool_kobj_release(struct ttm_pool_manager *m)
 219 {
 220
 221         free(m, M_TTM_POOLMGR);
 222 }
 223
 224 #if 0
 225 /* XXXKIB sysctl */
 226 static ssize_t ttm_pool_store(struct ttm_pool_manager *m,
 227                 struct attribute *attr, const char *buffer, size_t size)
 228 {
 229         int chars;
 230         unsigned val;
 231         chars = sscanf(buffer, "%u", &val);
 232         if (chars == 0)
 233                 return size;
 234
 235         /* Convert kb to number of pages */
 236         val = val / (PAGE_SIZE >> 10);
 237
 238         if (attr == &ttm_page_pool_max)
 239                 m->options.max_size = val;
 240         else if (attr == &ttm_page_pool_small)
 241                 m->options.small = val;
 242         else if (attr == &ttm_page_pool_alloc_size) {
 243                 if (val > NUM_PAGES_TO_ALLOC*8) {
 244                         pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n",
 245                                NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7),
 246                                NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10));
 247                         return size;
 248                 } else if (val > NUM_PAGES_TO_ALLOC) {
 249                         pr_warn("Setting allocation size to larger than %lu is not recommended\n",
 250                                 NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10));
 251                 }
 252                 m->options.alloc_size = val;
 253         }
 254
 255         return size;
 256 }
 257
 258 static ssize_t ttm_pool_show(struct ttm_pool_manager *m,
 259                 struct attribute *attr, char *buffer)
 260 {
 261         unsigned val = 0;
 262
 263         if (attr == &ttm_page_pool_max)
 264                 val = m->options.max_size;
 265         else if (attr == &ttm_page_pool_small)
 266                 val = m->options.small;
 267         else if (attr == &ttm_page_pool_alloc_size)
 268                 val = m->options.alloc_size;
 269
 270         val = val * (PAGE_SIZE >> 10);
 271
 272         return snprintf(buffer, PAGE_SIZE, "%u\n", val);
 273 }
 274 #endif
 275
 276 static struct ttm_pool_manager *_manager;
 277
 278 static int set_pages_array_wb(vm_page_t *pages, int addrinarray)
 279 {
 280 #ifdef TTM_HAS_AGP
 281         int i;
 282
 283         for (i = 0; i < addrinarray; i++)
 284                 pmap_page_set_memattr(pages[i], VM_MEMATTR_WRITE_BACK);
 285 #endif
 286         return 0;
 287 }
 288
 289 static int set_pages_array_wc(vm_page_t *pages, int addrinarray)
 290 {
 291 #ifdef TTM_HAS_AGP
 292         int i;
 293
 294         for (i = 0; i < addrinarray; i++)
 295                 pmap_page_set_memattr(pages[i], VM_MEMATTR_WRITE_COMBINING);
 296 #endif
 297         return 0;
 298 }
 299
 300 static int set_pages_array_uc(vm_page_t *pages, int addrinarray)
 301 {
 302 #ifdef TTM_HAS_AGP
 303         int i;
 304
 305         for (i = 0; i < addrinarray; i++)
 306                 pmap_page_set_memattr(pages[i], VM_MEMATTR_UNCACHEABLE);
 307 #endif
 308         return 0;
 309 }
 310
 311 /**
 312  * Select the right pool or requested caching state and ttm flags. */
 313 static struct ttm_page_pool *ttm_get_pool(int flags,
 314                 enum ttm_caching_state cstate)
 315 {
 316         int pool_index;
 317
 318         if (cstate == tt_cached)
 319                 return NULL;
 320
 321         if (cstate == tt_wc)
 322                 pool_index = 0x0;
 323         else
 324                 pool_index = 0x1;
 325
 326         if (flags & TTM_PAGE_FLAG_DMA32)
 327                 pool_index |= 0x2;
 328
 329         return &_manager->pools[pool_index];
 330 }
 331
 332 /* set memory back to wb and free the pages. */
 333 static void ttm_pages_put(vm_page_t *pages, unsigned npages)
 334 {
 335         unsigned i;
 336
 337         /* Our VM handles vm memattr automatically on the page free. */
 338         if (set_pages_array_wb(pages, npages))
 339                 printf("[TTM] Failed to set %d pages to wb!\n", npages);
 340         for (i = 0; i < npages; ++i)
 341                 ttm_vm_page_free(pages[i]);
 342 }
 343
 344 static void ttm_pool_update_free_locked(struct ttm_page_pool *pool,
 345                 unsigned freed_pages)
 346 {
 347         pool->npages -= freed_pages;
 348         pool->nfrees += freed_pages;
 349 }
 350
 351 /**
 352  * Free pages from pool.
 353  *
 354  * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC
 355  * number of pages in one go.
 356  *
 357  * @pool: to free the pages from
 358  * @free_all: If set to true will free all pages in pool
 359  **/
 360 static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free)
 361 {
 362         vm_page_t p, p1;
 363         vm_page_t *pages_to_free;
 364         unsigned freed_pages = 0,
 365                  npages_to_free = nr_free;
 366         unsigned i;
 367
 368         if (NUM_PAGES_TO_ALLOC < nr_free)
 369                 npages_to_free = NUM_PAGES_TO_ALLOC;
 370
 371         pages_to_free = malloc(npages_to_free * sizeof(vm_page_t),
 372             M_TEMP, M_WAITOK | M_ZERO);
 373
 374 restart:
 375         mtx_lock(&pool->lock);
 376
 377         TAILQ_FOREACH_REVERSE_SAFE(p, &pool->list, pglist, plinks.q, p1) {
 378                 if (freed_pages >= npages_to_free)
 379                         break;
 380
 381                 pages_to_free[freed_pages++] = p;
 382                 /* We can only remove NUM_PAGES_TO_ALLOC at a time. */
 383                 if (freed_pages >= NUM_PAGES_TO_ALLOC) {
 384                         /* remove range of pages from the pool */
 385                         for (i = 0; i < freed_pages; i++)
 386                                 TAILQ_REMOVE(&pool->list, pages_to_free[i], plinks.q);
 387
 388                         ttm_pool_update_free_locked(pool, freed_pages);
 389                         /**
 390                          * Because changing page caching is costly
 391                          * we unlock the pool to prevent stalling.
 392                          */
 393                         mtx_unlock(&pool->lock);
 394
 395                         ttm_pages_put(pages_to_free, freed_pages);
 396                         if (likely(nr_free != FREE_ALL_PAGES))
 397                                 nr_free -= freed_pages;
 398
 399                         if (NUM_PAGES_TO_ALLOC >= nr_free)
 400                                 npages_to_free = nr_free;
 401                         else
 402                                 npages_to_free = NUM_PAGES_TO_ALLOC;
 403
 404                         freed_pages = 0;
 405
 406                         /* free all so restart the processing */
 407                         if (nr_free)
 408                                 goto restart;
 409
 410                         /* Not allowed to fall through or break because
 411                          * following context is inside spinlock while we are
 412                          * outside here.
 413                          */
 414                         goto out;
 415
 416                 }
 417         }
 418
 419         /* remove range of pages from the pool */
 420         if (freed_pages) {
 421                 for (i = 0; i < freed_pages; i++)
 422                         TAILQ_REMOVE(&pool->list, pages_to_free[i], plinks.q);
 423
 424                 ttm_pool_update_free_locked(pool, freed_pages);
 425                 nr_free -= freed_pages;
 426         }
 427
 428         mtx_unlock(&pool->lock);
 429
 430         if (freed_pages)
 431                 ttm_pages_put(pages_to_free, freed_pages);
 432 out:
 433         free(pages_to_free, M_TEMP);
 434         return nr_free;
 435 }
 436
 437 /* Get good estimation how many pages are free in pools */
 438 static int ttm_pool_get_num_unused_pages(void)
 439 {
 440         unsigned i;
 441         int total = 0;
 442         for (i = 0; i < NUM_POOLS; ++i)
 443                 total += _manager->pools[i].npages;
 444
 445         return total;
 446 }
 447
 448 /**
 449  * Callback for mm to request pool to reduce number of page held.
 450  */
 451 static int ttm_pool_mm_shrink(void *arg)
 452 {
 453         static unsigned int start_pool = 0;
 454         unsigned i;
 455         unsigned pool_offset = atomic_fetchadd_int(&start_pool, 1);
 456         struct ttm_page_pool *pool;
 457         int shrink_pages = 100; /* XXXKIB */
 458
 459         pool_offset = pool_offset % NUM_POOLS;
 460         /* select start pool in round robin fashion */
 461         for (i = 0; i < NUM_POOLS; ++i) {
 462                 unsigned nr_free = shrink_pages;
 463                 if (shrink_pages == 0)
 464                         break;
 465                 pool = &_manager->pools[(i + pool_offset)%NUM_POOLS];
 466                 shrink_pages = ttm_page_pool_free(pool, nr_free);
 467         }
 468         /* return estimated number of unused pages in pool */
 469         return ttm_pool_get_num_unused_pages();
 470 }
 471
 472 static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager)
 473 {
 474
 475         manager->lowmem_handler = EVENTHANDLER_REGISTER(vm_lowmem,
 476             ttm_pool_mm_shrink, manager, EVENTHANDLER_PRI_ANY);
 477 }
 478
 479 static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager)
 480 {
 481
 482         EVENTHANDLER_DEREGISTER(vm_lowmem, manager->lowmem_handler);
 483 }
 484
 485 static int ttm_set_pages_caching(vm_page_t *pages,
 486                 enum ttm_caching_state cstate, unsigned cpages)
 487 {
 488         int r = 0;
 489         /* Set page caching */
 490         switch (cstate) {
 491         case tt_uncached:
 492                 r = set_pages_array_uc(pages, cpages);
 493                 if (r)
 494                         printf("[TTM] Failed to set %d pages to uc!\n", cpages);
 495                 break;
 496         case tt_wc:
 497                 r = set_pages_array_wc(pages, cpages);
 498                 if (r)
 499                         printf("[TTM] Failed to set %d pages to wc!\n", cpages);
 500                 break;
 501         default:
 502                 break;
 503         }
 504         return r;
 505 }
 506
 507 /**
 508  * Free pages the pages that failed to change the caching state. If there is
 509  * any pages that have changed their caching state already put them to the
 510  * pool.
 511  */
 512 static void ttm_handle_caching_state_failure(struct pglist *pages,
 513                 int ttm_flags, enum ttm_caching_state cstate,
 514                 vm_page_t *failed_pages, unsigned cpages)
 515 {
 516         unsigned i;
 517         /* Failed pages have to be freed */
 518         for (i = 0; i < cpages; ++i) {
 519                 TAILQ_REMOVE(pages, failed_pages[i], plinks.q);
 520                 ttm_vm_page_free(failed_pages[i]);
 521         }
 522 }
 523
 524 /**
 525  * Allocate new pages with correct caching.
 526  *
 527  * This function is reentrant if caller updates count depending on number of
 528  * pages returned in pages array.
 529  */
 530 static int ttm_alloc_new_pages(struct pglist *pages, int ttm_alloc_flags,
 531                 int ttm_flags, enum ttm_caching_state cstate, unsigned count)
 532 {
 533         vm_page_t *caching_array;
 534         vm_page_t p;
 535         int r = 0;
 536         unsigned i, cpages;
 537         unsigned max_cpages = min(count,
 538                         (unsigned)(PAGE_SIZE/sizeof(vm_page_t)));
 539
 540         /* allocate array for page caching change */
 541         caching_array = malloc(max_cpages * sizeof(vm_page_t), M_TEMP,
 542             M_WAITOK | M_ZERO);
 543
 544         for (i = 0, cpages = 0; i < count; ++i) {
 545                 p = ttm_vm_page_alloc(ttm_alloc_flags, cstate);
 546                 if (!p) {
 547                         printf("[TTM] Unable to get page %u\n", i);
 548
 549                         /* store already allocated pages in the pool after
 550                          * setting the caching state */
 551                         if (cpages) {
 552                                 r = ttm_set_pages_caching(caching_array,
 553                                                           cstate, cpages);
 554                                 if (r)
 555                                         ttm_handle_caching_state_failure(pages,
 556                                                 ttm_flags, cstate,
 557                                                 caching_array, cpages);
 558                         }
 559                         r = -ENOMEM;
 560                         goto out;
 561                 }
 562
 563 #ifdef CONFIG_HIGHMEM /* KIB: nop */
 564                 /* gfp flags of highmem page should never be dma32 so we
 565                  * we should be fine in such case
 566                  */
 567                 if (!PageHighMem(p))
 568 #endif
 569                 {
 570                         caching_array[cpages++] = p;
 571                         if (cpages == max_cpages) {
 572
 573                                 r = ttm_set_pages_caching(caching_array,
 574                                                 cstate, cpages);
 575                                 if (r) {
 576                                         ttm_handle_caching_state_failure(pages,
 577                                                 ttm_flags, cstate,
 578                                                 caching_array, cpages);
 579                                         goto out;
 580                                 }
 581                                 cpages = 0;
 582                         }
 583                 }
 584
 585                 TAILQ_INSERT_HEAD(pages, p, plinks.q);
 586         }
 587
 588         if (cpages) {
 589                 r = ttm_set_pages_caching(caching_array, cstate, cpages);
 590                 if (r)
 591                         ttm_handle_caching_state_failure(pages,
 592                                         ttm_flags, cstate,
 593                                         caching_array, cpages);
 594         }
 595 out:
 596         free(caching_array, M_TEMP);
 597
 598         return r;
 599 }
 600
 601 /**
 602  * Fill the given pool if there aren't enough pages and the requested number of
 603  * pages is small.
 604  */
 605 static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
 606     int ttm_flags, enum ttm_caching_state cstate, unsigned count)
 607 {
 608         vm_page_t p;
 609         int r;
 610         unsigned cpages = 0;
 611         /**
 612          * Only allow one pool fill operation at a time.
 613          * If pool doesn't have enough pages for the allocation new pages are
 614          * allocated from outside of pool.
 615          */
 616         if (pool->fill_lock)
 617                 return;
 618
 619         pool->fill_lock = true;
 620
 621         /* If allocation request is small and there are not enough
 622          * pages in a pool we fill the pool up first. */
 623         if (count < _manager->options.small
 624                 && count > pool->npages) {
 625                 struct pglist new_pages;
 626                 unsigned alloc_size = _manager->options.alloc_size;
 627
 628                 /**
 629                  * Can't change page caching if in irqsave context. We have to
 630                  * drop the pool->lock.
 631                  */
 632                 mtx_unlock(&pool->lock);
 633
 634                 TAILQ_INIT(&new_pages);
 635                 r = ttm_alloc_new_pages(&new_pages, pool->ttm_page_alloc_flags,
 636                     ttm_flags, cstate, alloc_size);
 637                 mtx_lock(&pool->lock);
 638
 639                 if (!r) {
 640                         TAILQ_CONCAT(&pool->list, &new_pages, plinks.q);
 641                         ++pool->nrefills;
 642                         pool->npages += alloc_size;
 643                 } else {
 644                         printf("[TTM] Failed to fill pool (%p)\n", pool);
 645                         /* If we have any pages left put them to the pool. */
 646                         TAILQ_FOREACH(p, &pool->list, plinks.q) {
 647                                 ++cpages;
 648                         }
 649                         TAILQ_CONCAT(&pool->list, &new_pages, plinks.q);
 650                         pool->npages += cpages;
 651                 }
 652
 653         }
 654         pool->fill_lock = false;
 655 }
 656
 657 /**
 658  * Cut 'count' number of pages from the pool and put them on the return list.
 659  *
 660  * @return count of pages still required to fulfill the request.
 661  */
 662 static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool,
 663                                         struct pglist *pages,
 664                                         int ttm_flags,
 665                                         enum ttm_caching_state cstate,
 666                                         unsigned count)
 667 {
 668         vm_page_t p;
 669         unsigned i;
 670
 671         mtx_lock(&pool->lock);
 672         ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count);
 673
 674         if (count >= pool->npages) {
 675                 /* take all pages from the pool */
 676                 TAILQ_CONCAT(pages, &pool->list, plinks.q);
 677                 count -= pool->npages;
 678                 pool->npages = 0;
 679                 goto out;
 680         }
 681         for (i = 0; i < count; i++) {
 682                 p = TAILQ_FIRST(&pool->list);
 683                 TAILQ_REMOVE(&pool->list, p, plinks.q);
 684                 TAILQ_INSERT_TAIL(pages, p, plinks.q);
 685         }
 686         pool->npages -= count;
 687         count = 0;
 688 out:
 689         mtx_unlock(&pool->lock);
 690         return count;
 691 }
 692
 693 /* Put all pages in pages list to correct pool to wait for reuse */
 694 static void ttm_put_pages(vm_page_t *pages, unsigned npages, int flags,
 695                           enum ttm_caching_state cstate)
 696 {
 697         struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
 698         unsigned i;
 699
 700         if (pool == NULL) {
 701                 /* No pool for this memory type so free the pages */
 702                 for (i = 0; i < npages; i++) {
 703                         if (pages[i]) {
 704                                 ttm_vm_page_free(pages[i]);
 705                                 pages[i] = NULL;
 706                         }
 707                 }
 708                 return;
 709         }
 710
 711         mtx_lock(&pool->lock);
 712         for (i = 0; i < npages; i++) {
 713                 if (pages[i]) {
 714                         TAILQ_INSERT_TAIL(&pool->list, pages[i], plinks.q);
 715                         pages[i] = NULL;
 716                         pool->npages++;
 717                 }
 718         }
 719         /* Check that we don't go over the pool limit */
 720         npages = 0;
 721         if (pool->npages > _manager->options.max_size) {
 722                 npages = pool->npages - _manager->options.max_size;
 723                 /* free at least NUM_PAGES_TO_ALLOC number of pages
 724                  * to reduce calls to set_memory_wb */
 725                 if (npages < NUM_PAGES_TO_ALLOC)
 726                         npages = NUM_PAGES_TO_ALLOC;
 727         }
 728         mtx_unlock(&pool->lock);
 729         if (npages)
 730                 ttm_page_pool_free(pool, npages);
 731 }
 732
 733 /*
 734  * On success pages list will hold count number of correctly
 735  * cached pages.
 736  */
 737 static int ttm_get_pages(vm_page_t *pages, unsigned npages, int flags,
 738                          enum ttm_caching_state cstate)
 739 {
 740         struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
 741         struct pglist plist;
 742         vm_page_t p = NULL;
 743         int gfp_flags;
 744         unsigned count;
 745         int r;
 746
 747         /* No pool for cached pages */
 748         if (pool == NULL) {
 749                 for (r = 0; r < npages; ++r) {
 750                         p = ttm_vm_page_alloc(flags, cstate);
 751                         if (!p) {
 752                                 printf("[TTM] Unable to allocate page\n");
 753                                 return -ENOMEM;
 754                         }
 755                         pages[r] = p;
 756                 }
 757                 return 0;
 758         }
 759
 760         /* combine zero flag to pool flags */
 761         gfp_flags = flags | pool->ttm_page_alloc_flags;
 762
 763         /* First we take pages from the pool */
 764         TAILQ_INIT(&plist);
 765         npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages);
 766         count = 0;
 767         TAILQ_FOREACH(p, &plist, plinks.q) {
 768                 pages[count++] = p;
 769         }
 770
 771         /* clear the pages coming from the pool if requested */
 772         if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) {
 773                 TAILQ_FOREACH(p, &plist, plinks.q) {
 774                         pmap_zero_page(p);
 775                 }
 776         }
 777
 778         /* If pool didn't have enough pages allocate new one. */
 779         if (npages > 0) {
 780                 /* ttm_alloc_new_pages doesn't reference pool so we can run
 781                  * multiple requests in parallel.
 782                  **/
 783                 TAILQ_INIT(&plist);
 784                 r = ttm_alloc_new_pages(&plist, gfp_flags, flags, cstate,
 785                     npages);
 786                 TAILQ_FOREACH(p, &plist, plinks.q) {
 787                         pages[count++] = p;
 788                 }
 789                 if (r) {
 790                         /* If there is any pages in the list put them back to
 791                          * the pool. */
 792                         printf("[TTM] Failed to allocate extra pages for large request\n");
 793                         ttm_put_pages(pages, count, flags, cstate);
 794                         return r;
 795                 }
 796         }
 797
 798         return 0;
 799 }
 800
 801 static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags,
 802                                       char *name)
 803 {
 804         mtx_init(&pool->lock, "ttmpool", NULL, MTX_DEF);
 805         pool->fill_lock = false;
 806         TAILQ_INIT(&pool->list);
 807         pool->npages = pool->nfrees = 0;
 808         pool->ttm_page_alloc_flags = flags;
 809         pool->name = name;
 810 }
 811
 812 int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages)
 813 {
 814
 815         if (_manager != NULL)
 816                 printf("[TTM] manager != NULL\n");
 817         printf("[TTM] Initializing pool allocator\n");
 818
 819         _manager = malloc(sizeof(*_manager), M_TTM_POOLMGR, M_WAITOK | M_ZERO);
 820
 821         ttm_page_pool_init_locked(&_manager->wc_pool, 0, "wc");
 822         ttm_page_pool_init_locked(&_manager->uc_pool, 0, "uc");
 823         ttm_page_pool_init_locked(&_manager->wc_pool_dma32,
 824             TTM_PAGE_FLAG_DMA32, "wc dma");
 825         ttm_page_pool_init_locked(&_manager->uc_pool_dma32,
 826             TTM_PAGE_FLAG_DMA32, "uc dma");
 827
 828         _manager->options.max_size = max_pages;
 829         _manager->options.small = SMALL_ALLOCATION;
 830         _manager->options.alloc_size = NUM_PAGES_TO_ALLOC;
 831
 832         refcount_init(&_manager->kobj_ref, 1);
 833         ttm_pool_mm_shrink_init(_manager);
 834
 835         return 0;
 836 }
 837
 838 void ttm_page_alloc_fini(void)
 839 {
 840         int i;
 841
 842         printf("[TTM] Finalizing pool allocator\n");
 843         ttm_pool_mm_shrink_fini(_manager);
 844
 845         for (i = 0; i < NUM_POOLS; ++i)
 846                 ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES);
 847
 848         if (refcount_release(&_manager->kobj_ref))
 849                 ttm_pool_kobj_release(_manager);
 850         _manager = NULL;
 851 }
 852
 853 int ttm_pool_populate(struct ttm_tt *ttm)
 854 {
 855         struct ttm_mem_global *mem_glob = ttm->glob->mem_glob;
 856         unsigned i;
 857         int ret;
 858
 859         if (ttm->state != tt_unpopulated)
 860                 return 0;
 861
 862         for (i = 0; i < ttm->num_pages; ++i) {
 863                 ret = ttm_get_pages(&ttm->pages[i], 1,
 864                                     ttm->page_flags,
 865                                     ttm->caching_state);
 866                 if (ret != 0) {
 867                         ttm_pool_unpopulate(ttm);
 868                         return -ENOMEM;
 869                 }
 870
 871                 ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
 872                                                 false, false);
 873                 if (unlikely(ret != 0)) {
 874                         ttm_pool_unpopulate(ttm);
 875                         return -ENOMEM;
 876                 }
 877         }
 878
 879         if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) {
 880                 ret = ttm_tt_swapin(ttm);
 881                 if (unlikely(ret != 0)) {
 882                         ttm_pool_unpopulate(ttm);
 883                         return ret;
 884                 }
 885         }
 886
 887         ttm->state = tt_unbound;
 888         return 0;
 889 }
 890
 891 void ttm_pool_unpopulate(struct ttm_tt *ttm)
 892 {
 893         unsigned i;
 894
 895         for (i = 0; i < ttm->num_pages; ++i) {
 896                 if (ttm->pages[i]) {
 897                         ttm_mem_global_free_page(ttm->glob->mem_glob,
 898                                                  ttm->pages[i]);
 899                         ttm_put_pages(&ttm->pages[i], 1,
 900                                       ttm->page_flags,
 901                                       ttm->caching_state);
 902                 }
 903         }
 904         ttm->state = tt_unpopulated;
 905 }
 906
 907 #if 0
 908 /* XXXKIB sysctl */
 909 int ttm_page_alloc_debugfs(struct seq_file *m, void *data)
 910 {
 911         struct ttm_page_pool *p;
 912         unsigned i;
 913         char *h[] = {"pool", "refills", "pages freed", "size"};
 914         if (!_manager) {
 915                 seq_printf(m, "No pool allocator running.\n");
 916                 return 0;
 917         }
 918         seq_printf(m, "%6s %12s %13s %8s\n",
 919                         h[0], h[1], h[2], h[3]);
 920         for (i = 0; i < NUM_POOLS; ++i) {
 921                 p = &_manager->pools[i];
 922
 923                 seq_printf(m, "%6s %12ld %13ld %8d\n",
 924                                 p->name, p->nrefills,
 925                                 p->nfrees, p->npages);
 926         }
 927         return 0;
 928 }
 929 #endif