sys/dev/mlx5/mlx5_core/mlx5_pagealloc.c

   1 /*-
   2  * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
  14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  16  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
  17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  23  * SUCH DAMAGE.
  24  *
  25  * $FreeBSD$
  26  */
  27
  28 #include <linux/kernel.h>
  29 #include <linux/module.h>
  30 #include <dev/mlx5/driver.h>
  31 #include "mlx5_core.h"
  32
  33 struct mlx5_pages_req {
  34         struct mlx5_core_dev *dev;
  35         u16     func_id;
  36         s32     npages;
  37         struct work_struct work;
  38 };
  39
  40 struct fw_page {
  41         struct rb_node          rb_node;
  42         u64                     addr;
  43         struct page            *page;
  44         u16                     func_id;
  45         unsigned long           bitmask;
  46         struct list_head        list;
  47         unsigned                free_count;
  48 };
  49
  50 struct mlx5_manage_pages_inbox {
  51         struct mlx5_inbox_hdr   hdr;
  52         __be16                  rsvd;
  53         __be16                  func_id;
  54         __be32                  num_entries;
  55         __be64                  pas[0];
  56 };
  57
  58 struct mlx5_manage_pages_outbox {
  59         struct mlx5_outbox_hdr  hdr;
  60         __be32                  num_entries;
  61         u8                      rsvd[4];
  62         __be64                  pas[0];
  63 };
  64
  65 enum {
  66         MAX_RECLAIM_TIME_MSECS  = 5000,
  67 };
  68
  69 enum {
  70         MLX5_MAX_RECLAIM_TIME_MILI      = 5000,
  71         MLX5_NUM_4K_IN_PAGE             = PAGE_SIZE / MLX5_ADAPTER_PAGE_SIZE,
  72 };
  73
  74 static int insert_page(struct mlx5_core_dev *dev, u64 addr, struct page *page, u16 func_id)
  75 {
  76         struct rb_root *root = &dev->priv.page_root;
  77         struct rb_node **new = &root->rb_node;
  78         struct rb_node *parent = NULL;
  79         struct fw_page *nfp;
  80         struct fw_page *tfp;
  81         int i;
  82
  83         while (*new) {
  84                 parent = *new;
  85                 tfp = rb_entry(parent, struct fw_page, rb_node);
  86                 if (tfp->addr < addr)
  87                         new = &parent->rb_left;
  88                 else if (tfp->addr > addr)
  89                         new = &parent->rb_right;
  90                 else
  91                         return -EEXIST;
  92         }
  93
  94         nfp = kzalloc(sizeof(*nfp), GFP_KERNEL);
  95
  96         nfp->addr = addr;
  97         nfp->page = page;
  98         nfp->func_id = func_id;
  99         nfp->free_count = MLX5_NUM_4K_IN_PAGE;
 100         for (i = 0; i < MLX5_NUM_4K_IN_PAGE; i++)
 101                 set_bit(i, &nfp->bitmask);
 102
 103         rb_link_node(&nfp->rb_node, parent, new);
 104         rb_insert_color(&nfp->rb_node, root);
 105         list_add(&nfp->list, &dev->priv.free_list);
 106
 107         return 0;
 108 }
 109
 110 static struct fw_page *find_fw_page(struct mlx5_core_dev *dev, u64 addr)
 111 {
 112         struct rb_root *root = &dev->priv.page_root;
 113         struct rb_node *tmp = root->rb_node;
 114         struct fw_page *result = NULL;
 115         struct fw_page *tfp;
 116
 117         while (tmp) {
 118                 tfp = rb_entry(tmp, struct fw_page, rb_node);
 119                 if (tfp->addr < addr) {
 120                         tmp = tmp->rb_left;
 121                 } else if (tfp->addr > addr) {
 122                         tmp = tmp->rb_right;
 123                 } else {
 124                         result = tfp;
 125                         break;
 126                 }
 127         }
 128
 129         return result;
 130 }
 131
 132 static int mlx5_cmd_query_pages(struct mlx5_core_dev *dev, u16 *func_id,
 133                                 s32 *npages, int boot)
 134 {
 135         u32 in[MLX5_ST_SZ_DW(query_pages_in)];
 136         u32 out[MLX5_ST_SZ_DW(query_pages_out)];
 137         int err;
 138
 139         memset(in, 0, sizeof(in));
 140
 141         MLX5_SET(query_pages_in, in, opcode, MLX5_CMD_OP_QUERY_PAGES);
 142         MLX5_SET(query_pages_in, in, op_mod,
 143                  boot ? MLX5_BOOT_PAGES : MLX5_INIT_PAGES);
 144
 145         memset(out, 0, sizeof(out));
 146         err = mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
 147         if (err)
 148                 return err;
 149
 150         *npages = MLX5_GET(query_pages_out, out, num_pages);
 151         *func_id = MLX5_GET(query_pages_out, out, function_id);
 152
 153         return 0;
 154 }
 155
 156 static int alloc_4k(struct mlx5_core_dev *dev, u64 *addr)
 157 {
 158         struct fw_page *fp;
 159         unsigned n;
 160
 161         if (list_empty(&dev->priv.free_list))
 162                 return -ENOMEM;
 163
 164         fp = list_entry(dev->priv.free_list.next, struct fw_page, list);
 165         n = find_first_bit(&fp->bitmask, 8 * sizeof(fp->bitmask));
 166         if (n >= MLX5_NUM_4K_IN_PAGE) {
 167                 mlx5_core_warn(dev, "alloc 4k bug\n");
 168                 return -ENOENT;
 169         }
 170         clear_bit(n, &fp->bitmask);
 171         fp->free_count--;
 172         if (!fp->free_count)
 173                 list_del(&fp->list);
 174
 175         *addr = fp->addr + n * MLX5_ADAPTER_PAGE_SIZE;
 176
 177         return 0;
 178 }
 179
 180 static void free_4k(struct mlx5_core_dev *dev, u64 addr)
 181 {
 182         struct fw_page *fwp;
 183         int n;
 184
 185         fwp = find_fw_page(dev, addr & PAGE_MASK);
 186         if (!fwp) {
 187                 mlx5_core_warn(dev, "page not found\n");
 188                 return;
 189         }
 190
 191         n = (addr & ~PAGE_MASK) >> MLX5_ADAPTER_PAGE_SHIFT;
 192         fwp->free_count++;
 193         set_bit(n, &fwp->bitmask);
 194         if (fwp->free_count == MLX5_NUM_4K_IN_PAGE) {
 195                 rb_erase(&fwp->rb_node, &dev->priv.page_root);
 196                 if (fwp->free_count != 1)
 197                         list_del(&fwp->list);
 198                 dma_unmap_page(&dev->pdev->dev, addr & PAGE_MASK, PAGE_SIZE,
 199                                DMA_BIDIRECTIONAL);
 200                 __free_page(fwp->page);
 201                 kfree(fwp);
 202         } else if (fwp->free_count == 1) {
 203                 list_add(&fwp->list, &dev->priv.free_list);
 204         }
 205 }
 206
 207 static int alloc_system_page(struct mlx5_core_dev *dev, u16 func_id)
 208 {
 209         struct page *page;
 210         u64 addr;
 211         int err;
 212
 213         page = alloc_page(GFP_HIGHUSER);
 214         if (!page) {
 215                 mlx5_core_warn(dev, "failed to allocate page\n");
 216                 return -ENOMEM;
 217         }
 218         addr = dma_map_page(&dev->pdev->dev, page, 0,
 219                             PAGE_SIZE, DMA_BIDIRECTIONAL);
 220         if (dma_mapping_error(&dev->pdev->dev, addr)) {
 221                 mlx5_core_warn(dev, "failed dma mapping page\n");
 222                 err = -ENOMEM;
 223                 goto out_alloc;
 224         }
 225         err = insert_page(dev, addr, page, func_id);
 226         if (err) {
 227                 mlx5_core_err(dev, "failed to track allocated page\n");
 228                 goto out_mapping;
 229         }
 230
 231         return 0;
 232
 233 out_mapping:
 234         dma_unmap_page(&dev->pdev->dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
 235
 236 out_alloc:
 237         __free_page(page);
 238         return err;
 239 }
 240 static int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages,
 241                       int notify_fail)
 242 {
 243         struct mlx5_manage_pages_inbox *in;
 244         struct mlx5_manage_pages_outbox out;
 245         struct mlx5_manage_pages_inbox *nin;
 246         int inlen;
 247         u64 addr;
 248         int err;
 249         int i;
 250
 251         inlen = sizeof(*in) + npages * sizeof(in->pas[0]);
 252         in = mlx5_vzalloc(inlen);
 253         if (!in) {
 254                 mlx5_core_warn(dev, "vzalloc failed %d\n", inlen);
 255                 return -ENOMEM;
 256         }
 257         memset(&out, 0, sizeof(out));
 258
 259         for (i = 0; i < npages; i++) {
 260 retry:
 261                 err = alloc_4k(dev, &addr);
 262                 if (err) {
 263                         if (err == -ENOMEM)
 264                                 err = alloc_system_page(dev, func_id);
 265                         if (err)
 266                                 goto out_4k;
 267
 268                         goto retry;
 269                 }
 270                 in->pas[i] = cpu_to_be64(addr);
 271         }
 272
 273         in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES);
 274         in->hdr.opmod = cpu_to_be16(MLX5_PAGES_GIVE);
 275         in->func_id = cpu_to_be16(func_id);
 276         in->num_entries = cpu_to_be32(npages);
 277         err = mlx5_cmd_exec(dev, in, inlen, &out, sizeof(out));
 278         if (err) {
 279                 mlx5_core_warn(dev, "func_id 0x%x, npages %d, err %d\n",
 280                                func_id, npages, err);
 281                 goto out_alloc;
 282         }
 283         dev->priv.fw_pages += npages;
 284
 285         if (out.hdr.status) {
 286                 err = mlx5_cmd_status_to_err(&out.hdr);
 287                 if (err) {
 288                         mlx5_core_warn(dev, "func_id 0x%x, npages %d, status %d\n",
 289                                        func_id, npages, out.hdr.status);
 290                         goto out_alloc;
 291                 }
 292         }
 293
 294         mlx5_core_dbg(dev, "err %d\n", err);
 295
 296         goto out_free;
 297
 298 out_alloc:
 299         if (notify_fail) {
 300                 nin = kzalloc(sizeof(*nin), GFP_KERNEL);
 301                 memset(&out, 0, sizeof(out));
 302                 nin->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES);
 303                 nin->hdr.opmod = cpu_to_be16(MLX5_PAGES_CANT_GIVE);
 304                 if (mlx5_cmd_exec(dev, nin, sizeof(*nin), &out, sizeof(out)))
 305                         mlx5_core_warn(dev, "page notify failed\n");
 306                 kfree(nin);
 307         }
 308
 309 out_4k:
 310         for (i--; i >= 0; i--)
 311                 free_4k(dev, be64_to_cpu(in->pas[i]));
 312 out_free:
 313         kvfree(in);
 314         return err;
 315 }
 316
 317 static int reclaim_pages(struct mlx5_core_dev *dev, u32 func_id, int npages,
 318                          int *nclaimed)
 319 {
 320         struct mlx5_manage_pages_inbox   in;
 321         struct mlx5_manage_pages_outbox *out;
 322         int num_claimed;
 323         int outlen;
 324         u64 addr;
 325         int err;
 326         int i;
 327
 328         if (nclaimed)
 329                 *nclaimed = 0;
 330
 331         memset(&in, 0, sizeof(in));
 332         outlen = sizeof(*out) + npages * sizeof(out->pas[0]);
 333         out = mlx5_vzalloc(outlen);
 334         if (!out)
 335                 return -ENOMEM;
 336
 337         in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES);
 338         in.hdr.opmod = cpu_to_be16(MLX5_PAGES_TAKE);
 339         in.func_id = cpu_to_be16(func_id);
 340         in.num_entries = cpu_to_be32(npages);
 341         mlx5_core_dbg(dev, "npages %d, outlen %d\n", npages, outlen);
 342         err = mlx5_cmd_exec(dev, &in, sizeof(in), out, outlen);
 343         if (err) {
 344                 mlx5_core_err(dev, "failed reclaiming pages\n");
 345                 goto out_free;
 346         }
 347
 348         if (out->hdr.status) {
 349                 err = mlx5_cmd_status_to_err(&out->hdr);
 350                 goto out_free;
 351         }
 352
 353         num_claimed = be32_to_cpu(out->num_entries);
 354         if (nclaimed)
 355                 *nclaimed = num_claimed;
 356
 357         dev->priv.fw_pages -= num_claimed;
 358
 359         for (i = 0; i < num_claimed; i++) {
 360                 addr = be64_to_cpu(out->pas[i]);
 361                 free_4k(dev, addr);
 362         }
 363
 364 out_free:
 365         kvfree(out);
 366         return err;
 367 }
 368
 369 static void pages_work_handler(struct work_struct *work)
 370 {
 371         struct mlx5_pages_req *req = container_of(work, struct mlx5_pages_req, work);
 372         struct mlx5_core_dev *dev = req->dev;
 373         int err = 0;
 374
 375         if (req->npages < 0)
 376                 err = reclaim_pages(dev, req->func_id, -1 * req->npages, NULL);
 377         else if (req->npages > 0)
 378                 err = give_pages(dev, req->func_id, req->npages, 1);
 379
 380         if (err)
 381                 mlx5_core_warn(dev, "%s fail %d\n",
 382                                req->npages < 0 ? "reclaim" : "give", err);
 383
 384         kfree(req);
 385 }
 386
 387 void mlx5_core_req_pages_handler(struct mlx5_core_dev *dev, u16 func_id,
 388                                  s32 npages)
 389 {
 390         struct mlx5_pages_req *req;
 391
 392         req = kzalloc(sizeof(*req), GFP_ATOMIC);
 393         if (!req) {
 394                 mlx5_core_warn(dev, "failed to allocate pages request\n");
 395                 return;
 396         }
 397
 398         req->dev = dev;
 399         req->func_id = func_id;
 400         req->npages = npages;
 401         INIT_WORK(&req->work, pages_work_handler);
 402         if (!queue_work(dev->priv.pg_wq, &req->work))
 403                 mlx5_core_warn(dev, "failed to queue pages handler work\n");
 404 }
 405
 406 int mlx5_satisfy_startup_pages(struct mlx5_core_dev *dev, int boot)
 407 {
 408         u16 uninitialized_var(func_id);
 409         s32 uninitialized_var(npages);
 410         int err;
 411
 412         err = mlx5_cmd_query_pages(dev, &func_id, &npages, boot);
 413         if (err)
 414                 return err;
 415
 416         mlx5_core_dbg(dev, "requested %d %s pages for func_id 0x%x\n",
 417                       npages, boot ? "boot" : "init", func_id);
 418
 419         return give_pages(dev, func_id, npages, 0);
 420 }
 421
 422 enum {
 423         MLX5_BLKS_FOR_RECLAIM_PAGES = 12
 424 };
 425
 426 static int optimal_reclaimed_pages(void)
 427 {
 428         struct mlx5_cmd_prot_block *block;
 429         struct mlx5_cmd_layout *lay;
 430         int ret;
 431
 432         ret = (sizeof(lay->out) + MLX5_BLKS_FOR_RECLAIM_PAGES * sizeof(block->data) -
 433                sizeof(struct mlx5_manage_pages_outbox)) /
 434                FIELD_SIZEOF(struct mlx5_manage_pages_outbox, pas[0]);
 435
 436         return ret;
 437 }
 438
 439 int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev)
 440 {
 441         int end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS);
 442         struct fw_page *fwp;
 443         struct rb_node *p;
 444         int nclaimed = 0;
 445         int err;
 446
 447         do {
 448                 p = rb_first(&dev->priv.page_root);
 449                 if (p) {
 450                         fwp = rb_entry(p, struct fw_page, rb_node);
 451                         err = reclaim_pages(dev, fwp->func_id,
 452                                             optimal_reclaimed_pages(),
 453                                             &nclaimed);
 454                         if (err) {
 455                                 mlx5_core_warn(dev, "failed reclaiming pages (%d)\n",
 456                                                err);
 457                                 return err;
 458                         }
 459                         if (nclaimed)
 460                                 end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS);
 461                 }
 462                 if (time_after(jiffies, end)) {
 463                         mlx5_core_warn(dev, "FW did not return all pages. giving up...\n");
 464                         break;
 465                 }
 466         } while (p);
 467
 468         return 0;
 469 }
 470
 471 void mlx5_pagealloc_init(struct mlx5_core_dev *dev)
 472 {
 473         dev->priv.page_root = RB_ROOT;
 474         INIT_LIST_HEAD(&dev->priv.free_list);
 475 }
 476
 477 void mlx5_pagealloc_cleanup(struct mlx5_core_dev *dev)
 478 {
 479         /* nothing */
 480 }
 481
 482 int mlx5_pagealloc_start(struct mlx5_core_dev *dev)
 483 {
 484         dev->priv.pg_wq = create_singlethread_workqueue("mlx5_page_allocator");
 485         if (!dev->priv.pg_wq)
 486                 return -ENOMEM;
 487
 488         return 0;
 489 }
 490
 491 void mlx5_pagealloc_stop(struct mlx5_core_dev *dev)
 492 {
 493         destroy_workqueue(dev->priv.pg_wq);
 494 }